diff --git a/.changeset/block-metadata-proto-pollution.md b/.changeset/block-metadata-proto-pollution.md new file mode 100644 index 00000000000..aa38beab4a4 --- /dev/null +++ b/.changeset/block-metadata-proto-pollution.md @@ -0,0 +1,5 @@ +--- +"@trigger.dev/core": patch +--- + +Prevent prototype pollution when applying run metadata operations or reconstructing nested telemetry attributes, while preserving legitimate `constructor` and `prototype` fields. diff --git a/.changeset/cli-login-poll-window.md b/.changeset/cli-login-poll-window.md new file mode 100644 index 00000000000..e0453895a50 --- /dev/null +++ b/.changeset/cli-login-poll-window.md @@ -0,0 +1,6 @@ +--- +"@trigger.dev/core": patch +"trigger.dev": patch +--- + +Require explicit browser approval for CLI and MCP login, with resilient polling while approval is pending. diff --git a/.changeset/worker-id-deployment-friendly-id.md b/.changeset/worker-id-deployment-friendly-id.md new file mode 100644 index 00000000000..2e62fee4b97 --- /dev/null +++ b/.changeset/worker-id-deployment-friendly-id.md @@ -0,0 +1,5 @@ +--- +"trigger.dev": patch +--- + +Deployed task telemetry now reports the deployment identifier (e.g. `deployment_abc123`) in the `worker.id` attribute, instead of an opaque internal value. Upgrade to get the readable identifier in your own OpenTelemetry exporters. diff --git a/.changeset/workload-deployment-token.md b/.changeset/workload-deployment-token.md new file mode 100644 index 00000000000..6a3c08cfc77 --- /dev/null +++ b/.changeset/workload-deployment-token.md @@ -0,0 +1,5 @@ +--- +"@trigger.dev/core": patch +--- + +Add helpers to mint and verify the deployment-scoped token used to authenticate run controllers to the platform. diff --git a/.env.example b/.env.example index f37a1a5194c..901fbd7b3e0 100644 --- a/.env.example +++ b/.env.example @@ -2,6 +2,7 @@ SESSION_SECRET=abcdef1234 MAGIC_LINK_SECRET=abcdef1234 ENCRYPTION_KEY=ae13021afef0819c3a307ad487071c06 # Must be a random 16 byte hex string. You can generate an encryption key by running `openssl rand -hex 16` in your terminal +MANAGED_WORKER_SECRET=abcdef1234 # Must match the supervisor's MANAGED_WORKER_SECRET LOGIN_ORIGIN=http://localhost:3030 DATABASE_URL=postgresql://postgres:postgres@localhost:5432/postgres?schema=public # This sets the URL used for direct connections to the database and should only be needed in limited circumstances diff --git a/.github/workflows/helm-prerelease.yml b/.github/workflows/helm-prerelease.yml index 37b909e415e..7d49868eeaf 100644 --- a/.github/workflows/helm-prerelease.yml +++ b/.github/workflows/helm-prerelease.yml @@ -52,12 +52,14 @@ jobs: - name: Lint Helm Chart run: | - helm lint ./hosting/k8s/helm/ + helm lint ./hosting/k8s/helm/ \ + --values ./hosting/k8s/helm/ci/lint-values.yaml - name: Render templates run: | helm template test-release ./hosting/k8s/helm/ \ --values ./hosting/k8s/helm/values.yaml \ + --values ./hosting/k8s/helm/ci/lint-values.yaml \ --output-dir ./helm-output - name: Validate manifests diff --git a/.github/workflows/release-helm.yml b/.github/workflows/release-helm.yml index a5afd8b1b24..0ce968ebc47 100644 --- a/.github/workflows/release-helm.yml +++ b/.github/workflows/release-helm.yml @@ -47,12 +47,14 @@ jobs: - name: Lint Helm Chart run: | - helm lint ./hosting/k8s/helm/ + helm lint ./hosting/k8s/helm/ \ + --values ./hosting/k8s/helm/ci/lint-values.yaml - name: Render templates run: | helm template test-release ./hosting/k8s/helm/ \ --values ./hosting/k8s/helm/values.yaml \ + --values ./hosting/k8s/helm/ci/lint-values.yaml \ --output-dir ./helm-output - name: Validate manifests diff --git a/.server-changes/otlp-ingestion-rate-limit.md b/.server-changes/otlp-ingestion-rate-limit.md new file mode 100644 index 00000000000..acb9c623122 --- /dev/null +++ b/.server-changes/otlp-ingestion-rate-limit.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: improvement +--- + +Added optional request rate limiting for telemetry ingestion endpoints. diff --git a/.server-changes/require-webapp-auth-secrets.md b/.server-changes/require-webapp-auth-secrets.md new file mode 100644 index 00000000000..52cb61d871a --- /dev/null +++ b/.server-changes/require-webapp-auth-secrets.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: breaking +--- + +Self-hosted deployments no longer ship shared default credentials; fresh installs generate their own. If yours still uses a previously published default, set a unique value before upgrading, or set `ALLOW_INSECURE_DEFAULT_SECRETS=true` to keep booting while you migrate. diff --git a/.server-changes/scope-deployment-background-worker-lookup.md b/.server-changes/scope-deployment-background-worker-lookup.md new file mode 100644 index 00000000000..74c2aa7ee6d --- /dev/null +++ b/.server-changes/scope-deployment-background-worker-lookup.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: fix +--- + +Background-worker deployment lookups are now scoped to the authenticated environment. diff --git a/.server-changes/scope-github-app-update-to-org.md b/.server-changes/scope-github-app-update-to-org.md new file mode 100644 index 00000000000..df417e52dd4 --- /dev/null +++ b/.server-changes/scope-github-app-update-to-org.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: fix +--- + +Updating a GitHub App installation from the callback flow is now scoped to your own organization, so an installation ID belonging to another organization can no longer be used to refresh that organization's installation record. The GitHub App installation session is also now single-use, so completing an installation callback invalidates its state and it can no longer be replayed. diff --git a/.server-changes/scope-schedule-and-env-var-writes.md b/.server-changes/scope-schedule-and-env-var-writes.md new file mode 100644 index 00000000000..3ddcd6a1aaa --- /dev/null +++ b/.server-changes/scope-schedule-and-env-var-writes.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: fix +--- + +Scope schedule and environment-variable writes to the caller's project and environment diff --git a/.server-changes/secure-compute-snapshot-callbacks.md b/.server-changes/secure-compute-snapshot-callbacks.md new file mode 100644 index 00000000000..5f362f3fa41 --- /dev/null +++ b/.server-changes/secure-compute-snapshot-callbacks.md @@ -0,0 +1,6 @@ +--- +area: supervisor +type: fix +--- + +Reject compute snapshot callbacks that do not match the snapshot request that created them. diff --git a/.server-changes/session-out-stream-private-auth.md b/.server-changes/session-out-stream-private-auth.md new file mode 100644 index 00000000000..bfc95c662f8 --- /dev/null +++ b/.server-changes/session-out-stream-private-auth.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: fix +--- + +Require secret-key authentication to initialize the session out (agent→client) stream, matching the append route. diff --git a/.server-changes/tighten-realtime-and-trace-sync.md b/.server-changes/tighten-realtime-and-trace-sync.md new file mode 100644 index 00000000000..51b4b2d416b --- /dev/null +++ b/.server-changes/tighten-realtime-and-trace-sync.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: fix +--- + +Live run and trace subscriptions now validate their identifiers more strictly and only return data from your own organization. diff --git a/.server-changes/tsql-window-function-allowlist.md b/.server-changes/tsql-window-function-allowlist.md new file mode 100644 index 00000000000..a56a881ae73 --- /dev/null +++ b/.server-changes/tsql-window-function-allowlist.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: fix +--- + +Window-function names in the query compiler are now validated against the allowlist, matching how other function calls are handled. diff --git a/.server-changes/workload-token-supervisor.md b/.server-changes/workload-token-supervisor.md new file mode 100644 index 00000000000..b7c82c88881 --- /dev/null +++ b/.server-changes/workload-token-supervisor.md @@ -0,0 +1,6 @@ +--- +area: supervisor +type: fix +--- + +Authenticate run controllers to the platform with a signed, deployment-scoped token. diff --git a/.server-changes/workload-token-webapp.md b/.server-changes/workload-token-webapp.md new file mode 100644 index 00000000000..b743c446308 --- /dev/null +++ b/.server-changes/workload-token-webapp.md @@ -0,0 +1,6 @@ +--- +area: webapp +type: fix +--- + +Verify that worker actions (starting, completing, and continuing a run, and reading its snapshots) target a run belonging to the caller's environment. diff --git a/apps/supervisor/.env.example b/apps/supervisor/.env.example index 4355a4eea30..f71cb88f3eb 100644 --- a/apps/supervisor/.env.example +++ b/apps/supervisor/.env.example @@ -1,8 +1,8 @@ # This needs to match the token of the worker group you want to connect to TRIGGER_WORKER_TOKEN= -# This needs to match the MANAGED_WORKER_SECRET env var on the webapp -MANAGED_WORKER_SECRET=managed-secret +# Must match the webapp's MANAGED_WORKER_SECRET. Generate with: openssl rand -hex 16 +MANAGED_WORKER_SECRET= # Point this at the webapp in prod TRIGGER_API_URL=http://localhost:3030 diff --git a/apps/supervisor/src/env.ts b/apps/supervisor/src/env.ts index 32e669b5914..6c954b4e76a 100644 --- a/apps/supervisor/src/env.ts +++ b/apps/supervisor/src/env.ts @@ -14,8 +14,17 @@ export const Env = z // Required settings TRIGGER_API_URL: z.string().url(), - TRIGGER_WORKER_TOKEN: z.string(), // accepts file:// path to read from a file + TRIGGER_WORKER_TOKEN: z.string().min(1), // accepts file:// path to read from a file MANAGED_WORKER_SECRET: z.string(), + + // Deployment token: sign a token into TRIGGER_DEPLOYMENT_ID at pod creation and verify it on + // inbound workload calls. "disabled" = off; "log" = mint + verify + metrics only; "enforce" = + // also reject invalid tokens. + WORKLOAD_TOKEN_SECRET: z.string().optional(), + WORKLOAD_TOKEN_ENFORCEMENT: z.enum(["disabled", "log", "enforce"]).default("disabled"), + // Absolute expiry for minted deployment tokens. Deterministic (no wall-clock issued-at) so every + // pod of a deployment carries an identical token; bump before this date. Must outlive any run. + WORKLOAD_TOKEN_EXP: z.string().datetime().default("2032-01-01T00:00:00.000Z"), OTEL_EXPORTER_OTLP_ENDPOINT: z.string().url(), // set on the runners // Workload API settings (coordinator mode) - the workload API is what the run controller connects to @@ -365,6 +374,14 @@ export const Env = z path: ["TRIGGER_WORKLOAD_API_DOMAIN"], }); } + if (data.WORKLOAD_TOKEN_ENFORCEMENT !== "disabled" && !data.WORKLOAD_TOKEN_SECRET) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: + "WORKLOAD_TOKEN_SECRET is required when WORKLOAD_TOKEN_ENFORCEMENT is not disabled", + path: ["WORKLOAD_TOKEN_SECRET"], + }); + } if ( data.TRIGGER_DEQUEUE_BACKPRESSURE_ENABLED && !data.TRIGGER_DEQUEUE_BACKPRESSURE_REDIS_HOST diff --git a/apps/supervisor/src/index.ts b/apps/supervisor/src/index.ts index 701b8a25e16..cf73be90eea 100644 --- a/apps/supervisor/src/index.ts +++ b/apps/supervisor/src/index.ts @@ -27,6 +27,7 @@ import { register } from "./metrics.js"; import { PodCleaner } from "./services/podCleaner.js"; import { FailedPodHandler } from "./services/failedPodHandler.js"; import { getWorkerToken } from "./workerToken.js"; +import { mintDeploymentToken } from "./workloadToken.js"; import { OtlpTraceService } from "./services/otlpTraceService.js"; import { WarmStartVerificationService, @@ -96,6 +97,7 @@ class ManagedSupervisor { COMPUTE_GATEWAY_AUTH_TOKEN, DOCKER_REGISTRY_PASSWORD, TRIGGER_DEQUEUE_BACKPRESSURE_REDIS_PASSWORD, + WORKLOAD_TOKEN_SECRET, ...envWithoutSecrets } = env; @@ -290,8 +292,10 @@ class ManagedSupervisor { }); } + const workerToken = getWorkerToken(); + this.workerSession = new SupervisorSession({ - workerToken: getWorkerToken(), + workerToken, apiUrl: env.TRIGGER_API_URL, instanceName: env.TRIGGER_WORKER_INSTANCE_NAME, managedWorkerSecret: env.MANAGED_WORKER_SECRET, @@ -569,6 +573,7 @@ class ManagedSupervisor { checkpointClient: this.checkpointClient, computeManager: this.computeManager, tracing: this.tracing, + snapshotCallbackSecret: workerToken, wideEventOpts: this.wideEventOpts, wideEventsNoisyRoutes: this.wideEventsNoisyRoutes, }); @@ -603,6 +608,15 @@ class ManagedSupervisor { throw new Error("Image is missing"); } + const deploymentToken = await mintDeploymentToken({ + deployment: message.deployment.friendlyId, + deployment_version: message.backgroundWorker.version, + environment_id: message.environment.id, + environment_type: message.environment.type, + org_id: message.organization.id, + project_id: message.project.id, + }); + await this.workloadManager.create({ dequeuedAt: message.dequeuedAt, dequeueResponseMs: timings.dequeueResponseMs, @@ -617,6 +631,7 @@ class ManagedSupervisor { deploymentFriendlyId: message.deployment.friendlyId, deploymentVersion: message.backgroundWorker.version, runtime: message.backgroundWorker.runtime, + deploymentToken, runId: message.run.id, runFriendlyId: message.run.friendlyId, version: message.version, diff --git a/apps/supervisor/src/services/computeSnapshotService.test.ts b/apps/supervisor/src/services/computeSnapshotService.test.ts index 44a13a4bdd3..0877f5e9d7a 100644 --- a/apps/supervisor/src/services/computeSnapshotService.test.ts +++ b/apps/supervisor/src/services/computeSnapshotService.test.ts @@ -20,13 +20,26 @@ function createService() { snapshot, } as unknown as ComputeWorkloadManager; + const submitSuspendCompletion = vi.fn(async () => ({ success: true })); + const service = new ComputeSnapshotService({ computeManager, - workerClient: {} as SupervisorHttpClient, + workerClient: { submitSuspendCompletion } as unknown as SupervisorHttpClient, wideEventOpts: { service: "supervisor-test", env: {}, enabled: false }, + snapshotCallbackSecret: "test-secret", }); - return { service, snapshot }; + return { service, snapshot, submitSuspendCompletion }; +} + +function dispatchedMetadata(snapshot: { + mock: { calls: Array }>> }; +}) { + const metadata = snapshot.mock.calls[0]?.[0]?.metadata; + if (!metadata) { + throw new Error("Snapshot was not dispatched"); + } + return metadata; } function delayedSnapshot(runnerId = "runner-1") { @@ -38,6 +51,24 @@ function delayedSnapshot(runnerId = "runner-1") { } describe("ComputeSnapshotService", () => { + it("refuses to construct with an empty callback secret", () => { + const computeManager = { + snapshotDelayMs: DELAY_MS, + snapshotDispatchLimit: 1, + snapshot: vi.fn(async () => true), + } as unknown as ComputeWorkloadManager; + + expect( + () => + new ComputeSnapshotService({ + computeManager, + workerClient: {} as SupervisorHttpClient, + wideEventOpts: { service: "supervisor-test", env: {}, enabled: false }, + snapshotCallbackSecret: "", + }) + ).toThrow(); + }); + it("dispatches a scheduled snapshot after the delay", async () => { const { service, snapshot } = createService(); try { @@ -46,7 +77,12 @@ describe("ComputeSnapshotService", () => { await vi.waitFor(() => expect(snapshot).toHaveBeenCalledTimes(1), { timeout: 2_000 }); expect(snapshot).toHaveBeenCalledWith({ runnerId: "runner-1", - metadata: { runId: "run_1", snapshotFriendlyId: "snapshot_1" }, + metadata: expect.objectContaining({ + runId: "run_1", + snapshotFriendlyId: "snapshot_1", + snapshotCallbackNonce: expect.any(String), + snapshotCallbackToken: expect.any(String), + }), }); } finally { service.stop(); @@ -121,8 +157,84 @@ describe("ComputeSnapshotService", () => { expect(snapshot).toHaveBeenCalledTimes(1); expect(snapshot).toHaveBeenCalledWith({ runnerId: "runner-1", - metadata: { runId: "run_1", snapshotFriendlyId: "snapshot_2" }, + metadata: expect.objectContaining({ + runId: "run_1", + snapshotFriendlyId: "snapshot_2", + snapshotCallbackNonce: expect.any(String), + snapshotCallbackToken: expect.any(String), + }), + }); + } finally { + service.stop(); + } + }); + + it("accepts a snapshot callback with the dispatched token", async () => { + const { service, snapshot, submitSuspendCompletion } = createService(); + try { + service.schedule("run_1", delayedSnapshot()); + + await vi.waitFor(() => expect(snapshot).toHaveBeenCalledTimes(1), { timeout: 2_000 }); + const metadata = dispatchedMetadata(snapshot); + + const result = await service.handleCallback({ + status: "completed", + instance_id: "instance_1", + snapshot_id: "compute_snapshot_1", + metadata, + }); + + expect(result).toEqual({ ok: true, status: 200 }); + expect(submitSuspendCompletion).toHaveBeenCalledWith({ + runId: "run_1", + snapshotId: "snapshot_1", + body: { + success: true, + checkpoint: { + type: "COMPUTE", + location: "compute_snapshot_1", + }, + }, + }); + } finally { + service.stop(); + } + }); + + it("rejects a snapshot callback without a valid token", async () => { + const { service, submitSuspendCompletion } = createService(); + try { + const result = await service.handleCallback({ + status: "completed", + instance_id: "instance_1", + snapshot_id: "compute_snapshot_1", + metadata: { runId: "run_1", snapshotFriendlyId: "snapshot_1" }, + }); + + expect(result).toEqual({ ok: false, status: 401 }); + expect(submitSuspendCompletion).not.toHaveBeenCalled(); + } finally { + service.stop(); + } + }); + + it("rejects a snapshot callback whose token is for a different snapshot", async () => { + const { service, snapshot, submitSuspendCompletion } = createService(); + try { + service.schedule("run_1", delayedSnapshot()); + + await vi.waitFor(() => expect(snapshot).toHaveBeenCalledTimes(1), { timeout: 2_000 }); + const metadata = dispatchedMetadata(snapshot); + + const result = await service.handleCallback({ + status: "completed", + instance_id: "instance_1", + snapshot_id: "compute_snapshot_1", + metadata: { ...metadata, snapshotFriendlyId: "snapshot_2" }, }); + + expect(result).toEqual({ ok: false, status: 401 }); + expect(submitSuspendCompletion).not.toHaveBeenCalled(); } finally { service.stop(); } diff --git a/apps/supervisor/src/services/computeSnapshotService.ts b/apps/supervisor/src/services/computeSnapshotService.ts index 216753fc12d..6af307d0e30 100644 --- a/apps/supervisor/src/services/computeSnapshotService.ts +++ b/apps/supervisor/src/services/computeSnapshotService.ts @@ -1,3 +1,4 @@ +import { createHmac, randomBytes, timingSafeEqual } from "node:crypto"; import pLimit from "p-limit"; import { SimpleStructuredLogger } from "@trigger.dev/core/v3/utils/structuredLogger"; import { parseTraceparent } from "@trigger.dev/core/v3/isomorphic"; @@ -16,6 +17,13 @@ import { type WideEventOptions, } from "../wideEvents/index.js"; +const SNAPSHOT_CALLBACK_NONCE_METADATA_KEY = "snapshotCallbackNonce"; +const SNAPSHOT_CALLBACK_TOKEN_METADATA_KEY = "snapshotCallbackToken"; + +// Domain-separation label so the callback-signing key is derived from, rather +// than equal to, the secret used for other protocols. Bump the suffix to rotate. +const SNAPSHOT_CALLBACK_KEY_INFO = "compute-snapshot-callback-v1"; + type DelayedSnapshot = { runnerId: string; runFriendlyId: string; @@ -34,6 +42,7 @@ export type ComputeSnapshotServiceOptions = { workerClient: SupervisorHttpClient; tracing?: OtlpTraceService; wideEventOpts: WideEventOptions; + snapshotCallbackSecret: string; }; export class ComputeSnapshotService { @@ -48,6 +57,7 @@ export class ComputeSnapshotService { private readonly workerClient: SupervisorHttpClient; private readonly tracing?: OtlpTraceService; private readonly wideEventOpts: WideEventOptions; + private readonly snapshotCallbackKey: Buffer; constructor(opts: ComputeSnapshotServiceOptions) { this.computeManager = opts.computeManager; @@ -55,6 +65,18 @@ export class ComputeSnapshotService { this.tracing = opts.tracing; this.wideEventOpts = opts.wideEventOpts; + // Reject an empty secret up front: an empty HMAC key would make callback + // tokens forgeable by anyone. Guarding here (rather than only at env parse) + // also covers the case where the secret is read from an empty file. + if (!opts.snapshotCallbackSecret) { + throw new Error("snapshotCallbackSecret must not be empty"); + } + // Derive a dedicated key by domain separation so the raw secret is never + // used directly as a MAC key for this protocol. + this.snapshotCallbackKey = createHmac("sha256", opts.snapshotCallbackSecret) + .update(SNAPSHOT_CALLBACK_KEY_INFO) + .digest(); + this.dispatchLimit = pLimit(this.computeManager.snapshotDispatchLimit); this.timerWheel = new TimerWheel({ delayMs: this.computeManager.snapshotDelayMs, @@ -146,15 +168,29 @@ export class ComputeSnapshotService { instanceId: body.instance_id, status: body.status, error: body.status === "failed" ? body.error : undefined, - metadata: body.metadata, + runId, + snapshotFriendlyId, durationMs: body.duration_ms, }); if (!runId || !snapshotFriendlyId) { - this.logger.error("Snapshot callback missing metadata", { body }); + this.logger.error("Snapshot callback missing metadata", { + status: body.status, + instanceId: body.instance_id, + metadataKeys: Object.keys(body.metadata ?? {}), + }); return { ok: false as const, status: 400 }; } + if (!this.#verifyCallbackToken(body.metadata, runId, snapshotFriendlyId)) { + this.logger.error("Snapshot callback failed token verification", { + runId, + snapshotFriendlyId, + instanceId: body.instance_id, + }); + return { ok: false as const, status: 401 }; + } + this.#emitSnapshotSpan(runId, body.duration_ms, snapshotId); if (body.status === "completed") { @@ -266,11 +302,18 @@ export class ComputeSnapshotService { }, }, async () => { + const callbackNonce = randomBytes(16).toString("hex"); const result = await this.computeManager.snapshot({ runnerId: snapshot.runnerId, metadata: { runId: snapshot.runFriendlyId, snapshotFriendlyId: snapshot.snapshotFriendlyId, + [SNAPSHOT_CALLBACK_NONCE_METADATA_KEY]: callbackNonce, + [SNAPSHOT_CALLBACK_TOKEN_METADATA_KEY]: this.#createCallbackToken( + callbackNonce, + snapshot.runFriendlyId, + snapshot.snapshotFriendlyId + ), }, }); @@ -281,6 +324,51 @@ export class ComputeSnapshotService { ); } + #createCallbackToken(nonce: string, runFriendlyId: string, snapshotFriendlyId: string): string { + return createHmac("sha256", this.snapshotCallbackKey) + .update(nonce) + .update("\0") + .update(runFriendlyId) + .update("\0") + .update(snapshotFriendlyId) + .digest("hex"); + } + + /** + * Verify that a callback carries a token this supervisor issued for the given + * run and snapshot. The token binds only the identifiers known at dispatch + * time (nonce, run, snapshot); it intentionally does not cover result fields + * such as the snapshot location or status/error, which are produced by the + * gateway after the snapshot and so cannot be signed in advance. Verification + * is also stateless, so a token is not single-use. + * + * This closes the primary risk (a caller that can merely reach the endpoint + * cannot mint a valid token, so cannot forge a result for an arbitrary run). + * It does not defend against an attacker who can observe a genuine callback + * and then replay it or alter its unsigned result fields - that relies on the + * gateway->supervisor callback channel being authenticated and encrypted. + */ + #verifyCallbackToken( + metadata: Record | undefined, + runFriendlyId: string, + snapshotFriendlyId: string + ): boolean { + const nonce = metadata?.[SNAPSHOT_CALLBACK_NONCE_METADATA_KEY]; + const token = metadata?.[SNAPSHOT_CALLBACK_TOKEN_METADATA_KEY]; + + if (!nonce || !token) { + return false; + } + + const expected = this.#createCallbackToken(nonce, runFriendlyId, snapshotFriendlyId); + const expectedBuffer = Buffer.from(expected, "hex"); + const tokenBuffer = Buffer.from(token, "hex"); + + return ( + expectedBuffer.length === tokenBuffer.length && timingSafeEqual(expectedBuffer, tokenBuffer) + ); + } + #emitSnapshotSpan(runFriendlyId: string, durationMs?: number, snapshotId?: string) { if (!this.tracing) return; diff --git a/apps/supervisor/src/workloadManager/compute.ts b/apps/supervisor/src/workloadManager/compute.ts index 857e129a83f..89360437dd8 100644 --- a/apps/supervisor/src/workloadManager/compute.ts +++ b/apps/supervisor/src/workloadManager/compute.ts @@ -151,7 +151,9 @@ export class ComputeWorkloadManager implements WorkloadManager { TRIGGER_DEQUEUED_AT_MS: String(opts.dequeuedAt.getTime()), TRIGGER_POD_SCHEDULED_AT_MS: String(Date.now()), TRIGGER_ENV_ID: opts.envId, - TRIGGER_DEPLOYMENT_ID: opts.deploymentFriendlyId, + TRIGGER_DEPLOYMENT_ID: opts.deploymentToken ?? opts.deploymentFriendlyId, + // Plain friendlyId for telemetry (worker.id), so it isn't the opaque token in DEPLOYMENT_ID. + TRIGGER_DEPLOYMENT_FRIENDLY_ID: opts.deploymentFriendlyId, TRIGGER_DEPLOYMENT_VERSION: opts.deploymentVersion, TRIGGER_RUN_ID: opts.runFriendlyId, TRIGGER_SNAPSHOT_ID: opts.snapshotFriendlyId, diff --git a/apps/supervisor/src/workloadManager/docker.ts b/apps/supervisor/src/workloadManager/docker.ts index 1d8fec1df78..c8049873b64 100644 --- a/apps/supervisor/src/workloadManager/docker.ts +++ b/apps/supervisor/src/workloadManager/docker.ts @@ -72,7 +72,9 @@ export class DockerWorkloadManager implements WorkloadManager { `TRIGGER_DEQUEUED_AT_MS=${opts.dequeuedAt.getTime()}`, `TRIGGER_POD_SCHEDULED_AT_MS=${Date.now()}`, `TRIGGER_ENV_ID=${opts.envId}`, - `TRIGGER_DEPLOYMENT_ID=${opts.deploymentFriendlyId}`, + `TRIGGER_DEPLOYMENT_ID=${opts.deploymentToken ?? opts.deploymentFriendlyId}`, + // Plain friendlyId for telemetry (worker.id), so it isn't the opaque token in DEPLOYMENT_ID. + `TRIGGER_DEPLOYMENT_FRIENDLY_ID=${opts.deploymentFriendlyId}`, `TRIGGER_DEPLOYMENT_VERSION=${opts.deploymentVersion}`, `TRIGGER_RUN_ID=${opts.runFriendlyId}`, `TRIGGER_SNAPSHOT_ID=${opts.snapshotFriendlyId}`, diff --git a/apps/supervisor/src/workloadManager/kubernetes.ts b/apps/supervisor/src/workloadManager/kubernetes.ts index 282ac80b4fd..0263b229e13 100644 --- a/apps/supervisor/src/workloadManager/kubernetes.ts +++ b/apps/supervisor/src/workloadManager/kubernetes.ts @@ -158,6 +158,11 @@ export class KubernetesWorkloadManager implements WorkloadManager { }, { name: "TRIGGER_DEPLOYMENT_ID", + value: opts.deploymentToken ?? opts.deploymentFriendlyId, + }, + { + // Plain friendlyId for telemetry (worker.id), not the opaque token in DEPLOYMENT_ID. + name: "TRIGGER_DEPLOYMENT_FRIENDLY_ID", value: opts.deploymentFriendlyId, }, { diff --git a/apps/supervisor/src/workloadManager/types.ts b/apps/supervisor/src/workloadManager/types.ts index 4945b1382d9..eda9162c7d6 100644 --- a/apps/supervisor/src/workloadManager/types.ts +++ b/apps/supervisor/src/workloadManager/types.ts @@ -44,6 +44,8 @@ export interface WorkloadManagerCreateOptions { deploymentVersion: string; // Canonical runtime identifier (e.g. "node", "node-22", "node-24") runtime?: string; + // When set, overrides the TRIGGER_DEPLOYMENT_ID value the runner forwards as its identity header. + deploymentToken?: string; runId: string; runFriendlyId: string; snapshotId: string; diff --git a/apps/supervisor/src/workloadServer/index.ts b/apps/supervisor/src/workloadServer/index.ts index bacdbbe06d9..86717438c72 100644 --- a/apps/supervisor/src/workloadServer/index.ts +++ b/apps/supervisor/src/workloadServer/index.ts @@ -25,6 +25,11 @@ import { type Namespace, Server, type Socket } from "socket.io"; import { z } from "zod"; import { env } from "../env.js"; import { register } from "../metrics.js"; +import { + verifyDeploymentIdHeader, + workloadTokenEnforced, + workloadTokensEnabled, +} from "../workloadToken.js"; import { ComputeSnapshotService, type RunTraceContext, @@ -86,6 +91,7 @@ type WorkloadServerOptions = { checkpointClient?: CheckpointClient; computeManager?: ComputeWorkloadManager; tracing?: OtlpTraceService; + snapshotCallbackSecret: string; wideEventOpts: WideEventOptions; /** When true, high-frequency HTTP routes also emit wide events. */ wideEventsNoisyRoutes: boolean; @@ -136,6 +142,7 @@ export class WorkloadServer extends EventEmitter { workerClient: opts.workerClient, tracing: opts.tracing, wideEventOpts: this.wideEventOpts, + snapshotCallbackSecret: opts.snapshotCallbackSecret, }); } @@ -169,6 +176,34 @@ export class WorkloadServer extends EventEmitter { return this.headerValueFromRequest(req, WORKLOAD_HEADERS.PROJECT_REF); } + /** + * Verify the deployment token from the workload deployment-id header and return the verified + * environment_id to forward upstream. The env id is only forwarded in enforce mode: in log mode + * we still verify + record metrics but attach no header (so the platform never scopes). Only + * enforce fails a request, and only for a present-but-invalid token; absent and legacy ids pass. + */ + private async authorizeWorkloadRequest( + req: IncomingMessage + ): Promise<{ ok: true; environmentId?: string } | { ok: false }> { + if (!workloadTokensEnabled) { + return { ok: true }; + } + + const result = await verifyDeploymentIdHeader(this.deploymentIdFromRequest(req), "http"); + + if (result.outcome === "jwt_invalid" && workloadTokenEnforced) { + return { ok: false }; + } + + return { + ok: true, + environmentId: + workloadTokenEnforced && result.outcome === "jwt_valid" + ? result.claims.environment_id + : undefined, + }; + } + /** * Sets common route meta on the wide-event state from URL params. */ @@ -250,11 +285,17 @@ export class WorkloadServer extends EventEmitter { "POST", async () => { const { req, reply, params, body } = ctx; + const auth = await this.authorizeWorkloadRequest(req); + if (!auth.ok) { + reply.empty(401); + return; + } const startResponse = await this.workerClient.startRunAttempt( params.runFriendlyId, params.snapshotFriendlyId, body, - this.runnerIdFromRequest(req) + this.runnerIdFromRequest(req), + auth.environmentId ); if (!startResponse.success) { @@ -286,6 +327,11 @@ export class WorkloadServer extends EventEmitter { "POST", async () => { const { req, reply, params, body } = ctx; + const auth = await this.authorizeWorkloadRequest(req); + if (!auth.ok) { + reply.empty(401); + return; + } const runnerId = this.runnerIdFromRequest(req); // A completion attempt invalidates any pending delayed snapshot @@ -304,7 +350,8 @@ export class WorkloadServer extends EventEmitter { params.runFriendlyId, params.snapshotFriendlyId, body, - runnerId + runnerId, + auth.environmentId ); if (!completeResponse.success) { @@ -336,6 +383,11 @@ export class WorkloadServer extends EventEmitter { "POST", async () => { const { req, reply, params, body } = ctx; + const auth = await this.authorizeWorkloadRequest(req); + if (!auth.ok) { + reply.empty(401); + return; + } const heartbeatResponse = await this.workerClient.heartbeatRun( params.runFriendlyId, params.snapshotFriendlyId, @@ -373,6 +425,11 @@ export class WorkloadServer extends EventEmitter { "GET", async () => { const { reply, params, req } = ctx; + const auth = await this.authorizeWorkloadRequest(req); + if (!auth.ok) { + reply.empty(401); + return; + } const runnerId = this.runnerIdFromRequest(req); const deploymentVersion = this.deploymentVersionFromRequest(req); const projectRef = this.projectRefFromRequest(req); @@ -469,6 +526,11 @@ export class WorkloadServer extends EventEmitter { "GET", async () => { const { req, reply, params } = ctx; + const auth = await this.authorizeWorkloadRequest(req); + if (!auth.ok) { + reply.empty(401); + return; + } this.logger.debug("Run continuation request", { params }); // Cancel any pending delayed snapshot for this run @@ -477,7 +539,8 @@ export class WorkloadServer extends EventEmitter { const continuationResult = await this.workerClient.continueRunExecution( params.runFriendlyId, params.snapshotFriendlyId, - this.runnerIdFromRequest(req) + this.runnerIdFromRequest(req), + auth.environmentId ); if (!continuationResult.success) { @@ -511,10 +574,16 @@ export class WorkloadServer extends EventEmitter { "GET", async () => { const { req, reply, params } = ctx; + const auth = await this.authorizeWorkloadRequest(req); + if (!auth.ok) { + reply.empty(401); + return; + } const sinceSnapshotResponse = await this.workerClient.getSnapshotsSince( params.runFriendlyId, params.snapshotFriendlyId, - this.runnerIdFromRequest(req) + this.runnerIdFromRequest(req), + auth.environmentId ); if (!sinceSnapshotResponse.success) { @@ -585,9 +654,18 @@ export class WorkloadServer extends EventEmitter { const { req, reply, params, body } = ctx; reply.empty(204); + // Redact TRIGGER_DEPLOYMENT_ID before relaying to the platform. + const sanitizedBody = + body.properties && "TRIGGER_DEPLOYMENT_ID" in body.properties + ? { + ...body, + properties: { ...body.properties, TRIGGER_DEPLOYMENT_ID: "[redacted]" }, + } + : body; + await this.workerClient.sendDebugLog( params.runFriendlyId, - body, + sanitizedBody, this.runnerIdFromRequest(req) ); }, @@ -681,7 +759,31 @@ export class WorkloadServer extends EventEmitter { return; } - this.logger.debug("[WS] auth success", socket.data); + if (workloadTokensEnabled) { + const result = await verifyDeploymentIdHeader(socket.data.deploymentId, "ws"); + + if (result.outcome === "jwt_invalid" && workloadTokenEnforced) { + this.logger.error("[WS] deployment token verification failed", { + runnerId: socket.data.runnerId, + }); + socket.disconnect(true); + return; + } + + // Re-source the deployment id from the verified claim; the raw header may be an opaque token. + // A legacy bare id is itself the friendlyId, so it's safe to keep. + socket.data.deploymentFriendlyId = + result.outcome === "jwt_valid" + ? result.claims.deployment + : result.outcome === "legacy_bare" + ? socket.data.deploymentId + : undefined; + } + + this.logger.debug("[WS] handshake complete", { + runnerId: socket.data.runnerId, + deploymentFriendlyId: socket.data.deploymentFriendlyId, + }); next(); }); @@ -693,7 +795,7 @@ export class WorkloadServer extends EventEmitter { const getSocketMetadata = () => { return { - deploymentId: socket.data.deploymentId, + deploymentId: socket.data.deploymentFriendlyId ?? socket.data.deploymentId, runId: socket.data.runFriendlyId, snapshotId: socket.data.snapshotId, runnerId: socket.data.runnerId, @@ -712,8 +814,9 @@ export class WorkloadServer extends EventEmitter { populate: (state) => { state.extras.event = event; setMeta(state, "run_id", friendlyId); - if (socket.data.deploymentId) { - setMeta(state, "deployment_id", socket.data.deploymentId); + const deploymentId = socket.data.deploymentFriendlyId ?? socket.data.deploymentId; + if (deploymentId) { + setMeta(state, "deployment_id", deploymentId); } if (socket.data.runnerId) setMeta(state, "runner_id", socket.data.runnerId); state.extras.socket_id = socket.id; @@ -725,6 +828,33 @@ export class WorkloadServer extends EventEmitter { const runConnected = (friendlyId: string) => { socketLogger.debug("runConnected", { ...getSocketMetadata() }); + // Only the owning runner may (re)bind a run. A live socket from a *different* + // runner keeps its binding so an unrelated connection can't hijack the run. But + // the newest socket for the *same* runner is a legitimate reconnection/handoff and + // is allowed to take over even while the stale socket still reports connected - + // otherwise, during a reconnect race the fresh socket would silently stay unbound + // (missing continue/cancel/suspend notifications) until the dead socket times out. + const existing = this.runSockets.get(friendlyId); + if (existing && existing.id !== socket.id && existing.connected) { + const sameRunner = + !!socket.data.runnerId && existing.data.runnerId === socket.data.runnerId; + + if (!sameRunner) { + socketLogger.warn("runConnected: run already bound to another socket", { + ...getSocketMetadata(), + friendlyId, + existingSocketId: existing.id, + }); + return; + } + + socketLogger.debug("runConnected: replacing stale socket for same runner", { + ...getSocketMetadata(), + friendlyId, + existingSocketId: existing.id, + }); + } + // If there's already a run ID set, we should "disconnect" it from this socket if (socket.data.runFriendlyId && socket.data.runFriendlyId !== friendlyId) { socketLogger.debug("runConnected: disconnecting existing run", { @@ -744,6 +874,22 @@ export class WorkloadServer extends EventEmitter { const runDisconnected = (friendlyId: string, reason: string) => { socketLogger.debug("runDisconnected", { ...getSocketMetadata() }); + // A newer socket may have taken over this run (same-runner reconnect race). If the + // run is now bound to a different socket, this stale socket must not clear the fresh + // binding or emit a spurious disconnect - just drop its own reference and bail. + const bound = this.runSockets.get(friendlyId); + if (bound && bound.id !== socket.id) { + socketLogger.debug("runDisconnected: run rebound to another socket, skipping", { + ...getSocketMetadata(), + friendlyId, + boundSocketId: bound.id, + }); + if (socket.data.runFriendlyId === friendlyId) { + socket.data.runFriendlyId = undefined; + } + return; + } + // The run is gone from this runner (crash, exit, or replaced by a new // run), so a pending delayed snapshot for it is stale. Genuine // waitpoint suspensions keep the socket connected, so this doesn't diff --git a/apps/supervisor/src/workloadServer/workloadServer.auth.test.ts b/apps/supervisor/src/workloadServer/workloadServer.auth.test.ts new file mode 100644 index 00000000000..3e8fb6e9de7 --- /dev/null +++ b/apps/supervisor/src/workloadServer/workloadServer.auth.test.ts @@ -0,0 +1,107 @@ +import { mintWorkloadDeploymentToken } from "@trigger.dev/core/v3"; +import { WORKLOAD_HEADERS } from "@trigger.dev/core/v3/workers"; +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; + +// Set enforce mode + secret before env.ts parses (vi.mock is hoisted above imports, so the secret +// must be a literal here). SECRET below mirrors it for use in the test body. +vi.mock("std-env", () => ({ + env: { + TRIGGER_API_URL: "http://localhost:3030", + TRIGGER_WORKER_TOKEN: "test-token", + MANAGED_WORKER_SECRET: "test-secret", + OTEL_EXPORTER_OTLP_ENDPOINT: "http://localhost:4318", + WORKLOAD_TOKEN_SECRET: "integration-test-secret", + WORKLOAD_TOKEN_ENFORCEMENT: "enforce", + }, +})); + +const SECRET = "integration-test-secret"; +const EXP = Math.floor(Date.UTC(2032, 0, 1) / 1000); + +const { WorkloadServer } = await import("./index.js"); + +const PORT = 18732; +const BASE = `http://127.0.0.1:${PORT}`; + +function claims(environmentId = "env_test_123") { + return { + deployment: "deployment_test", + deployment_version: "20260710.1", + environment_id: environmentId, + environment_type: "PRODUCTION", + org_id: "org_1", + project_id: "proj_1", + }; +} + +// Records the args each relay method is called with so we can assert the forwarded claim. +const calls: { getSnapshotsSince: any[][] } = { getSnapshotsSince: [] }; + +const workerClient = { + getSnapshotsSince: vi.fn(async (...args: any[]) => { + calls.getSnapshotsSince.push(args); + return { success: true as const, data: { snapshots: [] } }; + }), +} as any; + +let server: InstanceType; + +beforeAll(async () => { + server = new WorkloadServer({ + port: PORT, + workerClient, + snapshotCallbackSecret: "snapshot-callback-secret", + wideEventOpts: { service: "supervisor", env: { nodeId: "test" }, enabled: false }, + wideEventsNoisyRoutes: false, + }); + await server.start(); +}); + +afterAll(async () => { + await server.stop(); +}); + +function snapshotsSince(deploymentIdHeader?: string) { + const headers: Record = { + [WORKLOAD_HEADERS.RUNNER_ID]: "runner_1", + }; + if (deploymentIdHeader !== undefined) { + headers[WORKLOAD_HEADERS.DEPLOYMENT_ID] = deploymentIdHeader; + } + return fetch(`${BASE}/api/v1/workload-actions/runs/run_1/snapshots/since/snap_1`, { headers }); +} + +describe("WorkloadServer auth (enforce mode)", () => { + it("allows a valid token and forwards the verified environment_id", async () => { + const token = await mintWorkloadDeploymentToken(claims("env_forwarded_42"), SECRET, EXP); + const res = await snapshotsSince(token); + + expect(res.status).toBe(200); + const lastCall = calls.getSnapshotsSince.at(-1)!; + // getSnapshotsSince(runId, snapshotId, runnerId, environmentId) + expect(lastCall[3]).toBe("env_forwarded_42"); + }); + + it("rejects a token signed with the wrong secret (401) and does not relay", async () => { + const before = calls.getSnapshotsSince.length; + const badToken = await mintWorkloadDeploymentToken(claims(), "wrong-secret", EXP); + const res = await snapshotsSince(badToken); + + expect(res.status).toBe(401); + expect(calls.getSnapshotsSince.length).toBe(before); + }); + + it("allows a legacy bare friendlyId and forwards no environment_id", async () => { + const res = await snapshotsSince("deployment_legacy_bare"); + + expect(res.status).toBe(200); + expect(calls.getSnapshotsSince.at(-1)![3]).toBeUndefined(); + }); + + it("allows an absent token and forwards no environment_id", async () => { + const res = await snapshotsSince(undefined); + + expect(res.status).toBe(200); + expect(calls.getSnapshotsSince.at(-1)![3]).toBeUndefined(); + }); +}); diff --git a/apps/supervisor/src/workloadServer/workloadServer.authLog.test.ts b/apps/supervisor/src/workloadServer/workloadServer.authLog.test.ts new file mode 100644 index 00000000000..8e66f738f64 --- /dev/null +++ b/apps/supervisor/src/workloadServer/workloadServer.authLog.test.ts @@ -0,0 +1,87 @@ +import { mintWorkloadDeploymentToken } from "@trigger.dev/core/v3"; +import { WORKLOAD_HEADERS } from "@trigger.dev/core/v3/workers"; +import { afterAll, beforeAll, describe, expect, it, vi } from "vitest"; + +// Log mode: mint + verify + metrics, but the platform must NOT be scoped, so no environment_id is +// forwarded even for a valid token. (vi.mock is hoisted; secret literal here, mirrored below.) +vi.mock("std-env", () => ({ + env: { + TRIGGER_API_URL: "http://localhost:3030", + TRIGGER_WORKER_TOKEN: "test-token", + MANAGED_WORKER_SECRET: "test-secret", + OTEL_EXPORTER_OTLP_ENDPOINT: "http://localhost:4318", + WORKLOAD_TOKEN_SECRET: "integration-test-secret", + WORKLOAD_TOKEN_ENFORCEMENT: "log", + }, +})); + +const SECRET = "integration-test-secret"; +const EXP = Math.floor(Date.UTC(2032, 0, 1) / 1000); + +const { WorkloadServer } = await import("./index.js"); + +const PORT = 18733; +const BASE = `http://127.0.0.1:${PORT}`; + +const claims = { + deployment: "deployment_test", + deployment_version: "20260710.1", + environment_id: "env_should_not_forward", + environment_type: "PRODUCTION", + org_id: "org_1", + project_id: "proj_1", +}; + +const calls: { getSnapshotsSince: any[][] } = { getSnapshotsSince: [] }; + +const workerClient = { + getSnapshotsSince: vi.fn(async (...args: any[]) => { + calls.getSnapshotsSince.push(args); + return { success: true as const, data: { snapshots: [] } }; + }), +} as any; + +let server: InstanceType; + +beforeAll(async () => { + server = new WorkloadServer({ + port: PORT, + workerClient, + snapshotCallbackSecret: "snapshot-callback-secret", + wideEventOpts: { service: "supervisor", env: { nodeId: "test" }, enabled: false }, + wideEventsNoisyRoutes: false, + }); + await server.start(); +}); + +afterAll(async () => { + await server.stop(); +}); + +describe("WorkloadServer auth (log mode)", () => { + it("allows a valid token but forwards no environment_id", async () => { + const token = await mintWorkloadDeploymentToken(claims, SECRET, EXP); + const res = await fetch(`${BASE}/api/v1/workload-actions/runs/run_1/snapshots/since/snap_1`, { + headers: { + [WORKLOAD_HEADERS.RUNNER_ID]: "runner_1", + [WORKLOAD_HEADERS.DEPLOYMENT_ID]: token, + }, + }); + + expect(res.status).toBe(200); + expect(calls.getSnapshotsSince.at(-1)![3]).toBeUndefined(); + }); + + it("does not reject an invalid token in log mode", async () => { + const badToken = await mintWorkloadDeploymentToken(claims, "wrong-secret", EXP); + const res = await fetch(`${BASE}/api/v1/workload-actions/runs/run_1/snapshots/since/snap_1`, { + headers: { + [WORKLOAD_HEADERS.RUNNER_ID]: "runner_1", + [WORKLOAD_HEADERS.DEPLOYMENT_ID]: badToken, + }, + }); + + expect(res.status).toBe(200); + expect(calls.getSnapshotsSince.at(-1)![3]).toBeUndefined(); + }); +}); diff --git a/apps/supervisor/src/workloadToken.ts b/apps/supervisor/src/workloadToken.ts new file mode 100644 index 00000000000..914411dd735 --- /dev/null +++ b/apps/supervisor/src/workloadToken.ts @@ -0,0 +1,85 @@ +import { + classifyDeploymentIdHeader, + mintWorkloadDeploymentToken, + type WorkloadDeploymentTokenClaims, + type WorkloadDeploymentTokenInput, +} from "@trigger.dev/core/v3"; +import { Counter } from "prom-client"; +import { env } from "./env.js"; +import { register } from "./metrics.js"; + +const secret = env.WORKLOAD_TOKEN_SECRET; + +// Absolute expiry (epoch seconds) shared by every mint, so tokens stay byte-deterministic per +// deployment regardless of when/where a pod is created. +const tokenExpSeconds = Math.floor(new Date(env.WORKLOAD_TOKEN_EXP).getTime() / 1000); + +/** Mint + verify run in "log" (dry-run) and "enforce"; the env superRefine guarantees a secret then. */ +export const workloadTokensEnabled = env.WORKLOAD_TOKEN_ENFORCEMENT !== "disabled"; + +/** Only "enforce" rejects a present-but-invalid token; "log" observes and always allows. */ +export const workloadTokenEnforced = env.WORKLOAD_TOKEN_ENFORCEMENT === "enforce"; + +const mintCounter = new Counter({ + name: "workload_token_minted_total", + help: "Deployment tokens minted and injected into TRIGGER_DEPLOYMENT_ID at pod creation", + labelNames: ["env_type"] as const, + registers: [register], +}); + +export type WorkloadAuthTransport = "http" | "ws"; +export type WorkloadAuthOutcome = "jwt_valid" | "jwt_invalid" | "legacy_bare" | "token_absent"; + +const verifyCounter = new Counter({ + name: "workload_auth_verify_total", + help: "Runner-boundary token verification outcomes at the supervisor workload server", + labelNames: ["outcome", "transport", "env_type"] as const, + registers: [register], +}); + +export async function mintDeploymentToken( + claims: WorkloadDeploymentTokenInput +): Promise { + if (!workloadTokensEnabled || !secret) { + return undefined; + } + + const token = await mintWorkloadDeploymentToken(claims, secret, tokenExpSeconds); + mintCounter.inc({ env_type: claims.environment_type }); + return token; +} + +export type VerifiedDeploymentHeader = + | { outcome: "jwt_valid"; claims: WorkloadDeploymentTokenClaims } + | { outcome: "jwt_invalid" | "legacy_bare" | "token_absent"; claims?: undefined }; + +/** + * Verify the deployment-id header value and record the outcome. "jwt_valid" returns the claims so the + * caller can forward the verified environment_id upstream; other outcomes carry no trusted data. + */ +export async function verifyDeploymentIdHeader( + value: string | undefined, + transport: WorkloadAuthTransport +): Promise { + const result = await classify(value); + verifyCounter.inc({ + outcome: result.outcome, + transport, + env_type: result.outcome === "jwt_valid" ? result.claims.environment_type : "unknown", + }); + return result; +} + +async function classify(value: string | undefined): Promise { + if (!value || !secret) { + return { outcome: "token_absent" }; + } + + const result = await classifyDeploymentIdHeader(value, secret); + + if (result.outcome === "jwt_valid" && result.claims) { + return { outcome: "jwt_valid", claims: result.claims }; + } + + return { outcome: result.outcome === "jwt_valid" ? "jwt_invalid" : result.outcome }; +} diff --git a/apps/webapp/app/entry.server.tsx b/apps/webapp/app/entry.server.tsx index 672916d8103..da1ab5a433a 100644 --- a/apps/webapp/app/entry.server.tsx +++ b/apps/webapp/app/entry.server.tsx @@ -300,6 +300,7 @@ singleton("SentryTenantContextProcessor", () => { export { apiRateLimiter } from "./services/apiRateLimit.server"; export { engineRateLimiter } from "./services/engineRateLimit.server"; +export { otlpRateLimiter } from "./services/otlpRateLimit.server"; export { runWithHttpContext } from "./services/httpAsyncStorage.server"; export { tenantContextMiddleware } from "./services/tenantContextResolver.server"; export { socketIo } from "./v3/handleSocketIo.server"; diff --git a/apps/webapp/app/env.server.ts b/apps/webapp/app/env.server.ts index 973560ca07f..53b5edf0a46 100644 --- a/apps/webapp/app/env.server.ts +++ b/apps/webapp/app/env.server.ts @@ -85,6 +85,29 @@ const S2EnvSchema = z.preprocess( ]) ); +// Previously published secret values must never be accepted, including when +// an existing deployment or external secret manager still supplies one. +const INSECURE_SECRET_VALUES = [ + "managed-secret", + "2818143646516f6fffd707b36f334bbb", + "44da78b7bbb0dfe709cf38931d25dcdd", + "f686147ab967943ebbe9ed3b496e465a", + "447c29678f9eaf289e9c4b70d3dd8a7f", +]; + +// Escape hatch for deployments that can't rotate a published default yet (e.g. +// ENCRYPTION_KEY protects existing data). Read raw: a refine can't see the +// sibling parsed flag. +const allowInsecureDefaultSecrets = ["true", "1"].includes( + (process.env.ALLOW_INSECURE_DEFAULT_SECRETS ?? "").toLowerCase().trim() +); + +const isNotInsecureSecret = (value: string) => + allowInsecureDefaultSecrets || !INSECURE_SECRET_VALUES.includes(value); + +const INSECURE_SECRET_MESSAGE = + "must not be a known-insecure published default; set a strong, unique value. If you cannot rotate it yet (e.g. it protects existing encrypted data or active sessions), set ALLOW_INSECURE_DEFAULT_SECRETS=1 to boot while you migrate."; + const EnvironmentSchema = z .object({ NODE_ENV: z.union([z.literal("development"), z.literal("production"), z.literal("test")]), @@ -188,14 +211,15 @@ const EnvironmentSchema = z // Control-plane cache relax knobs. Unset -> defaults (DEFAULT_CP_CACHE_TTL_MS / _MAX_ENTRIES). CONTROL_PLANE_CACHE_TTL_MS: z.coerce.number().int().optional(), CONTROL_PLANE_CACHE_MAX_ENTRIES: z.coerce.number().int().optional(), - SESSION_SECRET: z.string(), - MAGIC_LINK_SECRET: z.string(), + SESSION_SECRET: z.string().min(1).refine(isNotInsecureSecret, INSECURE_SECRET_MESSAGE), + MAGIC_LINK_SECRET: z.string().min(1).refine(isNotInsecureSecret, INSECURE_SECRET_MESSAGE), ENCRYPTION_KEY: z .string() .refine( (val) => Buffer.from(val, "utf8").length === 32, "ENCRYPTION_KEY must be exactly 32 bytes" - ), + ) + .refine(isNotInsecureSecret, INSECURE_SECRET_MESSAGE), WHITELISTED_EMAILS: z .string() .refine(isValidRegex, "WHITELISTED_EMAILS must be a valid regex.") @@ -547,6 +571,22 @@ const EnvironmentSchema = z API_RATE_LIMIT_JWT_WINDOW: z.string().default("1m"), API_RATE_LIMIT_JWT_TOKENS: z.coerce.number().int().default(60), + // Per-IP rate limit for the unauthenticated OTLP ingestion endpoints + // (/otel/*). Bounds unauthenticated request rates. Opt-in + // (disabled by default): because it keys on the source IP, it is only + // safe to enable when each client presents a distinct IP through a proxy + // that appends the real client IP to X-Forwarded-For. Enabling it where + // many clients share one egress IP (e.g. behind NAT or a shared proxy) + // would collapse that traffic into a single bucket and could throttle + // legitimate telemetry. Set OTLP_RATE_LIMIT_ENABLED=1 to enable, then tune + // OTLP_RATE_LIMIT_MAX / OTLP_RATE_LIMIT_WINDOW for expected volume. + OTLP_RATE_LIMIT_ENABLED: z.string().default("0"), + OTLP_RATE_LIMIT_WINDOW: z + .string() + .regex(/^\d+ ?(?:ms|s|m|h|d)$/) + .default("1m"), + OTLP_RATE_LIMIT_MAX: z.coerce.number().int().positive().default(3000), + DEPOT_TOKEN: z.string().optional(), DEPOT_ORG_ID: z.string().optional(), DEPOT_REGION: z.string().default("us-east-1"), @@ -668,7 +708,18 @@ const EnvironmentSchema = z EVENTS_LOAD_SHEDDING_THRESHOLD: z.coerce.number().int().default(100000), EVENTS_LOAD_SHEDDING_ENABLED: z.string().default("1"), - MANAGED_WORKER_SECRET: z.string().default("managed-secret"), + MANAGED_WORKER_SECRET: z.string().min(1).refine(isNotInsecureSecret, INSECURE_SECRET_MESSAGE), + + // Allow booting with a known-insecure published default secret. Temporary + // bridge for deployments that can't rotate yet; rotate as soon as possible. + ALLOW_INSECURE_DEFAULT_SECRETS: BoolEnv.default(false), + + // Tenant scoping on worker actions is header-driven (folded into the engine snapshot read) and + // needs no flag. This is only the no-header fallback: when "1", a worker action on a run created + // after WORKLOAD_TOKEN_CUTOFF without a verified env header is rejected; runs on or before the + // cutoff pass (grandfathered). Default off = no run-row read, byte-for-byte today's behavior. + WORKLOAD_CREATED_AT_GATE_ENABLED: z.string().default("0"), + WORKLOAD_TOKEN_CUTOFF: z.string().datetime().optional(), // Development OTEL environment variables DEV_OTEL_EXPORTER_OTLP_ENDPOINT: z.string().optional(), @@ -2090,3 +2141,24 @@ const EnvironmentSchema = z export type Environment = z.infer; export const env = EnvironmentSchema.parse(process.env); + +if (env.ALLOW_INSECURE_DEFAULT_SECRETS) { + const insecure = ( + [ + ["SESSION_SECRET", env.SESSION_SECRET], + ["MAGIC_LINK_SECRET", env.MAGIC_LINK_SECRET], + ["ENCRYPTION_KEY", env.ENCRYPTION_KEY], + ["MANAGED_WORKER_SECRET", env.MANAGED_WORKER_SECRET], + ] as const + ) + .filter(([, value]) => INSECURE_SECRET_VALUES.includes(value)) + .map(([name]) => name); + + if (insecure.length > 0) { + console.warn( + `⚠️ ALLOW_INSECURE_DEFAULT_SECRETS is enabled and these secrets still use a known-insecure published default: ${insecure.join( + ", " + )}. This is insecure - rotate them as soon as you can.` + ); + } +} diff --git a/apps/webapp/app/models/schedules.server.ts b/apps/webapp/app/models/schedules.server.ts index 72d08c4fb9f..b9d72665caf 100644 --- a/apps/webapp/app/models/schedules.server.ts +++ b/apps/webapp/app/models/schedules.server.ts @@ -1,4 +1,4 @@ -import type { Prisma } from "~/db.server"; +import type { Prisma, PrismaClientOrTransaction, TaskSchedule } from "@trigger.dev/database"; export function scheduleUniqWhereClause( projectId: string, @@ -35,3 +35,48 @@ export function scheduleWhereClause( deduplicationKey: scheduleId, }; } + +/** + * Resolve a schedule's visibility for an environment-scoped caller. + * + * - "visible": the schedule exists in the project and has at least one + * instance bound to `environmentId` (or has no instances yet). + * - "hidden": the schedule exists but none of its instances live in the + * caller's environment. + * - "missing": no schedule exists for the (project, scheduleId) pair. + * + * A schedule can be bound to several environments at once, so visibility + * mirrors the "some instance is in this environment" rule the schedule + * list uses: a schedule that is listed for a key must also be readable + * and mutable by that key. This still rejects cross-environment access to + * schedules the caller has no instance in, and `scheduleWhereClause` + * already confines the lookup to the caller's project. + * + * The tri-state lets PUT (upsert) disambiguate "hidden" (refuse) from + * "missing" (fall through to create). DELETE/GET treat hidden and + * missing the same way. + */ +export type ScheduleEnvVisibility = + | { status: "visible"; schedule: TaskSchedule } + | { status: "hidden" } + | { status: "missing" }; + +export async function getScheduleEnvVisibility( + prisma: PrismaClientOrTransaction, + projectId: string, + scheduleId: string, + environmentId: string +): Promise { + const schedule = await prisma.taskSchedule.findFirst({ + where: scheduleWhereClause(projectId, scheduleId), + include: { instances: { select: { environmentId: true } } }, + }); + + if (!schedule) return { status: "missing" }; + + const { instances, ...rest } = schedule; + if (instances.length === 0) return { status: "visible", schedule: rest }; + const scoped = instances.some((i) => i.environmentId === environmentId); + if (!scoped) return { status: "hidden" }; + return { status: "visible", schedule: rest }; +} diff --git a/apps/webapp/app/routes/_app.github.callback/route.tsx b/apps/webapp/app/routes/_app.github.callback/route.tsx index cd67f27f54c..94e13ae1854 100644 --- a/apps/webapp/app/routes/_app.github.callback/route.tsx +++ b/apps/webapp/app/routes/_app.github.callback/route.tsx @@ -1,6 +1,9 @@ import { type LoaderFunctionArgs } from "@remix-run/node"; import { z } from "zod"; -import { validateGitHubAppInstallSession } from "~/services/gitHubSession.server"; +import { + destroyGitHubAppInstallSession, + validateGitHubAppInstallSession, +} from "~/services/gitHubSession.server"; import { linkGitHubAppInstallation, updateGitHubAppInstallation } from "~/services/gitHub.server"; import { logger } from "~/services/logger.server"; import { redirectWithErrorMessage, redirectWithSuccessMessage } from "~/models/message.server"; @@ -75,6 +78,15 @@ export async function loader({ request }: LoaderFunctionArgs) { return redirectWithErrorMessage(redirectTo, request, "Failed to install GitHub app"); } + // The install session is single-use: once a callback consumes an + // installation_id, invalidate the state cookie so the same initiation + // cannot be replayed against other installation_ids. + const clearInstallSession = await destroyGitHubAppInstallSession(cookieHeader); + const consumingSession = (response: Response) => { + response.headers.append("Set-Cookie", clearInstallSession); + return response; + }; + switch (callbackData.setup_action) { case "install": { const [error] = await tryCatch( @@ -85,23 +97,33 @@ export async function loader({ request }: LoaderFunctionArgs) { logger.error("Failed to link GitHub App installation", { error, }); - return redirectWithErrorMessage(redirectTo, request, "Failed to install GitHub app"); + return consumingSession( + await redirectWithErrorMessage(redirectTo, request, "Failed to install GitHub app") + ); } - return redirectWithSuccessMessage(redirectTo, request, "GitHub App installed successfully"); + return consumingSession( + await redirectWithSuccessMessage(redirectTo, request, "GitHub App installed successfully") + ); } case "update": { - const [error] = await tryCatch(updateGitHubAppInstallation(callbackData.installation_id)); + const [error] = await tryCatch( + updateGitHubAppInstallation(callbackData.installation_id, organizationId) + ); if (error) { logger.error("Failed to update GitHub App installation", { error, }); - return redirectWithErrorMessage(redirectTo, request, "Failed to update GitHub App"); + return consumingSession( + await redirectWithErrorMessage(redirectTo, request, "Failed to update GitHub App") + ); } - return redirectWithSuccessMessage(redirectTo, request, "GitHub App updated successfully"); + return consumingSession( + await redirectWithSuccessMessage(redirectTo, request, "GitHub App updated successfully") + ); } case "request": { diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.environment-variables.new/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.environment-variables.new/route.tsx index d552b3104fd..02d6ef00c4d 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.environment-variables.new/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.environment-variables.new/route.tsx @@ -55,6 +55,7 @@ import { } from "~/utils/pathBuilder"; import { EnvironmentVariablesRepository } from "~/v3/environmentVariables/environmentVariablesRepository.server"; import { EnvironmentVariableKey } from "~/v3/environmentVariables/repository"; +import { findUnauthorizedEnvironmentId } from "~/v3/writableEnvironments"; const Variable = z.object({ key: EnvironmentVariableKey, @@ -164,6 +165,31 @@ export const action = dashboardAction( return json(submission.reply({ formErrors: ["Project not found"] })); } + // The submitted `environmentIds` are user-supplied. Shared env types are + // writable by any member; a DEV env only by its owner. See + // findUnauthorizedEnvironmentId. + const submittedEnvs = await prisma.runtimeEnvironment.findMany({ + where: { + projectId: project.id, + id: { in: submission.value.environmentIds }, + }, + select: { id: true, type: true, orgMember: { select: { userId: true } } }, + }); + const unauthorizedEnvironmentId = findUnauthorizedEnvironmentId( + submittedEnvs, + submission.value.environmentIds, + userId + ); + if (unauthorizedEnvironmentId) { + return json( + submission.reply({ + fieldErrors: { + environmentIds: ["One or more of the selected environments is not writable by you."], + }, + }) + ); + } + const repository = new EnvironmentVariablesRepository(prisma); const result = await repository.create(project.id, { ...submission.value, diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.environment-variables/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.environment-variables/route.tsx index b0b2a893144..cdbe364d88e 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.environment-variables/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.environment-variables/route.tsx @@ -78,6 +78,7 @@ import { v3NewEnvironmentVariablesPath, } from "~/utils/pathBuilder"; import { EnvironmentVariablesRepository } from "~/v3/environmentVariables/environmentVariablesRepository.server"; +import { findUnauthorizedEnvironmentId } from "~/v3/writableEnvironments"; import { DeleteEnvironmentVariableValue, EditEnvironmentVariableValue, @@ -267,6 +268,24 @@ export const action = dashboardAction( return json(submission.reply({ formErrors: ["Project not found"] })); } + // Per-env write gate for the mutating value actions: `environmentId` is a + // user-supplied hidden field and the repository only checks project + // membership. Mirrors the create route's check. + if (submission.value.action === "edit" || submission.value.action === "delete") { + const submittedEnvs = await prisma.runtimeEnvironment.findMany({ + where: { projectId: project.id, id: submission.value.environmentId }, + select: { id: true, type: true, orgMember: { select: { userId: true } } }, + }); + const unauthorizedEnvironmentId = findUnauthorizedEnvironmentId( + submittedEnvs, + [submission.value.environmentId], + userId + ); + if (unauthorizedEnvironmentId) { + return json(submission.reply({ formErrors: ["This environment is not writable by you."] })); + } + } + switch (submission.value.action) { case "edit": { const repository = new EnvironmentVariablesRepository(prisma); diff --git a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.schedules.$scheduleParam/route.tsx b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.schedules.$scheduleParam/route.tsx index cd433467c77..a3a0ef05470 100644 --- a/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.schedules.$scheduleParam/route.tsx +++ b/apps/webapp/app/routes/_app.orgs.$organizationSlug.projects.$projectParam.env.$envParam.schedules.$scheduleParam/route.tsx @@ -6,7 +6,6 @@ import { z } from "zod"; import { ExitIcon } from "~/assets/icons/ExitIcon"; import { LinkButton } from "~/components/primitives/Buttons"; import { ScheduleInspector } from "~/components/schedules/ScheduleInspector"; -import { prisma } from "~/db.server"; import { useEnvironment } from "~/hooks/useEnvironment"; import { useOrganization } from "~/hooks/useOrganizations"; import { useProject } from "~/hooks/useProject"; @@ -78,11 +77,7 @@ export const action = async ({ request, params }: ActionFunctionArgs) => { // `_format=json` → return JSON instead of redirecting; caller stays put. const wantsJson = formData.get("_format") === "json"; - const project = await prisma.project.findFirst({ - where: { - slug: projectParam, - }, - }); + const project = await findProjectBySlug(organizationSlug, projectParam, userId); if (!project) { const message = `No project found with slug ${projectParam}`; diff --git a/apps/webapp/app/routes/account.authorization-code.$authorizationCode/route.tsx b/apps/webapp/app/routes/account.authorization-code.$authorizationCode/route.tsx index 5bd9550e66e..067f6af4465 100644 --- a/apps/webapp/app/routes/account.authorization-code.$authorizationCode/route.tsx +++ b/apps/webapp/app/routes/account.authorization-code.$authorizationCode/route.tsx @@ -1,14 +1,19 @@ import { CheckCircleIcon } from "@heroicons/react/24/solid"; -import type { LoaderFunctionArgs } from "@remix-run/server-runtime"; -import { typedjson, useTypedLoaderData } from "remix-typedjson"; +import { Form } from "@remix-run/react"; +import type { ActionFunctionArgs, LoaderFunctionArgs } from "@remix-run/server-runtime"; +import { typedjson, useTypedActionData, useTypedLoaderData } from "remix-typedjson"; import { z } from "zod"; import { AppContainer, MainCenteredContainer } from "~/components/layout/AppLayout"; +import { Button } from "~/components/primitives/Buttons"; import { Callout } from "~/components/primitives/Callout"; import { Header1 } from "~/components/primitives/Headers"; import { Icon } from "~/components/primitives/Icon"; import { Paragraph } from "~/components/primitives/Paragraph"; import { logger } from "~/services/logger.server"; -import { createPersonalAccessTokenFromAuthorizationCode } from "~/services/personalAccessToken.server"; +import { + createPersonalAccessTokenFromAuthorizationCode, + isAuthorizationCodeMintable, +} from "~/services/personalAccessToken.server"; import { requireUserId } from "~/services/session.server"; const ParamsSchema = z.object({ @@ -20,49 +25,57 @@ const SearchParamsSchema = z.object({ clientName: z.string().optional(), }); -export const loader = async ({ request, params }: LoaderFunctionArgs) => { - const userId = await requireUserId(request); - +function parseParams(params: unknown) { const parsedParams = ParamsSchema.safeParse(params); - if (!parsedParams.success) { logger.info("Invalid params", { params }); - throw new Response(undefined, { - status: 400, - statusText: "Invalid params", - }); + throw new Response(undefined, { status: 400, statusText: "Invalid params" }); } + return parsedParams.data; +} +function parseSearch(request: Request) { const url = new URL(request.url); const searchObject = Object.fromEntries(url.searchParams.entries()); - const searchParams = SearchParamsSchema.safeParse(searchObject); - const source = (searchParams.success ? searchParams.data.source : undefined) ?? "cli"; const clientName = (searchParams.success ? searchParams.data.clientName : undefined) ?? "unknown"; + return { source, clientName }; +} + +// The loader only renders a consent screen; minting/binding a PAT happens in +// the `action`, behind an explicit "Authorize" POST. +export const loader = async ({ request, params }: LoaderFunctionArgs) => { + await requireUserId(request); + + const { authorizationCode } = parseParams(params); + const { source, clientName } = parseSearch(request); + + const mintable = await isAuthorizationCodeMintable(authorizationCode); + + return typedjson({ + status: mintable ? ("consent" as const) : ("invalid" as const), + source, + clientName, + }); +}; + +export const action = async ({ request, params }: ActionFunctionArgs) => { + const userId = await requireUserId(request); + + const { authorizationCode } = parseParams(params); + const { source, clientName } = parseSearch(request); try { - const _personalAccessToken = await createPersonalAccessTokenFromAuthorizationCode( - parsedParams.data.authorizationCode, - userId - ); - return typedjson({ - success: true as const, - source, - clientName, - }); + await createPersonalAccessTokenFromAuthorizationCode(authorizationCode, userId); + return typedjson({ success: true as const, source, clientName }); } catch (error) { if (error instanceof Response) { throw error; } if (error instanceof Error) { - return typedjson({ - success: false as const, - error: error.message, - source, - clientName, - }); + return typedjson({ success: false as const, error: error.message, source, clientName }); } logger.error(JSON.stringify(error)); @@ -74,32 +87,78 @@ export const loader = async ({ request, params }: LoaderFunctionArgs) => { }; export default function Page() { - const result = useTypedLoaderData(); + const loaderData = useTypedLoaderData(); + const actionData = useTypedActionData(); + + // After the consent POST: success or failure. + if (actionData) { + return ( + + {actionData.success ? ( +
+ + Successfully + authenticated + + + {getInstructionsForSource(actionData.source, actionData.clientName)} + +
+ ) : ( +
+ Authentication failed + + {actionData.error} + + + There was a problem authenticating you, please try logging in with your CLI again. + +
+ )} + + ); + } + + // Initial GET: invalid/expired code, or the consent prompt. + if (loaderData.status === "invalid") { + return ( + +
+ Authentication failed + + This login link is invalid or has expired. + + + Please try logging in with your CLI again to get a fresh link. + +
+ + ); + } + return ( + +
+ Authorize login + {getConsentPrompt(loaderData.source, loaderData.clientName)} +
+ +
+ + Only authorize if you started this login yourself. If you didn't, close this page. + +
+ + ); +} + +function AuthShell({ children }: { children: React.ReactNode }) { return ( -
- {result.success ? ( -
- - Successfully - authenticated - - {getInstructionsForSource(result.source, result.clientName)} -
- ) : ( -
- Authentication failed - - {result.error} - - - There was a problem authenticating you, please try logging in with your CLI again. - -
- )} -
+
{children}
 ); @@ -113,6 +172,18 @@ const prettyClientNames: Record = { "claude-ai": "Claude Desktop", }; +function getConsentPrompt(source: string, clientName: string) { + if (source === "mcp") { + const pretty = prettyClientNames[clientName] ?? clientName; + if (pretty && pretty !== "unknown") { + return `Authorize ${pretty} to access your Trigger.dev account?`; + } + return `Authorize this MCP client to access your Trigger.dev account?`; + } + + return `Authorize the Trigger.dev CLI to access your account?`; +} + function getInstructionsForSource(source: string, clientName: string) { if (source === "mcp") { if (clientName) { diff --git a/apps/webapp/app/routes/api.v1.authorization-code.ts b/apps/webapp/app/routes/api.v1.authorization-code.ts index 1a56ef04096..fa7a36673f9 100644 --- a/apps/webapp/app/routes/api.v1.authorization-code.ts +++ b/apps/webapp/app/routes/api.v1.authorization-code.ts @@ -3,7 +3,12 @@ import { json } from "@remix-run/server-runtime"; import type { CreateAuthorizationCodeResponse } from "@trigger.dev/core/v3"; import { env } from "~/env.server"; import { logger } from "~/services/logger.server"; +import { + AuthorizationCodeRateLimitError, + checkAuthorizationCodeMintRateLimit, +} from "~/services/authCodeRateLimiter.server"; import { createAuthorizationCode } from "~/services/personalAccessToken.server"; +import { extractClientIp } from "~/utils/extractClientIp.server"; /** Used to create an AuthorizationCode, that can then be used to obtain a Personal Access Token by logging in with the provided URL */ export async function action({ request }: ActionFunctionArgs) { @@ -14,8 +19,24 @@ export async function action({ request }: ActionFunctionArgs) { return { status: 405, body: "Method Not Allowed" }; } - //there is no authentication on this endpoint, anyone can create an AuthorizationCode. - //they're only used to allow a user to login, when they'll then receive a Personal Access Token + //this endpoint is unauthenticated (codes only allow a user to log in), so it's + //rate-limited per client IP. Keyed by X-Forwarded-For; if there's no trustworthy + //client IP we skip the limit rather than bucket everyone together. Self-hosters + //wanting per-IP limiting should front the app with a proxy that sets X-Forwarded-For. + const clientIp = extractClientIp(request.headers.get("x-forwarded-for")); + if (clientIp) { + try { + await checkAuthorizationCodeMintRateLimit(clientIp); + } catch (error) { + if (error instanceof AuthorizationCodeRateLimitError) { + return json( + { error: "Too many requests, please try again later." }, + { status: 429, headers: { "Retry-After": Math.ceil(error.retryAfter / 1000).toString() } } + ); + } + throw error; + } + } try { const authorizationCode = await createAuthorizationCode(); diff --git a/apps/webapp/app/routes/api.v1.deployments.$deploymentId.cancel.ts b/apps/webapp/app/routes/api.v1.deployments.$deploymentId.cancel.ts index 56b93fe17e3..d14f7a8bc22 100644 --- a/apps/webapp/app/routes/api.v1.deployments.$deploymentId.cancel.ts +++ b/apps/webapp/app/routes/api.v1.deployments.$deploymentId.cancel.ts @@ -45,7 +45,7 @@ export async function action({ request, params }: ActionFunctionArgs) { const deploymentService = new DeploymentService(); return await deploymentService - .cancelDeployment(authenticatedEnv, deploymentId, { + .cancelDeployment({ id: authenticatedEnv.id }, deploymentId, { canceledReason: body.data.reason, }) .match( diff --git a/apps/webapp/app/routes/api.v1.schedules.$scheduleId.activate.ts b/apps/webapp/app/routes/api.v1.schedules.$scheduleId.activate.ts index 9cc8b7173c7..99ca3159954 100644 --- a/apps/webapp/app/routes/api.v1.schedules.$scheduleId.activate.ts +++ b/apps/webapp/app/routes/api.v1.schedules.$scheduleId.activate.ts @@ -2,7 +2,7 @@ import type { ActionFunctionArgs } from "@remix-run/server-runtime"; import { json } from "@remix-run/server-runtime"; import { z } from "zod"; import { prisma } from "~/db.server"; -import { scheduleUniqWhereClause, scheduleWhereClause } from "~/models/schedules.server"; +import { getScheduleEnvVisibility, scheduleUniqWhereClause } from "~/models/schedules.server"; import { ViewSchedulePresenter } from "~/presenters/v3/ViewSchedulePresenter.server"; import { authenticateApiRequest } from "~/services/apiAuth.server"; import { logger } from "~/services/logger.server"; @@ -34,14 +34,17 @@ export async function action({ request, params }: ActionFunctionArgs) { } try { - const existingSchedule = await prisma.taskSchedule.findFirst({ - where: scheduleWhereClause( - authenticationResult.environment.projectId, - parsedParams.data.scheduleId - ), - }); - - if (!existingSchedule) { + // Env-scoped API keys can only toggle schedules that have an instance in + // their own environment. Without this a key scoped to one environment + // could enable/disable a schedule that only runs in another environment + // of the same project. + const visibility = await getScheduleEnvVisibility( + prisma, + authenticationResult.environment.projectId, + parsedParams.data.scheduleId, + authenticationResult.environment.id + ); + if (visibility.status !== "visible") { return json({ error: "Schedule not found" }, { status: 404 }); } diff --git a/apps/webapp/app/routes/api.v1.schedules.$scheduleId.deactivate.ts b/apps/webapp/app/routes/api.v1.schedules.$scheduleId.deactivate.ts index eb985bea728..3c9514ef8e3 100644 --- a/apps/webapp/app/routes/api.v1.schedules.$scheduleId.deactivate.ts +++ b/apps/webapp/app/routes/api.v1.schedules.$scheduleId.deactivate.ts @@ -2,7 +2,7 @@ import type { ActionFunctionArgs } from "@remix-run/server-runtime"; import { json } from "@remix-run/server-runtime"; import { z } from "zod"; import { prisma } from "~/db.server"; -import { scheduleUniqWhereClause, scheduleWhereClause } from "~/models/schedules.server"; +import { getScheduleEnvVisibility, scheduleUniqWhereClause } from "~/models/schedules.server"; import { ViewSchedulePresenter } from "~/presenters/v3/ViewSchedulePresenter.server"; import { authenticateApiRequest } from "~/services/apiAuth.server"; import { logger } from "~/services/logger.server"; @@ -34,14 +34,17 @@ export async function action({ request, params }: ActionFunctionArgs) { } try { - const existingSchedule = await prisma.taskSchedule.findFirst({ - where: scheduleWhereClause( - authenticationResult.environment.projectId, - parsedParams.data.scheduleId - ), - }); - - if (!existingSchedule) { + // Env-scoped API keys can only toggle schedules that have an instance in + // their own environment. Without this a key scoped to one environment + // could enable/disable a schedule that only runs in another environment + // of the same project. + const visibility = await getScheduleEnvVisibility( + prisma, + authenticationResult.environment.projectId, + parsedParams.data.scheduleId, + authenticationResult.environment.id + ); + if (visibility.status !== "visible") { return json({ error: "Schedule not found" }, { status: 404 }); } diff --git a/apps/webapp/app/routes/api.v1.schedules.$scheduleId.ts b/apps/webapp/app/routes/api.v1.schedules.$scheduleId.ts index 9da9ffcb121..4f7e8d8c164 100644 --- a/apps/webapp/app/routes/api.v1.schedules.$scheduleId.ts +++ b/apps/webapp/app/routes/api.v1.schedules.$scheduleId.ts @@ -5,7 +5,7 @@ import { UpdateScheduleOptions } from "@trigger.dev/core/v3"; import { z } from "zod"; import { Prisma, prisma } from "~/db.server"; import { clientSafeErrorMessage } from "~/utils/prismaErrors"; -import { scheduleUniqWhereClause } from "~/models/schedules.server"; +import { getScheduleEnvVisibility, scheduleUniqWhereClause } from "~/models/schedules.server"; import { ViewSchedulePresenter } from "~/presenters/v3/ViewSchedulePresenter.server"; import { authenticateApiRequest } from "~/services/apiAuth.server"; import { logger } from "~/services/logger.server"; @@ -38,6 +38,16 @@ export async function action({ request, params }: ActionFunctionArgs) { switch (method) { case "DELETE": { + const visibility = await getScheduleEnvVisibility( + prisma, + authenticationResult.environment.projectId, + parsedParams.data.scheduleId, + authenticationResult.environment.id + ); + if (visibility.status !== "visible") { + return json({ error: "Schedule not found" }, { status: 404 }); + } + try { const deletedSchedule = await prisma.taskSchedule.delete({ where: scheduleUniqWhereClause( @@ -76,6 +86,19 @@ export async function action({ request, params }: ActionFunctionArgs) { return json({ error: "Invalid request body", issues: body.error.issues }, { status: 400 }); } + // Env-scoped API keys can't see or mutate a schedule whose + // instances live in a different environment. "hidden" → refuse; + // "missing" → fall through to the upsert's create path. + const visibility = await getScheduleEnvVisibility( + prisma, + authenticationResult.environment.projectId, + parsedParams.data.scheduleId, + authenticationResult.environment.id + ); + if (visibility.status === "hidden") { + return json({ error: "Schedule not found" }, { status: 404 }); + } + const service = new UpsertTaskScheduleService(); try { @@ -137,6 +160,16 @@ export async function loader({ request, params }: LoaderFunctionArgs) { ); } + const visibility = await getScheduleEnvVisibility( + prisma, + authenticationResult.environment.projectId, + parsedParams.data.scheduleId, + authenticationResult.environment.id + ); + if (visibility.status !== "visible") { + return json({ error: "Schedule not found" }, { status: 404 }); + } + const presenter = new ViewSchedulePresenter(); const result = await presenter.call({ diff --git a/apps/webapp/app/routes/api.v1.token.ts b/apps/webapp/app/routes/api.v1.token.ts index 7763f9bb56e..e3c85652035 100644 --- a/apps/webapp/app/routes/api.v1.token.ts +++ b/apps/webapp/app/routes/api.v1.token.ts @@ -4,6 +4,10 @@ import type { GetPersonalAccessTokenResponse } from "@trigger.dev/core/v3"; import { GetPersonalAccessTokenRequestSchema } from "@trigger.dev/core/v3"; import { generateErrorMessage } from "zod-error"; import { logger } from "~/services/logger.server"; +import { + AuthorizationCodeRateLimitError, + checkAuthorizationCodeTokenPollRateLimit, +} from "~/services/authCodeRateLimiter.server"; import { getPersonalAccessTokenFromAuthorizationCode } from "~/services/personalAccessToken.server"; import { clientSafeErrorMessage } from "~/utils/prismaErrors"; @@ -25,6 +29,20 @@ export async function action({ request }: ActionFunctionArgs) { return json({ error: generateErrorMessage(body.error.issues) }, { status: 422 }); } + // Per-code rate limit (keyed by the code, not the IP, so the CLI's poll loop + // isn't broken behind a shared NAT). + try { + await checkAuthorizationCodeTokenPollRateLimit(body.data.authorizationCode); + } catch (error) { + if (error instanceof AuthorizationCodeRateLimitError) { + return json( + { error: "Too many requests, please try again later." }, + { status: 429, headers: { "Retry-After": Math.ceil(error.retryAfter / 1000).toString() } } + ); + } + throw error; + } + try { const personalAccessToken = await getPersonalAccessTokenFromAuthorizationCode( body.data.authorizationCode diff --git a/apps/webapp/app/routes/engine.v1.worker-actions.runs.$runFriendlyId.snapshots.$snapshotFriendlyId.attempts.complete.ts b/apps/webapp/app/routes/engine.v1.worker-actions.runs.$runFriendlyId.snapshots.$snapshotFriendlyId.attempts.complete.ts index 051b6d4d04f..342ca3f2822 100644 --- a/apps/webapp/app/routes/engine.v1.worker-actions.runs.$runFriendlyId.snapshots.$snapshotFriendlyId.attempts.complete.ts +++ b/apps/webapp/app/routes/engine.v1.worker-actions.runs.$runFriendlyId.snapshots.$snapshotFriendlyId.attempts.complete.ts @@ -18,6 +18,7 @@ export const action = createActionWorkerApiRoute( body, params, runnerId, + environmentId, }): Promise> => { const { completion } = body; const { runFriendlyId, snapshotFriendlyId } = params; @@ -27,6 +28,7 @@ export const action = createActionWorkerApiRoute( snapshotFriendlyId, completion, runnerId, + environmentId, }); return json({ result: completeResult }); diff --git a/apps/webapp/app/routes/engine.v1.worker-actions.runs.$runFriendlyId.snapshots.$snapshotFriendlyId.attempts.start.ts b/apps/webapp/app/routes/engine.v1.worker-actions.runs.$runFriendlyId.snapshots.$snapshotFriendlyId.attempts.start.ts index a28a362ed93..558fa0a0c01 100644 --- a/apps/webapp/app/routes/engine.v1.worker-actions.runs.$runFriendlyId.snapshots.$snapshotFriendlyId.attempts.start.ts +++ b/apps/webapp/app/routes/engine.v1.worker-actions.runs.$runFriendlyId.snapshots.$snapshotFriendlyId.attempts.start.ts @@ -18,6 +18,7 @@ export const action = createActionWorkerApiRoute( body, params, runnerId, + environmentId, }): Promise> => { const { runFriendlyId, snapshotFriendlyId } = params; @@ -26,6 +27,7 @@ export const action = createActionWorkerApiRoute( snapshotFriendlyId, isWarmStart: body.isWarmStart, runnerId, + environmentId, }); return json(runExecutionData); diff --git a/apps/webapp/app/routes/engine.v1.worker-actions.runs.$runFriendlyId.snapshots.$snapshotFriendlyId.continue.ts b/apps/webapp/app/routes/engine.v1.worker-actions.runs.$runFriendlyId.snapshots.$snapshotFriendlyId.continue.ts index 3c264adcd23..1aef207aac4 100644 --- a/apps/webapp/app/routes/engine.v1.worker-actions.runs.$runFriendlyId.snapshots.$snapshotFriendlyId.continue.ts +++ b/apps/webapp/app/routes/engine.v1.worker-actions.runs.$runFriendlyId.snapshots.$snapshotFriendlyId.continue.ts @@ -17,6 +17,7 @@ export const loader = createLoaderWorkerApiRoute( authenticatedWorker, params, runnerId, + environmentId, }): Promise> => { const { runFriendlyId, snapshotFriendlyId } = params; @@ -27,10 +28,17 @@ export const loader = createLoaderWorkerApiRoute( runFriendlyId, snapshotFriendlyId, runnerId, + environmentId, }); return json(continuationResult); } catch (error) { + // An authorization rejection is thrown as a Response; propagate it as-is rather than + // masking it as a generic 422. + if (error instanceof Response) { + throw error; + } + logger.warn("Failed to continue run execution", { runFriendlyId, snapshotFriendlyId, diff --git a/apps/webapp/app/routes/engine.v1.worker-actions.runs.$runFriendlyId.snapshots.latest.ts b/apps/webapp/app/routes/engine.v1.worker-actions.runs.$runFriendlyId.snapshots.latest.ts index 73df07748b9..46d84741061 100644 --- a/apps/webapp/app/routes/engine.v1.worker-actions.runs.$runFriendlyId.snapshots.latest.ts +++ b/apps/webapp/app/routes/engine.v1.worker-actions.runs.$runFriendlyId.snapshots.latest.ts @@ -13,11 +13,13 @@ export const loader = createLoaderWorkerApiRoute( async ({ authenticatedWorker, params, + environmentId, }): Promise> => { const { runFriendlyId } = params; const executionData = await authenticatedWorker.getLatestSnapshot({ runFriendlyId, + environmentId, }); if (!executionData) { diff --git a/apps/webapp/app/routes/engine.v1.worker-actions.runs.$runFriendlyId.snapshots.since.$snapshotId.ts b/apps/webapp/app/routes/engine.v1.worker-actions.runs.$runFriendlyId.snapshots.since.$snapshotId.ts index 87e1ed5eedf..7b91dc476c7 100644 --- a/apps/webapp/app/routes/engine.v1.worker-actions.runs.$runFriendlyId.snapshots.since.$snapshotId.ts +++ b/apps/webapp/app/routes/engine.v1.worker-actions.runs.$runFriendlyId.snapshots.since.$snapshotId.ts @@ -14,12 +14,14 @@ export const loader = createLoaderWorkerApiRoute( async ({ authenticatedWorker, params, + environmentId, }): Promise> => { const { runFriendlyId, snapshotId } = params; const snapshots = await authenticatedWorker.getSnapshotsSince({ runFriendlyId, snapshotId, + environmentId, }); if (!snapshots) { diff --git a/apps/webapp/app/routes/logout.tsx b/apps/webapp/app/routes/logout.tsx index 20b4705f352..fe6368ee3be 100644 --- a/apps/webapp/app/routes/logout.tsx +++ b/apps/webapp/app/routes/logout.tsx @@ -1,6 +1,8 @@ -import { type ActionFunction, type LoaderFunction } from "@remix-run/node"; +import { redirect, type ActionFunction, type LoaderFunction } from "@remix-run/node"; +import { env } from "~/env.server"; import { authenticator } from "~/services/auth.server"; import { sanitizeRedirectPath } from "~/utils"; +import { isSameOriginNavigation } from "~/utils/sameOriginNavigation"; import { SSO_SESSION_EXPIRED_REASON } from "~/utils/ssoSession"; function logoutRedirectTo(request: Request): string { @@ -18,5 +20,10 @@ export const action: ActionFunction = async ({ request }) => { }; export const loader: LoaderFunction = async ({ request }) => { + // GET /logout is state-changing, so reject cross-site navigations. + if (!isSameOriginNavigation(request, env.LOGIN_ORIGIN)) { + throw redirect("/"); + } + return await authenticator.logout(request, { redirectTo: logoutRedirectTo(request) }); }; diff --git a/apps/webapp/app/routes/realtime.v1.runs.ts b/apps/webapp/app/routes/realtime.v1.runs.ts index ca2998f2972..465e7ab4f41 100644 --- a/apps/webapp/app/routes/realtime.v1.runs.ts +++ b/apps/webapp/app/routes/realtime.v1.runs.ts @@ -2,6 +2,7 @@ import { z } from "zod"; import { getRequestAbortSignal } from "~/services/httpAsyncStorage.server"; import { resolveRealtimeStreamClient } from "~/services/realtime/resolveRealtimeStreamClient.server"; import { anyResource, createLoaderApiRoute } from "~/services/routeBuilders/apiBuilder.server"; +import { UNSAFE_REALTIME_TAG_CHARS } from "~/v3/electricShape.server"; const SearchParamsSchema = z.object({ tags: z @@ -9,6 +10,20 @@ const SearchParamsSchema = z.object({ .optional() .transform((value) => { return value ? value.split(",") : undefined; + }) + .superRefine((tags, ctx) => { + if (!tags) return; + for (const tag of tags) { + // Mirror the runtime sanitiser's reject list so the API returns 400 + // instead of a 500. Single quotes are allowed — escaped downstream. + if (UNSAFE_REALTIME_TAG_CHARS.test(tag) || tag.length === 0) { + ctx.addIssue({ + code: z.ZodIssueCode.custom, + message: `Invalid tag: ${JSON.stringify(tag)}`, + }); + return; + } + } }), createdAt: z.string().optional(), }); diff --git a/apps/webapp/app/routes/realtime.v1.sessions.$session.$io.ts b/apps/webapp/app/routes/realtime.v1.sessions.$session.$io.ts index e846e851be8..b4ff0ba9500 100644 --- a/apps/webapp/app/routes/realtime.v1.sessions.$session.$io.ts +++ b/apps/webapp/app/routes/realtime.v1.sessions.$session.$io.ts @@ -34,6 +34,16 @@ const { action } = createActionApiRoute( }, }, async ({ params, authentication }) => { + // `.out` is the agent→client channel. Only PRIVATE (secret key) auth — + // i.e. the agent run itself — may initialize it. Session-scoped JWTs carry + // `write:sessions:` for `.in`; without this gate they could obtain + // credentials to forge assistant chunks on their own session's `.out`. + if (params.io === "out" && authentication.type !== "PRIVATE") { + return new Response("Initializing the out channel requires secret key authentication", { + status: 403, + }); + } + // Row-optional addressing. The agent calls PUT initialize as part // of `session.out.writer()`, by which time it has already created // the row at bind, so a missing row here is an unusual case diff --git a/apps/webapp/app/routes/resources.$projectId.deployments.$deploymentShortCode.cancel.ts b/apps/webapp/app/routes/resources.$projectId.deployments.$deploymentShortCode.cancel.ts index dd1c452c51c..239d0982f9e 100644 --- a/apps/webapp/app/routes/resources.$projectId.deployments.$deploymentShortCode.cancel.ts +++ b/apps/webapp/app/routes/resources.$projectId.deployments.$deploymentShortCode.cancel.ts @@ -75,7 +75,7 @@ export const action = dashboardAction( prisma.workerDeployment.findUnique({ select: { friendlyId: true, - projectId: true, + environmentId: true, }, where: { projectId_shortCode: { @@ -96,10 +96,7 @@ export const action = dashboardAction( const result = await verifyProjectMembership() .andThen(findDeploymentFriendlyId) .andThen((deployment) => - deploymentService.cancelDeployment( - { projectId: deployment.projectId }, - deployment.friendlyId - ) + deploymentService.cancelDeployment({ id: deployment.environmentId }, deployment.friendlyId) ); if (result.isErr()) { diff --git a/apps/webapp/app/routes/sync.traces.$traceId.ts b/apps/webapp/app/routes/sync.traces.$traceId.ts index 1a06dd48854..d7695014b56 100644 --- a/apps/webapp/app/routes/sync.traces.$traceId.ts +++ b/apps/webapp/app/routes/sync.traces.$traceId.ts @@ -1,15 +1,33 @@ import type { LoaderFunctionArgs } from "@remix-run/node"; +import { z } from "zod"; import { $replica } from "~/db.server"; import { env } from "~/env.server"; import { logger } from "~/services/logger.server"; import { getUserId } from "~/services/session.server"; import { longPollingFetch } from "~/utils/longPollingFetch"; +import { + OtelTraceIdSchema, + RESERVED_ELECTRIC_SHAPE_PARAMS, + buildElectricTraceWhereClause, +} from "~/v3/electricShape.server"; + +const Params = z.object({ + traceId: OtelTraceIdSchema, +}); export async function loader({ params, request }: LoaderFunctionArgs) { try { const userId = await getUserId(request); - logger.log(`/sync/traces/${params.traceId}`, { userId }); + const parsedParams = Params.safeParse(params); + if (!parsedParams.success) { + // Treat a malformed traceId as not-found rather than 400 to avoid + // signalling the validator. + return new Response("Not found", { status: 404 }); + } + const { traceId } = parsedParams.data; + + logger.log(`/sync/traces/${traceId}`, { userId }); if (!userId) { return new Response("No user found in cookie", { status: 401 }); @@ -20,7 +38,7 @@ export async function loader({ params, request }: LoaderFunctionArgs) { organizationId: true, }, where: { - traceId: params.traceId, + traceId, }, }); @@ -41,11 +59,19 @@ export async function loader({ params, request }: LoaderFunctionArgs) { const url = new URL(request.url); const originUrl = new URL(`${env.ELECTRIC_ORIGIN}/v1/shape/public."TaskEvent"`); + // Strip params we set ourselves so the caller can't override them. url.searchParams.forEach((value, key) => { + if (RESERVED_ELECTRIC_SHAPE_PARAMS.has(key)) return; originUrl.searchParams.set(key, value); }); - originUrl.searchParams.set("where", `"traceId"='${params.traceId}'`); + originUrl.searchParams.set( + "where", + buildElectricTraceWhereClause({ + traceId, + scope: { column: "organizationId", id: trace.organizationId }, + }) + ); const finalUrl = originUrl.toString(); diff --git a/apps/webapp/app/routes/sync.traces.runs.$traceId.ts b/apps/webapp/app/routes/sync.traces.runs.$traceId.ts index 5fd1f7c65c0..24870cf2f49 100644 --- a/apps/webapp/app/routes/sync.traces.runs.$traceId.ts +++ b/apps/webapp/app/routes/sync.traces.runs.$traceId.ts @@ -5,17 +5,27 @@ import { env } from "~/env.server"; import { logger } from "~/services/logger.server"; import { getUserId } from "~/services/session.server"; import { longPollingFetch } from "~/utils/longPollingFetch"; -import { runStore } from "~/v3/runStore.server"; +import { + OtelTraceIdSchema, + RESERVED_ELECTRIC_SHAPE_PARAMS, + buildElectricTraceWhereClause, +} from "~/v3/electricShape.server"; import { controlPlaneResolver } from "~/v3/runOpsMigration/controlPlaneResolver.server"; +import { runStore } from "~/v3/runStore.server"; const Params = z.object({ - traceId: z.string(), + traceId: OtelTraceIdSchema, }); export async function loader({ params, request }: LoaderFunctionArgs) { try { const userId = await getUserId(request); - const { traceId } = Params.parse(params); + + const parsedParams = Params.safeParse(params); + if (!parsedParams.success) { + return new Response("Not found", { status: 404 }); + } + const { traceId } = parsedParams.data; logger.log(`/sync/runs/${traceId}`, { userId }); @@ -29,6 +39,7 @@ export async function loader({ params, request }: LoaderFunctionArgs) { }, { select: { + projectId: true, runtimeEnvironmentId: true, }, }, @@ -40,7 +51,7 @@ export async function loader({ params, request }: LoaderFunctionArgs) { // primary before 404ing so a live run's realtime trace feed isn't spuriously not-found. run = await runStore.findRunOnPrimary( { traceId }, - { select: { runtimeEnvironmentId: true } } + { select: { projectId: true, runtimeEnvironmentId: true } } ); } @@ -67,11 +78,22 @@ export async function loader({ params, request }: LoaderFunctionArgs) { const url = new URL(request.url); const originUrl = new URL(`${env.ELECTRIC_ORIGIN}/v1/shape/public."TaskRun"`); + // Strip params we set ourselves so the caller can't override them. url.searchParams.forEach((value, key) => { + if (RESERVED_ELECTRIC_SHAPE_PARAMS.has(key)) return; originUrl.searchParams.set(key, value); }); - originUrl.searchParams.set("where", `"traceId"='${traceId}'`); + originUrl.searchParams.set( + "where", + // Scope by non-null projectId, not the nullable organizationId (legacy + // rows would vanish). Tenant-safe: membership was verified against this + // project's org and a trace's runs all live in one project. + buildElectricTraceWhereClause({ + traceId, + scope: { column: "projectId", id: run.projectId }, + }) + ); const finalUrl = originUrl.toString(); diff --git a/apps/webapp/app/services/apiRateLimit.server.ts b/apps/webapp/app/services/apiRateLimit.server.ts index fbd59cc74cd..837d3ee5895 100644 --- a/apps/webapp/app/services/apiRateLimit.server.ts +++ b/apps/webapp/app/services/apiRateLimit.server.ts @@ -48,6 +48,9 @@ export const apiRateLimiter = authorizationRateLimitMiddleware({ // Allow /api/v1/tasks/:id/callback/:secret pathWhiteList: [ "/api/internal/stripe_webhooks", + // Keep allowlisted: these CLI endpoints are intentionally unauthenticated, + // so this Authorization-header-keyed limiter would 401 them. They are + // throttled separately by authCodeRateLimiter.server.ts. "/api/v1/authorization-code", "/api/v1/token", "/api/v1/usage/ingest", diff --git a/apps/webapp/app/services/authCodeRateLimiter.server.ts b/apps/webapp/app/services/authCodeRateLimiter.server.ts new file mode 100644 index 00000000000..5c88c094c2c --- /dev/null +++ b/apps/webapp/app/services/authCodeRateLimiter.server.ts @@ -0,0 +1,85 @@ +import { Ratelimit } from "@upstash/ratelimit"; +import { createHash } from "node:crypto"; +import { env } from "~/env.server"; +import { createRedisRateLimitClient, RateLimiter } from "~/services/rateLimiter.server"; +import { singleton } from "~/utils/singleton"; + +/** + * Rate limiting for the unauthenticated CLI auth-code endpoints + * (`/api/v1/authorization-code` mint + `/api/v1/token` poll). The global + * limiter keys on the Authorization header, which these endpoints don't carry, + * so it can't throttle them — this module does. + */ +export class AuthorizationCodeRateLimitError extends Error { + public readonly retryAfter: number; + + constructor(retryAfter: number) { + super("Authorization code rate limit exceeded."); + this.name = "AuthorizationCodeRateLimitError"; + this.retryAfter = retryAfter; + } +} + +function getRedisClient() { + return createRedisRateLimitClient({ + port: env.RATE_LIMIT_REDIS_PORT, + host: env.RATE_LIMIT_REDIS_HOST, + username: env.RATE_LIMIT_REDIS_USERNAME, + password: env.RATE_LIMIT_REDIS_PASSWORD, + tlsDisabled: env.RATE_LIMIT_REDIS_TLS_DISABLED === "true", + clusterMode: env.RATE_LIMIT_REDIS_CLUSTER_MODE_ENABLED === "1", + }); +} + +// Minting is unauthenticated and a real login mints one code. Cap per IP, with +// headroom for many users behind a shared NAT. +const authorizationCodeMintIpRateLimiter = singleton( + "authorizationCodeMintIpRateLimiter", + () => + new RateLimiter({ + redisClient: getRedisClient(), + keyPrefix: "auth:authcode:mint:ip", + limiter: Ratelimit.slidingWindow(30, "1 m"), // 30 code mints / min / IP + logSuccess: false, + logFailure: true, + }) +); + +// Keyed by the code, not the IP: the CLI polls this endpoint ~1/s per login, so +// IP-keying would break logins behind a shared NAT. The ~60/min cadence stays +// under the cap. The code is hashed first so it never lands in a Redis key or log. +const authorizationCodeTokenPollRateLimiter = singleton( + "authorizationCodeTokenPollRateLimiter", + () => + new RateLimiter({ + redisClient: getRedisClient(), + keyPrefix: "auth:authcode:token:code", + limiter: Ratelimit.slidingWindow(100, "1 m"), // 100 polls / min / code (CLI polls ~60/min) + logSuccess: false, + logFailure: false, + }) +); + +function hashCode(code: string): string { + return createHash("sha256").update(code).digest("hex").slice(0, 32); +} + +export async function checkAuthorizationCodeMintRateLimit(ip: string): Promise { + const result = await authorizationCodeMintIpRateLimiter.limit(ip); + + if (!result.success) { + const retryAfter = new Date(result.reset).getTime() - Date.now(); + throw new AuthorizationCodeRateLimitError(retryAfter); + } +} + +export async function checkAuthorizationCodeTokenPollRateLimit( + authorizationCode: string +): Promise { + const result = await authorizationCodeTokenPollRateLimiter.limit(hashCode(authorizationCode)); + + if (!result.success) { + const retryAfter = new Date(result.reset).getTime() - Date.now(); + throw new AuthorizationCodeRateLimitError(retryAfter); + } +} diff --git a/apps/webapp/app/services/gitHub.server.ts b/apps/webapp/app/services/gitHub.server.ts index e67895fe066..00ec2087f04 100644 --- a/apps/webapp/app/services/gitHub.server.ts +++ b/apps/webapp/app/services/gitHub.server.ts @@ -58,26 +58,32 @@ export async function linkGitHubAppInstallation( } /** - * Links a GitHub App installation to a Trigger organization + * Updates a GitHub App installation owned by the given Trigger organization */ -export async function updateGitHubAppInstallation(installationId: number): Promise { +export async function updateGitHubAppInstallation( + installationId: number, + organizationId: string +): Promise { if (!githubApp) { throw new Error("GitHub App is not enabled"); } - const octokit = await githubApp.getInstallationOctokit(installationId); - const { data: installation } = await octokit.rest.apps.getInstallation({ - installation_id: installationId, - }); - + // Scope the lookup to the caller's organization so a cross-tenant + // installation_id cannot update another org's record. Resolve ownership + // before calling GitHub to avoid burning the victim's API rate limit. const existingInstallation = await prisma.githubAppInstallation.findFirst({ - where: { appInstallationId: installationId }, + where: { appInstallationId: installationId, organizationId }, }); if (!existingInstallation) { throw new Error("GitHub App installation not found"); } + const octokit = await githubApp.getInstallationOctokit(installationId); + const { data: installation } = await octokit.rest.apps.getInstallation({ + installation_id: installationId, + }); + const repositorySelection = installation.repository_selection === "all" ? "ALL" : "SELECTED"; // repos are updated asynchronously via webhook events diff --git a/apps/webapp/app/services/otlpRateLimit.server.ts b/apps/webapp/app/services/otlpRateLimit.server.ts new file mode 100644 index 00000000000..eef07990ecd --- /dev/null +++ b/apps/webapp/app/services/otlpRateLimit.server.ts @@ -0,0 +1,91 @@ +import { Ratelimit } from "@upstash/ratelimit"; +import type { NextFunction, Request, Response } from "express"; +import { env } from "~/env.server"; +import { createRedisRateLimitClient, RateLimiter } from "~/services/rateLimiter.server"; +import { extractClientIp } from "~/utils/extractClientIp.server"; +import { singleton } from "~/utils/singleton"; +import { logger } from "./logger.server"; + +const OTLP_PATH = /^\/otel\//i; + +function getOtlpIpRateLimiter() { + return singleton( + "otlpIpRateLimiter", + () => + new RateLimiter({ + redisClient: createRedisRateLimitClient({ + port: env.RATE_LIMIT_REDIS_PORT, + host: env.RATE_LIMIT_REDIS_HOST, + username: env.RATE_LIMIT_REDIS_USERNAME, + password: env.RATE_LIMIT_REDIS_PASSWORD, + tlsDisabled: env.RATE_LIMIT_REDIS_TLS_DISABLED === "true", + clusterMode: env.RATE_LIMIT_REDIS_CLUSTER_MODE_ENABLED === "1", + }), + keyPrefix: "otlp:ip", + limiter: Ratelimit.slidingWindow( + env.OTLP_RATE_LIMIT_MAX, + env.OTLP_RATE_LIMIT_WINDOW as Parameters[1] + ), + logSuccess: false, + logFailure: true, + }) + ); +} + +/** + * Per-IP rate limiter for the OTLP ingestion endpoints (`/otel/*`). + * + * These endpoints are currently unauthenticated (see SEC-98), so the source IP + * is the only identity available to key on. This bounds unauthenticated + * request rates and is NOT a substitute for authenticating the + * endpoints. It fails open (allows the request) whenever the source cannot be + * identified or the limiter backend errors, so a limiter outage never drops + * legitimate telemetry. + * + * Opt-in (disabled unless `OTLP_RATE_LIMIT_ENABLED=1`). Because it keys on the + * source IP, two preconditions must hold before enabling it, or it can drop + * legitimate telemetry: + * + * 1. Each client must present a distinct IP. Where many clients share one + * egress IP (e.g. behind NAT or a shared proxy) their traffic collapses + * into a single bucket and can be throttled together. Size + * `OTLP_RATE_LIMIT_MAX` for the aggregate volume of a shared source, not a + * single client. + * 2. The IP must be trustworthy. `extractClientIp` takes the last + * `X-Forwarded-For` hop, which is only spoof-resistant behind a proxy that + * appends the real client IP. Without such a proxy the value is + * client-controlled and the per-IP bound is bypassable. + */ +export async function otlpRateLimiter(req: Request, res: Response, next: NextFunction) { + if (env.OTLP_RATE_LIMIT_ENABLED !== "1") { + return next(); + } + + if (req.method.toUpperCase() === "OPTIONS" || !OTLP_PATH.test(req.path)) { + return next(); + } + + const xff = req.headers["x-forwarded-for"]; + const ip = extractClientIp(Array.isArray(xff) ? xff.join(",") : (xff ?? null)) ?? req.ip; + + if (!ip) { + // Fail open: without a source we cannot fairly rate limit. + return next(); + } + + try { + const { success, reset } = await getOtlpIpRateLimiter().limit(ip); + + if (!success) { + const retryAfterSeconds = Math.max(1, Math.ceil((reset - Date.now()) / 1000)); + res.setHeader("Retry-After", retryAfterSeconds.toString()); + res.status(429).send("Too Many Requests"); + return; + } + } catch (error) { + // Fail open: a rate-limiter backend outage must not drop telemetry. + logger.warn("otlpRateLimiter: limiter error, allowing request", { error }); + } + + return next(); +} diff --git a/apps/webapp/app/services/personalAccessToken.server.ts b/apps/webapp/app/services/personalAccessToken.server.ts index a0821d6602e..d80ca6fa3f4 100644 --- a/apps/webapp/app/services/personalAccessToken.server.ts +++ b/apps/webapp/app/services/personalAccessToken.server.ts @@ -18,6 +18,10 @@ const tokenGenerator = customAlphabet("123456789abcdefghijkmnopqrstuvwxyz", toke // staleness is fine. export const PAT_LAST_ACCESSED_THROTTLE_MS = 5 * 60 * 1000; +// How long an unconsumed CLI authorization code stays valid. Shared constant so +// the mint and read paths can't drift. +export const AUTHORIZATION_CODE_TTL_MS = 10 * 60 * 1000; + type CreatePersonalAccessTokenOptions = { name: string; userId: string; @@ -60,16 +64,16 @@ export type ObfuscatedPersonalAccessToken = Awaited< /** Gets a PersonalAccessToken from an Auth Code, this only works within 10 mins of the auth code being created */ export async function getPersonalAccessTokenFromAuthorizationCode(authorizationCode: string) { - //only allow authorization codes that were created less than 10 mins ago - const tenMinutesAgo = new Date(Date.now() - 10 * 60 * 1000); - const code = await prisma.authorizationCode.findUnique({ + // Only allow authorization codes created within the short consent window. + const validAfter = new Date(Date.now() - AUTHORIZATION_CODE_TTL_MS); + const code = await prisma.authorizationCode.findFirst({ select: { personalAccessToken: true, }, where: { code: authorizationCode, createdAt: { - gte: tenMinutesAgo, + gte: validAfter, }, }, }); @@ -280,6 +284,27 @@ export function isPersonalAccessToken(token: string) { return token.startsWith(tokenPrefix); } +/** + * Read-only check that an authorization code is still mintable: it exists, is + * unconsumed (`personalAccessTokenId: null`), and within the TTL. Lets the + * consent-screen loader show Authorize vs expired/invalid without minting a PAT. + */ +export async function isAuthorizationCodeMintable( + authorizationCode: string, + prismaClient = prisma +): Promise { + const validAfter = new Date(Date.now() - AUTHORIZATION_CODE_TTL_MS); + const code = await prismaClient.authorizationCode.findFirst({ + where: { + code: authorizationCode, + personalAccessTokenId: null, + createdAt: { gte: validAfter }, + }, + select: { id: true }, + }); + return code !== null; +} + export function createAuthorizationCode() { return prisma.authorizationCode.create({ data: { @@ -293,14 +318,14 @@ export async function createPersonalAccessTokenFromAuthorizationCode( authorizationCode: string, userId: string ) { - //only allow authorization codes that were created less than 10 mins ago - const tenMinutesAgo = new Date(Date.now() - 10 * 60 * 1000); - const code = await prisma.authorizationCode.findUnique({ + // Only allow authorization codes created within the short consent window. + const validAfter = new Date(Date.now() - AUTHORIZATION_CODE_TTL_MS); + const code = await prisma.authorizationCode.findFirst({ where: { code: authorizationCode, personalAccessTokenId: null, createdAt: { - gte: tenMinutesAgo, + gte: validAfter, }, }, }); diff --git a/apps/webapp/app/services/realtimeClient.server.ts b/apps/webapp/app/services/realtimeClient.server.ts index 9797eeebf40..bf86340641b 100644 --- a/apps/webapp/app/services/realtimeClient.server.ts +++ b/apps/webapp/app/services/realtimeClient.server.ts @@ -15,6 +15,7 @@ import { RedisCacheStore } from "./unkey/redisCacheStore.server"; import { env } from "~/env.server"; import type { API_VERSIONS } from "~/api/versions"; import { CURRENT_API_VERSION } from "~/api/versions"; +import { sanitizeRealtimeTagsForSql } from "~/v3/electricShape.server"; export interface CachedLimitProvider { getCachedLimit: (organizationId: string, defaultValue: number) => Promise; @@ -171,7 +172,10 @@ export class RealtimeClient { const whereClauses: string[] = [`"runtimeEnvironmentId"='${environment.id}'`]; if (params.tags) { - whereClauses.push(`"runTags" @> ARRAY[${params.tags.map((t) => `'${t}'`).join(",")}]`); + // Reject unsafe chars and escape single quotes so tag values can't + // break out of the Electric SQL string literal. + const safeTags = sanitizeRealtimeTagsForSql(params.tags); + whereClauses.push(`"runTags" @> ARRAY[${safeTags.map((t) => `'${t}'`).join(",")}]`); } const createdAtFilter = await this.#calculateCreatedAtFilter(url, params.createdAt); diff --git a/apps/webapp/app/services/routeBuilders/apiBuilder.server.ts b/apps/webapp/app/services/routeBuilders/apiBuilder.server.ts index a74d49d2290..94c2929f5f7 100644 --- a/apps/webapp/app/services/routeBuilders/apiBuilder.server.ts +++ b/apps/webapp/app/services/routeBuilders/apiBuilder.server.ts @@ -1537,6 +1537,7 @@ type WorkerLoaderHandlerFunction< ? z.infer : undefined; runnerId?: string; + environmentId?: string; }) => Promise; export function createLoaderWorkerApiRoute< @@ -1601,6 +1602,8 @@ export function createLoaderWorkerApiRoute< } const runnerId = request.headers.get(WORKER_HEADERS.RUNNER_ID) ?? undefined; + // `|| undefined` so a blank header can't become a zero-match snapshot filter (→ false reject). + const environmentId = request.headers.get(WORKER_HEADERS.ENVIRONMENT_ID) || undefined; const result = await handler({ params: parsedParams, @@ -1609,6 +1612,7 @@ export function createLoaderWorkerApiRoute< request, headers: parsedHeaders, runnerId, + environmentId, }); return result; } catch (error) { @@ -1660,6 +1664,7 @@ type WorkerActionHandlerFunction< ? z.infer : undefined; runnerId?: string; + environmentId?: string; }) => Promise; export function createActionWorkerApiRoute< @@ -1758,6 +1763,8 @@ export function createActionWorkerApiRoute< } const runnerId = request.headers.get(WORKER_HEADERS.RUNNER_ID) ?? undefined; + // `|| undefined` so a blank header can't become a zero-match snapshot filter (→ false reject). + const environmentId = request.headers.get(WORKER_HEADERS.ENVIRONMENT_ID) || undefined; const result = await handler({ params: parsedParams, @@ -1767,6 +1774,7 @@ export function createActionWorkerApiRoute< body: parsedBody, headers: parsedHeaders, runnerId, + environmentId, }); return result; } catch (error) { diff --git a/apps/webapp/app/v3/electricShape.server.ts b/apps/webapp/app/v3/electricShape.server.ts new file mode 100644 index 00000000000..c430cc0273c --- /dev/null +++ b/apps/webapp/app/v3/electricShape.server.ts @@ -0,0 +1,69 @@ +import { z } from "zod"; + +/** + * OTel trace IDs are 32 lowercase hex chars. The traceparent parser only + * checks the dash-delimited format, so crafted ids can be persisted and later + * interpolated into shape `where` clauses. Validate here to close the SQLi vector. + */ +export const OtelTraceIdSchema = z + .string() + .regex(/^[0-9a-f]{32}$/, "traceId must be 32 lowercase hex characters"); + +/** Params the sync routes set themselves; stripped from incoming requests. */ +export const RESERVED_ELECTRIC_SHAPE_PARAMS = new Set(["where", "table", "columns"]); + +const CUID_LIKE = /^[a-z][a-z0-9_]*$/i; + +/** + * Tenant column a trace shape is scoped by. TaskEvent scopes by non-null + * organizationId; TaskRun scopes by non-null projectId (its organizationId is + * nullable). The column is from this fixed union, never user input, so it's + * safe to interpolate. + */ +export type TraceScope = + | { column: "organizationId"; id: string } + | { column: "projectId"; id: string }; + +/** + * Build the Electric Shape `where` clause for the trace sync routes. Both ids + * are re-validated as defense-in-depth so a missed call site can't bypass scope. + */ +export function buildElectricTraceWhereClause(args: { + traceId: string; + scope: TraceScope; +}): string { + const { traceId, scope } = args; + if (!OtelTraceIdSchema.safeParse(traceId).success) { + throw new Error("buildElectricTraceWhereClause: unsafe traceId"); + } + if (!CUID_LIKE.test(scope.id)) { + throw new Error("buildElectricTraceWhereClause: unsafe scope id"); + } + return `"traceId"='${traceId}' AND "${scope.column}"='${scope.id}'`; +} + +/** + * Characters rejected in realtime tag values — the single source of truth + * shared by the apiBuilder Zod refine (`realtime.v1.runs.ts`) and the runtime + * sanitiser. Rejects control chars/DEL, backslash, and double-quote. Single + * quotes are allowed and escaped (`'` → `''`) in `sanitizeRealtimeTagForSql`. + */ +export const UNSAFE_REALTIME_TAG_CHARS = /[\x00-\x1f\x7f\\"]/; + +/** + * Sanitise a tag value for interpolation into an Electric Shape `where` clause: + * reject unsafe chars, escape single quotes per SQL standard. + */ +export function sanitizeRealtimeTagForSql(tag: string): string { + if (typeof tag !== "string" || tag.length === 0) { + throw new Error("Invalid realtime tag: empty"); + } + if (UNSAFE_REALTIME_TAG_CHARS.test(tag)) { + throw new Error(`Invalid realtime tag: ${JSON.stringify(tag)} — contains unsafe character`); + } + return tag.replace(/'/g, "''"); +} + +export function sanitizeRealtimeTagsForSql(tags: string[]): string[] { + return tags.map(sanitizeRealtimeTagForSql); +} diff --git a/apps/webapp/app/v3/environmentVariables/environmentVariablesRepository.server.ts b/apps/webapp/app/v3/environmentVariables/environmentVariablesRepository.server.ts index a8f2ce701ec..1a0b8247347 100644 --- a/apps/webapp/app/v3/environmentVariables/environmentVariablesRepository.server.ts +++ b/apps/webapp/app/v3/environmentVariables/environmentVariablesRepository.server.ts @@ -82,7 +82,10 @@ export class EnvironmentVariablesRepository implements Repository { return { success: false as const, error: "Project not found" }; } - if (options.environmentIds.every((v) => !project.environments.some((e) => e.id === v))) { + // Reject if ANY supplied environmentId is outside the caller's project. + // `.some` (not `.every`) so one in-project id can't let a mixed array + // through. + if (options.environmentIds.some((v) => !project.environments.some((e) => e.id === v))) { return { success: false as const, error: `Environment not found` }; } @@ -291,7 +294,9 @@ export class EnvironmentVariablesRepository implements Repository { return { success: false as const, error: "Project not found" }; } - if (options.values.every((v) => !project.environments.some((e) => e.id === v.environmentId))) { + // Same guard as `create()`: reject if ANY supplied environmentId is + // outside the caller's project (`.some`, not `.every`). + if (options.values.some((v) => !project.environments.some((e) => e.id === v.environmentId))) { return { success: false as const, error: `Environment not found` }; } diff --git a/apps/webapp/app/v3/otlpTransform.server.ts b/apps/webapp/app/v3/otlpTransform.server.ts index d66f1a8b821..4de989b0a45 100644 --- a/apps/webapp/app/v3/otlpTransform.server.ts +++ b/apps/webapp/app/v3/otlpTransform.server.ts @@ -12,6 +12,7 @@ import type { import { SeverityNumber, Span_SpanKind, Status_StatusCode } from "@trigger.dev/otlp-importer"; import type { MetricsV1Input } from "@internal/clickhouse"; import { generateSpanId } from "./eventRepository/common.server"; +import { unwrapWorkerIdInMetadata } from "./workerIdUnwrap.server"; import type { CreatableEventKind, CreatableEventStatus, @@ -468,7 +469,11 @@ function resolveDataPointContext( function extractEventProperties(attributes: KeyValue[], prefix?: string) { return { - metadata: convertSelectedKeyValueItemsToMap(attributes, [SemanticInternalAttributes.METADATA]), + // Decode a deployment-token worker.id back to its friendlyId so the credential never lands in + // stored span/log metadata (the metrics path unwraps its own worker_id separately). + metadata: unwrapWorkerIdInMetadata( + convertSelectedKeyValueItemsToMap(attributes, [SemanticInternalAttributes.METADATA]) + ), environmentId: extractStringAttribute(attributes, [ prefix, SemanticInternalAttributes.ENVIRONMENT_ID, diff --git a/apps/webapp/app/v3/services/checkSchedule.server.ts b/apps/webapp/app/v3/services/checkSchedule.server.ts index e6429a2adc1..beb5de4b596 100644 --- a/apps/webapp/app/v3/services/checkSchedule.server.ts +++ b/apps/webapp/app/v3/services/checkSchedule.server.ts @@ -1,6 +1,7 @@ import { ZodError } from "zod"; import { CronPattern } from "../schedules"; import { BaseService, ServiceValidationError } from "./baseService.server"; +import { resolveProjectScopedEnvironments } from "./resolveProjectScopedEnvironments"; import { getLimit } from "~/services/platform.v3.server"; import { getTimezones } from "~/utils/timezones.server"; import { env } from "~/env.server"; @@ -82,7 +83,19 @@ export class CheckScheduleService extends BaseService { throw new ServiceValidationError("Project not found"); } - const environments = project.environments.filter((env) => environmentIds.includes(env.id)); + // Reject (don't silently drop) any environmentId that doesn't belong to the + // authorized project. + const scopedEnvironments = resolveProjectScopedEnvironments( + environmentIds, + project.environments + ); + if (scopedEnvironments.kind === "foreign") { + throw new ServiceValidationError( + `Environment ${scopedEnvironments.foreignEnvironmentId} does not belong to this project.` + ); + } + + const environments = scopedEnvironments.environments; if (environments.some((env) => env.archivedAt)) { throw new ServiceValidationError("Can't add or edit a schedule for an archived branch"); } diff --git a/apps/webapp/app/v3/services/createDeploymentBackgroundWorkerV4.server.ts b/apps/webapp/app/v3/services/createDeploymentBackgroundWorkerV4.server.ts index a53dee42353..66a17bb9f65 100644 --- a/apps/webapp/app/v3/services/createDeploymentBackgroundWorkerV4.server.ts +++ b/apps/webapp/app/v3/services/createDeploymentBackgroundWorkerV4.server.ts @@ -51,6 +51,7 @@ export class CreateDeploymentBackgroundWorkerServiceV4 extends BaseService { const deployment = await this._prisma.workerDeployment.findFirst({ where: { friendlyId: deploymentId, + environmentId: environment.id, }, }); diff --git a/apps/webapp/app/v3/services/deleteTaskSchedule.server.ts b/apps/webapp/app/v3/services/deleteTaskSchedule.server.ts index a696fd1ffa9..601f7470af5 100644 --- a/apps/webapp/app/v3/services/deleteTaskSchedule.server.ts +++ b/apps/webapp/app/v3/services/deleteTaskSchedule.server.ts @@ -1,3 +1,4 @@ +import { scheduleWhereClause } from "~/models/schedules.server"; import { BaseService } from "./baseService.server"; type Options = { @@ -28,9 +29,7 @@ export class DeleteTaskScheduleService extends BaseService { try { const schedule = await this._prisma.taskSchedule.findFirst({ - where: { - friendlyId, - }, + where: scheduleWhereClause(projectId, friendlyId), }); if (!schedule) { diff --git a/apps/webapp/app/v3/services/deployment.server.ts b/apps/webapp/app/v3/services/deployment.server.ts index 71292f4093b..f726bba3d6d 100644 --- a/apps/webapp/app/v3/services/deployment.server.ts +++ b/apps/webapp/app/v3/services/deployment.server.ts @@ -169,7 +169,7 @@ export class DeploymentService extends BaseService { }) ); - return this.getDeployment(authenticatedEnv.projectId, friendlyId) + return this.getDeployment(authenticatedEnv.id, friendlyId) .andThen(validateDeployment) .andThen((deployment) => { if (deployment.status === "PENDING") { @@ -191,7 +191,7 @@ export class DeploymentService extends BaseService { * @param data Cancelation reason. */ public cancelDeployment( - authenticatedEnv: Pick, + authenticatedEnv: Pick, friendlyId: string, data?: Partial> ) { @@ -246,7 +246,7 @@ export class DeploymentService extends BaseService { cause: error, })); - return this.getDeployment(authenticatedEnv.projectId, friendlyId) + return this.getDeployment(authenticatedEnv.id, friendlyId) .andThen(validateDeployment) .andThen(cancelDeployment) .andThen(({ deployment }) => @@ -278,7 +278,7 @@ export class DeploymentService extends BaseService { * @param friendlyId The friendly deployment ID. */ public generateRegistryCredentials( - authenticatedEnv: Pick, + authenticatedEnv: Pick, friendlyId: string ) { const validateDeployment = ( @@ -326,7 +326,7 @@ export class DeploymentService extends BaseService { }); }); - return this.getDeployment(authenticatedEnv.projectId, friendlyId) + return this.getDeployment(authenticatedEnv.id, friendlyId) .andThen(validateDeployment) .andThen(getDeploymentRegion) .andThen(generateCredentials); @@ -472,12 +472,12 @@ export class DeploymentService extends BaseService { ); } - private getDeployment(projectId: string, friendlyId: string) { + private getDeployment(environmentId: string, friendlyId: string) { return fromPromise( this._prisma.workerDeployment.findFirst({ where: { friendlyId, - projectId, + environmentId, }, select: { status: true, diff --git a/apps/webapp/app/v3/services/initializeDeployment.server.ts b/apps/webapp/app/v3/services/initializeDeployment.server.ts index 1eef2e12bcd..abb99082dd6 100644 --- a/apps/webapp/app/v3/services/initializeDeployment.server.ts +++ b/apps/webapp/app/v3/services/initializeDeployment.server.ts @@ -286,7 +286,7 @@ export class InitializeDeploymentService extends BaseService { }); return deploymentService - .cancelDeployment(environment, deployment.friendlyId, { + .cancelDeployment({ id: environment.id }, deployment.friendlyId, { canceledReason: "Failed to enqueue build, please try again shortly.", }) .orTee((cancelError) => diff --git a/apps/webapp/app/v3/services/resolveProjectScopedEnvironments.ts b/apps/webapp/app/v3/services/resolveProjectScopedEnvironments.ts new file mode 100644 index 00000000000..631eefac718 --- /dev/null +++ b/apps/webapp/app/v3/services/resolveProjectScopedEnvironments.ts @@ -0,0 +1,27 @@ +// Resolve a caller-supplied list of environment ids against the environments +// that belong to the authorized project. Any id not in the project is reported +// as `foreign` so the caller can reject the request rather than silently drop +// it. Returns a discriminated result instead of throwing so it stays +// dependency-free and unit-testable. + +export type ProjectScopedEnvironmentsResult = + | { kind: "foreign"; foreignEnvironmentId: string } + | { kind: "ok"; environments: E[] }; + +export function resolveProjectScopedEnvironments( + environmentIds: string[], + projectEnvironments: ReadonlyArray +): ProjectScopedEnvironmentsResult { + const byId = new Map(projectEnvironments.map((e) => [e.id, e])); + + const foreignEnvironmentId = environmentIds.find((id) => !byId.has(id)); + // Explicit undefined check: an empty-string id is a foreign id that must be + // rejected, but it is falsy, so a truthiness test would silently drop it + // instead. + if (foreignEnvironmentId !== undefined) { + return { kind: "foreign", foreignEnvironmentId }; + } + + const environments = environmentIds.map((id) => byId.get(id)).filter((e): e is E => Boolean(e)); + return { kind: "ok", environments }; +} diff --git a/apps/webapp/app/v3/services/setActiveOnTaskSchedule.server.ts b/apps/webapp/app/v3/services/setActiveOnTaskSchedule.server.ts index 5afa96d7c1f..a78e4722e1b 100644 --- a/apps/webapp/app/v3/services/setActiveOnTaskSchedule.server.ts +++ b/apps/webapp/app/v3/services/setActiveOnTaskSchedule.server.ts @@ -1,3 +1,4 @@ +import { scheduleUniqWhereClause, scheduleWhereClause } from "~/models/schedules.server"; import { BaseService } from "./baseService.server"; type Options = { @@ -29,9 +30,7 @@ export class SetActiveOnTaskScheduleService extends BaseService { try { const schedule = await this._prisma.taskSchedule.findFirst({ - where: { - friendlyId, - }, + where: scheduleWhereClause(projectId, friendlyId), }); if (!schedule) { @@ -43,9 +42,7 @@ export class SetActiveOnTaskScheduleService extends BaseService { } await this._prisma.taskSchedule.update({ - where: { - friendlyId, - }, + where: scheduleUniqWhereClause(projectId, friendlyId), data: { active, }, diff --git a/apps/webapp/app/v3/services/worker/workerGroupTokenService.server.ts b/apps/webapp/app/v3/services/worker/workerGroupTokenService.server.ts index b1660065646..23d20e108fb 100644 --- a/apps/webapp/app/v3/services/worker/workerGroupTokenService.server.ts +++ b/apps/webapp/app/v3/services/worker/workerGroupTokenService.server.ts @@ -18,11 +18,14 @@ import { fromFriendlyId } from "@trigger.dev/core/v3/isomorphic"; import { WORKER_HEADERS, type WorkerQueueClass } from "@trigger.dev/core/v3/workers"; import type { RuntimeEnvironment, WorkerInstanceGroup } from "@trigger.dev/database"; import { Prisma, WorkerInstanceGroupType } from "@trigger.dev/database"; -import { SENSITIVE_WORKER_HEADERS, sanitizeWorkerHeaders } from "./sanitizeWorkerHeaders"; +import { json } from "@remix-run/server-runtime"; import { createHash, timingSafeEqual } from "crypto"; import { customAlphabet } from "nanoid"; +import { Counter } from "prom-client"; import { z } from "zod"; import { env } from "~/env.server"; +import { metricsRegister } from "~/metrics.server"; +import { evaluateCreatedAtGate } from "./workloadTokenAuthorization.server"; import { isWorkerQueueDequeueDisabled, recordBlockedDequeue, @@ -42,6 +45,30 @@ const authenticatedWorkerInstanceCache = singleton( createAuthenticatedWorkerInstanceCache ); +// Opt-in suppression of untokened worker actions on runs created after the cutoff. Only the +// no-header path ever reads a run row, and only when this is on - default off means feature-off is +// byte-for-byte today's behavior (no extra reads). Tenant scoping itself is header-driven (folded +// into the engine snapshot read) and needs no platform flag. +const workloadCreatedAtGateEnabled = env.WORKLOAD_CREATED_AT_GATE_ENABLED === "1"; +const workloadTokenCutoff = env.WORKLOAD_TOKEN_CUTOFF + ? new Date(env.WORKLOAD_TOKEN_CUTOFF) + : undefined; + +if (workloadCreatedAtGateEnabled && !workloadTokenCutoff) { + logger.warn( + "WORKLOAD_CREATED_AT_GATE_ENABLED is set but WORKLOAD_TOKEN_CUTOFF is missing; the created-at gate stays off until a cutoff is configured" + ); +} + +type WorkloadGateAction = "start" | "complete" | "continue" | "snapshots_since"; + +const workloadAuthGateCounter = new Counter({ + name: "workload_auth_gate_total", + help: "Deployment token authorization outcomes on worker actions", + labelNames: ["outcome", "action"] as const, + registers: [metricsRegister], +}); + function createAuthenticatedWorkerInstanceCache() { return createCache({ authenticatedWorkerInstance: new Namespace( @@ -188,6 +215,7 @@ export class WorkerGroupTokenService extends WithRunEngine { if (a.byteLength !== b.byteLength) { logger.error("[WorkerGroupTokenService] Managed secret length mismatch", { + managedWorkerSecret, headers: this.sanitizeHeaders(request), }); return; @@ -195,6 +223,7 @@ export class WorkerGroupTokenService extends WithRunEngine { if (!timingSafeEqual(a, b)) { logger.error("[WorkerGroupTokenService] Managed secret mismatch", { + managedWorkerSecret, headers: this.sanitizeHeaders(request), }); return; @@ -316,10 +345,16 @@ export class WorkerGroupTokenService extends WithRunEngine { } } - // Strip sensitive headers before logging request headers — see - // `sanitizeWorkerHeaders`. - private sanitizeHeaders(request: Request, denylist = SENSITIVE_WORKER_HEADERS) { - return sanitizeWorkerHeaders(request.headers, denylist); + private sanitizeHeaders(request: Request, skipHeaders = ["authorization"]) { + const sanitizedHeaders: Partial> = {}; + + for (const [key, value] of request.headers.entries()) { + if (!skipHeaders.includes(key.toLowerCase())) { + sanitizedHeaders[key] = value; + } + } + + return sanitizedHeaders; } } @@ -398,6 +433,55 @@ export class AuthenticatedWorkerInstance extends WithRunEngine { }); } + /** + * The no-header fallback. When the env header is present the engine scopes the snapshot read by it + * (nothing to do here). When it's absent, this optionally suppresses runs created after the cutoff - + * a run that new enough should have carried a token, so a missing one is treated as out-of-scope. + * Only runs when the gate is enabled AND a cutoff is set; that's the ONLY path that reads a run row. + */ + private async assertCreatedAtGate({ + runId, + environmentId, + action, + }: { + runId: string; + environmentId?: string; + action: WorkloadGateAction; + }): Promise { + if (environmentId) { + // Scoping is delegated to the engine snapshot read; no run-row read here. Recorded so the + // platform can see how much traffic is env-scoped as enforcement rolls out. + workloadAuthGateCounter.inc({ outcome: "env_scoped", action }); + return; + } + + if (!workloadCreatedAtGateEnabled || !workloadTokenCutoff) { + return; + } + + const run = await this._engine.runStore.findRun({ id: runId }, { select: { createdAt: true } }); + + if (!run) { + // Let the engine method surface the canonical not-found error. + return; + } + + const { allow, outcome } = evaluateCreatedAtGate({ + runCreatedAt: run.createdAt, + cutoff: workloadTokenCutoff, + }); + + workloadAuthGateCounter.inc({ outcome, action }); + + if (!allow) { + logger.warn("[workload-auth] rejecting untokened worker action created after cutoff", { + action, + runId, + }); + throw json({ error: "Run does not belong to this worker" }, { status: 403 }); + } + } + async heartbeatRun({ runFriendlyId, snapshotFriendlyId, @@ -420,22 +504,31 @@ export class AuthenticatedWorkerInstance extends WithRunEngine { snapshotFriendlyId, isWarmStart, runnerId, + environmentId, }: { runFriendlyId: string; snapshotFriendlyId: string; isWarmStart?: boolean; runnerId?: string; + environmentId?: string; }): Promise< StartRunAttemptResult & { envVars: Record; } > { + await this.assertCreatedAtGate({ + runId: fromFriendlyId(runFriendlyId), + environmentId, + action: "start", + }); + const engineResult = await this._engine.startRunAttempt({ runId: fromFriendlyId(runFriendlyId), snapshotId: fromFriendlyId(snapshotFriendlyId), isWarmStart, workerId: this.workerInstanceId, runnerId, + environmentId, }); const defaultMachinePreset = machinePresetFromName(defaultMachine); @@ -470,24 +563,42 @@ export class AuthenticatedWorkerInstance extends WithRunEngine { snapshotFriendlyId, completion, runnerId, + environmentId, }: { runFriendlyId: string; snapshotFriendlyId: string; completion: TaskRunExecutionResult; runnerId?: string; + environmentId?: string; }): Promise { + await this.assertCreatedAtGate({ + runId: fromFriendlyId(runFriendlyId), + environmentId, + action: "complete", + }); + return await this._engine.completeRunAttempt({ runId: fromFriendlyId(runFriendlyId), snapshotId: fromFriendlyId(snapshotFriendlyId), completion, workerId: this.workerInstanceId, runnerId, + environmentId, }); } - async getLatestSnapshot({ runFriendlyId }: { runFriendlyId: string }) { + async getLatestSnapshot({ + runFriendlyId, + environmentId, + }: { + runFriendlyId: string; + environmentId?: string; + }) { + // No created-at gate: the only untokened caller is an internal warm-start poll that legitimately + // has no token, so an absent header must not reject. When a header is present the engine scopes. return await this._engine.getRunExecutionData({ runId: fromFriendlyId(runFriendlyId), + environmentId, }); } @@ -515,29 +626,47 @@ export class AuthenticatedWorkerInstance extends WithRunEngine { runFriendlyId, snapshotFriendlyId, runnerId, + environmentId, }: { runFriendlyId: string; snapshotFriendlyId: string; runnerId?: string; + environmentId?: string; }) { + await this.assertCreatedAtGate({ + runId: fromFriendlyId(runFriendlyId), + environmentId, + action: "continue", + }); + return await this._engine.continueRunExecution({ runId: fromFriendlyId(runFriendlyId), snapshotId: fromFriendlyId(snapshotFriendlyId), workerId: this.workerInstanceId, runnerId, + environmentId, }); } async getSnapshotsSince({ runFriendlyId, snapshotId, + environmentId, }: { runFriendlyId: string; snapshotId: string; + environmentId?: string; }) { + await this.assertCreatedAtGate({ + runId: fromFriendlyId(runFriendlyId), + environmentId, + action: "snapshots_since", + }); + return await this._engine.getSnapshotsSince({ runId: fromFriendlyId(runFriendlyId), snapshotId: fromFriendlyId(snapshotId), + environmentId, }); } diff --git a/apps/webapp/app/v3/services/worker/workloadTokenAuthorization.server.ts b/apps/webapp/app/v3/services/worker/workloadTokenAuthorization.server.ts new file mode 100644 index 00000000000..a38097b62b2 --- /dev/null +++ b/apps/webapp/app/v3/services/worker/workloadTokenAuthorization.server.ts @@ -0,0 +1,27 @@ +/** + * The no-header created-at gate. Tenant scoping is handled by the env-scoped snapshot read in the + * engine; this only covers the fallback where no verified env header was forwarded (caller predates + * tokens, or the supervisor isn't enforcing): + * + * run created on/before cutoff -> allow (grandfather legacy untokened runs) + * run created after cutoff -> reject (a new-enough run should have carried a token) + * + * Pure and env-import-free so it stays trivially testable. + */ + +export type CreatedAtGateOutcome = "grandfathered" | "suppressed"; + +export type CreatedAtGateEvaluation = { + outcome: CreatedAtGateOutcome; + allow: boolean; +}; + +export function evaluateCreatedAtGate(params: { + runCreatedAt: Date; + cutoff: Date; +}): CreatedAtGateEvaluation { + const createdAfterCutoff = params.runCreatedAt.getTime() > params.cutoff.getTime(); + return createdAfterCutoff + ? { outcome: "suppressed", allow: false } + : { outcome: "grandfathered", allow: true }; +} diff --git a/apps/webapp/app/v3/workerIdUnwrap.server.ts b/apps/webapp/app/v3/workerIdUnwrap.server.ts new file mode 100644 index 00000000000..7300b740690 --- /dev/null +++ b/apps/webapp/app/v3/workerIdUnwrap.server.ts @@ -0,0 +1,75 @@ +// Decodes a `worker_id` that may be a deployment token back to its friendlyId, so the stored/queried +// value stays a short, stable id. Runs on the OTEL ingest hot path: base64url + JSON decode only (no +// verify), LRU-memoized. Fails safe — a non-token value passes through unchanged. + +import { LRUCache } from "lru-cache"; +import { SemanticInternalAttributes } from "@trigger.dev/core/v3"; + +// The runner flattens `worker.id` (the raw TRIGGER_DEPLOYMENT_ID) into span/log metadata under this +// key, mirroring how the runner composes it, so it can't drift. +const WORKER_ID_METADATA_KEY = `${SemanticInternalAttributes.METADATA}.${SemanticInternalAttributes.WORKER_ID}`; + +/** + * Unwrap, in place, the `worker.id` a runner stamps into span/log metadata: a deployment-token value + * becomes its friendlyId, everything else is left untouched. Keeps the credential out of stored + * telemetry and the value a stable deployment id (the metrics path unwraps its own worker_id). + */ +export function unwrapWorkerIdInMetadata< + T extends Record, +>(metadata: T | undefined): T | undefined { + if (metadata) { + const value = metadata[WORKER_ID_METADATA_KEY]; + if (typeof value === "string") { + (metadata as Record)[WORKER_ID_METADATA_KEY] = + unwrapWorkerId(value); + } + } + return metadata; +} + +// LRU, not FIFO: the active-deployment working set is unbounded, so an insertion-order memo thrashes +// once it exceeds the cap. Raw lru-cache (not @internal/cache's async wrapper) since this is sync. +const memo = new LRUCache({ max: 32_768 }); + +export function unwrapWorkerId(value: string | undefined): string | undefined { + if (!value) { + return value; + } + + // Fast path: a minted token is a JWT, whose base64url header always begins "eyJ". Anything else — + // a bare deployment friendlyId, "unmanaged", a dev id — returns immediately with no decode and no + // memo entry, so the non-token case costs nothing (and stays free once telemetry no longer carries + // tokens at all). + if (!value.startsWith("eyJ")) { + return value; + } + + const cached = memo.get(value); + if (cached !== undefined) { + return cached; + } + + const unwrapped = decodeDeploymentFriendlyId(value) ?? value; + memo.set(value, unwrapped); + + return unwrapped; +} + +function decodeDeploymentFriendlyId(value: string): string | undefined { + const parts = value.split("."); + // Not a JWT (three non-empty segments) → a legacy bare friendlyId; leave it as-is. + if (parts.length !== 3 || !parts[1]) { + return undefined; + } + + try { + const payload = JSON.parse(Buffer.from(parts[1], "base64url").toString("utf8")) as { + deployment?: unknown; + }; + return typeof payload.deployment === "string" && payload.deployment.length > 0 + ? payload.deployment + : undefined; + } catch { + return undefined; + } +} diff --git a/apps/webapp/app/v3/writableEnvironments.ts b/apps/webapp/app/v3/writableEnvironments.ts new file mode 100644 index 00000000000..af3ff12e8d9 --- /dev/null +++ b/apps/webapp/app/v3/writableEnvironments.ts @@ -0,0 +1,33 @@ +// Which environments a user may write env vars to. Shared env types +// (preview/staging/production) are writable by any project member; DEVELOPMENT +// environments are per-user and only writable by their owner. + +export type WriteCheckEnvironment = { + id: string; + type: string; + orgMember: { userId: string } | null; +}; + +const SHARED_ENV_TYPES = new Set(["PREVIEW", "STAGING", "PRODUCTION"]); + +/** + * Return the first submitted id the user may NOT write to — either it isn't one + * of the project's environments or it's a DEV env owned by someone else. + * Returns null when every submitted id is writable by `userId`. + */ +export function findUnauthorizedEnvironmentId( + projectEnvironments: ReadonlyArray, + submittedIds: ReadonlyArray, + userId: string +): string | null { + const byId = new Map(projectEnvironments.map((e) => [e.id, e])); + for (const id of submittedIds) { + const env = byId.get(id); + if (!env) return id; + const writable = + SHARED_ENV_TYPES.has(env.type) || + (env.type === "DEVELOPMENT" && env.orgMember?.userId === userId); + if (!writable) return id; + } + return null; +} diff --git a/apps/webapp/package.json b/apps/webapp/package.json index 31b2bb9f831..60495a47156 100644 --- a/apps/webapp/package.json +++ b/apps/webapp/package.json @@ -154,6 +154,9 @@ "isbot": "^3.6.5", "jose": "^5.4.0", "json-stable-stringify": "^1.3.0", + "jsonpointer": "^5.0.1", + "lodash.omit": "^4.5.0", + "lru-cache": "^11.2.4", "lucide-react": "^0.229.0", "marked": "^4.0.18", "match-sorter": "^6.3.4", diff --git a/apps/webapp/server.ts b/apps/webapp/server.ts index 3794441abbd..fb4e2035947 100644 --- a/apps/webapp/server.ts +++ b/apps/webapp/server.ts @@ -147,6 +147,7 @@ if (ENABLE_CLUSTER && cluster.isPrimary) { const wss: WebSocketServer | undefined = build.entry.module.wss; const apiRateLimiter: RateLimitMiddleware = build.entry.module.apiRateLimiter; const engineRateLimiter: RateLimitMiddleware = build.entry.module.engineRateLimiter; + const otlpRateLimiter: RequestHandler = build.entry.module.otlpRateLimiter; const runWithHttpContext: RunWithHttpContextFunction = build.entry.module.runWithHttpContext; const tenantContextMiddleware: RequestHandler = build.entry.module.tenantContextMiddleware; @@ -198,6 +199,7 @@ if (ENABLE_CLUSTER && cluster.isPrimary) { app.use(apiRateLimiter); app.use(engineRateLimiter); + app.use(otlpRateLimiter); app.use(tenantContextMiddleware); diff --git a/apps/webapp/test/authorizationCodeConsent.test.ts b/apps/webapp/test/authorizationCodeConsent.test.ts new file mode 100644 index 00000000000..85cda08da9b --- /dev/null +++ b/apps/webapp/test/authorizationCodeConsent.test.ts @@ -0,0 +1,58 @@ +import { containerTest } from "@internal/testcontainers"; +import { describe, expect, vi } from "vitest"; +import { + AUTHORIZATION_CODE_TTL_MS, + isAuthorizationCodeMintable, +} from "~/services/personalAccessToken.server"; + +vi.setConfig({ testTimeout: 30_000 }); + +function randomCode() { + return `code_${Math.random().toString(36).slice(2)}${Math.random().toString(36).slice(2)}`; +} + +// Lock the read-only + TTL properties `isAuthorizationCodeMintable` relies on: +// the loader must not bind a PAT, and expired codes must not be mintable. +describe("authorization code consent gate", () => { + containerTest( + "a fresh, unconsumed code is mintable and checking it does NOT bind a PAT", + async ({ prisma }) => { + const created = await prisma.authorizationCode.create({ data: { code: randomCode() } }); + + expect(await isAuthorizationCodeMintable(created.code, prisma)).toBe(true); + + // The check is read-only — it must not bind a Personal Access Token. + const after = await prisma.authorizationCode.findFirst({ where: { id: created.id } }); + expect(after?.personalAccessTokenId).toBeNull(); + } + ); + + containerTest("a code older than the TTL is not mintable", async ({ prisma }) => { + const created = await prisma.authorizationCode.create({ data: { code: randomCode() } }); + + await prisma.authorizationCode.update({ + where: { id: created.id }, + data: { createdAt: new Date(Date.now() - AUTHORIZATION_CODE_TTL_MS - 1_000) }, + }); + + expect(await isAuthorizationCodeMintable(created.code, prisma)).toBe(false); + }); + + containerTest( + "a code created just inside the TTL is still mintable (CLI flow not broken)", + async ({ prisma }) => { + const created = await prisma.authorizationCode.create({ data: { code: randomCode() } }); + + await prisma.authorizationCode.update({ + where: { id: created.id }, + data: { createdAt: new Date(Date.now() - (AUTHORIZATION_CODE_TTL_MS - 30_000)) }, + }); + + expect(await isAuthorizationCodeMintable(created.code, prisma)).toBe(true); + } + ); + + containerTest("an unknown code is not mintable", async ({ prisma }) => { + expect(await isAuthorizationCodeMintable(randomCode(), prisma)).toBe(false); + }); +}); diff --git a/apps/webapp/test/checkSchedule.test.ts b/apps/webapp/test/checkSchedule.test.ts new file mode 100644 index 00000000000..bd872bbcec2 --- /dev/null +++ b/apps/webapp/test/checkSchedule.test.ts @@ -0,0 +1,60 @@ +import { containerTest } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import { describe, expect, vi } from "vitest"; +import { resolveProjectScopedEnvironments } from "~/v3/services/resolveProjectScopedEnvironments"; + +vi.setConfig({ testTimeout: 60_000 }); + +// Exercises the environment-scoping primitive CheckScheduleService relies on +// (`resolveProjectScopedEnvironments`) with real RuntimeEnvironment rows, +// imported directly to avoid `~/db.server` and its eager global-prisma connect. + +async function seedProjectWithEnv(prisma: PrismaClient, slugBase: string) { + const slug = `${slugBase}_${Math.random().toString(36).slice(2, 10)}`; + const organization = await prisma.organization.create({ data: { title: slug, slug } }); + const project = await prisma.project.create({ + data: { name: slug, slug, organizationId: organization.id, externalRef: slug }, + }); + const environment = await prisma.runtimeEnvironment.create({ + data: { + slug: `${slug}-prod`, + type: "PRODUCTION", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_prod_${slug}`, + pkApiKey: `pk_prod_${slug}`, + shortcode: slug.slice(0, 6), + }, + }); + return { organization, project, environment }; +} + +function projectEnvironments(prisma: PrismaClient, projectId: string) { + return prisma.runtimeEnvironment.findMany({ where: { projectId }, select: { id: true } }); +} + +describe("resolveProjectScopedEnvironments (schedule env scoping)", () => { + containerTest("rejects an environment id that belongs to another project", async ({ prisma }) => { + const a = await seedProjectWithEnv(prisma, "orga"); + const b = await seedProjectWithEnv(prisma, "orgb"); + + const result = resolveProjectScopedEnvironments( + [a.environment.id, b.environment.id], + await projectEnvironments(prisma, a.project.id) + ); + + expect(result.kind).toBe("foreign"); + expect(result).toMatchObject({ foreignEnvironmentId: b.environment.id }); + }); + + containerTest("accepts environment ids that belong to the project", async ({ prisma }) => { + const a = await seedProjectWithEnv(prisma, "orga"); + + const result = resolveProjectScopedEnvironments( + [a.environment.id], + await projectEnvironments(prisma, a.project.id) + ); + + expect(result.kind).toBe("ok"); + }); +}); diff --git a/apps/webapp/test/deleteTaskSchedule.test.ts b/apps/webapp/test/deleteTaskSchedule.test.ts new file mode 100644 index 00000000000..320b44bf4f0 --- /dev/null +++ b/apps/webapp/test/deleteTaskSchedule.test.ts @@ -0,0 +1,75 @@ +import { containerTest } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import { describe, expect, vi } from "vitest"; +import { scheduleWhereClause } from "~/models/schedules.server"; + +vi.setConfig({ testTimeout: 60_000 }); + +// Exercises the project-scoping primitive DeleteTaskScheduleService relies on +// (`scheduleWhereClause`) directly against a real database, to avoid importing +// `~/db.server` and its eager global-prisma connect. + +async function seedProject(prisma: PrismaClient, slugBase: string) { + const slug = `${slugBase}_${Math.random().toString(36).slice(2, 10)}`; + const organization = await prisma.organization.create({ data: { title: slug, slug } }); + const project = await prisma.project.create({ + data: { name: slug, slug, organizationId: organization.id, externalRef: slug }, + }); + return { organization, project }; +} + +function seedSchedule(prisma: PrismaClient, projectId: string, friendlyId: string) { + return prisma.taskSchedule.create({ + data: { + friendlyId, + taskIdentifier: "my-task", + projectId, + generatorExpression: "0 * * * *", + generatorDescription: "every hour", + type: "IMPERATIVE", + }, + }); +} + +describe("scheduleWhereClause (delete lookup scoping)", () => { + containerTest( + "a schedule from another project is not found when scoped to the caller's project", + async ({ prisma }) => { + const a = await seedProject(prisma, "orga"); + const b = await seedProject(prisma, "orgb"); + const victim = await seedSchedule( + prisma, + b.project.id, + `sched_${Math.random().toString(36).slice(2, 10)}` + ); + + // Scoped to A's project: the cross-tenant schedule is invisible (the + // `projectId` in the where is what prevents a cross-project delete). + const fromA = await prisma.taskSchedule.findFirst({ + where: scheduleWhereClause(a.project.id, victim.friendlyId), + }); + expect(fromA).toBeNull(); + + // Scoped to its own project: found. + const fromB = await prisma.taskSchedule.findFirst({ + where: scheduleWhereClause(b.project.id, victim.friendlyId), + }); + expect(fromB?.id).toBe(victim.id); + } + ); + + containerTest("the where pins projectId for both id shapes", async ({ prisma }) => { + const a = await seedProject(prisma, "orga"); + + // friendlyId shape + expect(scheduleWhereClause(a.project.id, "sched_abc")).toMatchObject({ + friendlyId: "sched_abc", + projectId: a.project.id, + }); + // deduplicationKey shape + expect(scheduleWhereClause(a.project.id, "my-dedup-key")).toMatchObject({ + projectId: a.project.id, + deduplicationKey: "my-dedup-key", + }); + }); +}); diff --git a/apps/webapp/test/env.server.test.ts b/apps/webapp/test/env.server.test.ts new file mode 100644 index 00000000000..0fe13efbe26 --- /dev/null +++ b/apps/webapp/test/env.server.test.ts @@ -0,0 +1,84 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; + +const originalEnv = process.env; + +const requiredEnv = { + NODE_ENV: "test", + DATABASE_URL: "postgresql://test:test@localhost:5432/test", + DIRECT_URL: "postgresql://test:test@localhost:5432/test", + SESSION_SECRET: "test-session-secret", + MAGIC_LINK_SECRET: "test-magic-link-secret", + ENCRYPTION_KEY: "test-encryption-keeeeey-32-bytes", + CLICKHOUSE_URL: "http://localhost:8123", + DEPLOY_REGISTRY_HOST: "registry.example.com", + MANAGED_WORKER_SECRET: "test-managed-worker-secret", +}; + +describe("webapp environment secrets", () => { + afterEach(() => { + process.env = originalEnv; + vi.resetModules(); + }); + + it.each(["SESSION_SECRET", "MAGIC_LINK_SECRET", "ENCRYPTION_KEY", "MANAGED_WORKER_SECRET"])( + "requires %s to be explicitly set", + async (key) => { + process.env = { ...requiredEnv }; + delete process.env[key]; + + await expect(import("../app/env.server")).rejects.toThrow(key); + } + ); + + it.each(["SESSION_SECRET", "MAGIC_LINK_SECRET", "ENCRYPTION_KEY", "MANAGED_WORKER_SECRET"])( + "rejects an empty %s", + async (key) => { + process.env = { ...requiredEnv, [key]: "" }; + + await expect(import("../app/env.server")).rejects.toThrow(key); + } + ); + + it.each([ + ["SESSION_SECRET", "2818143646516f6fffd707b36f334bbb"], + ["MAGIC_LINK_SECRET", "44da78b7bbb0dfe709cf38931d25dcdd"], + ["ENCRYPTION_KEY", "f686147ab967943ebbe9ed3b496e465a"], + ["MANAGED_WORKER_SECRET", "managed-secret"], + ["MANAGED_WORKER_SECRET", "447c29678f9eaf289e9c4b70d3dd8a7f"], + ])("rejects the known-insecure default value for %s", async (key, insecureValue) => { + process.env = { ...requiredEnv, [key]: insecureValue }; + + await expect(import("../app/env.server")).rejects.toThrow(key); + }); + + it("accepts explicitly configured secrets", async () => { + process.env = { ...requiredEnv }; + + const { env } = await import("../app/env.server"); + + expect(env.SESSION_SECRET).toBe(requiredEnv.SESSION_SECRET); + expect(env.MAGIC_LINK_SECRET).toBe(requiredEnv.MAGIC_LINK_SECRET); + expect(env.ENCRYPTION_KEY).toBe(requiredEnv.ENCRYPTION_KEY); + expect(env.MANAGED_WORKER_SECRET).toBe(requiredEnv.MANAGED_WORKER_SECRET); + }); + + it("allows a known-insecure default when ALLOW_INSECURE_DEFAULT_SECRETS is set", async () => { + process.env = { + ...requiredEnv, + ALLOW_INSECURE_DEFAULT_SECRETS: "1", + ENCRYPTION_KEY: "f686147ab967943ebbe9ed3b496e465a", + MANAGED_WORKER_SECRET: "managed-secret", + }; + + const { env } = await import("../app/env.server"); + + expect(env.ENCRYPTION_KEY).toBe("f686147ab967943ebbe9ed3b496e465a"); + expect(env.MANAGED_WORKER_SECRET).toBe("managed-secret"); + }); + + it("still rejects an empty secret even with ALLOW_INSECURE_DEFAULT_SECRETS", async () => { + process.env = { ...requiredEnv, ALLOW_INSECURE_DEFAULT_SECRETS: "1", SESSION_SECRET: "" }; + + await expect(import("../app/env.server")).rejects.toThrow("SESSION_SECRET"); + }); +}); diff --git a/apps/webapp/test/environmentVariablesRepository.test.ts b/apps/webapp/test/environmentVariablesRepository.test.ts index 4025d8fd2eb..b5661c89cbe 100644 --- a/apps/webapp/test/environmentVariablesRepository.test.ts +++ b/apps/webapp/test/environmentVariablesRepository.test.ts @@ -180,4 +180,80 @@ describe("EnvironmentVariablesRepository.getVariableValuesForKeys", () => { expect(crossProjectRequest.size).toBe(0); }); + + postgresTest( + "create() rejects a mix of in-project and foreign environmentIds without writing foreign values", + async ({ prisma }) => { + const { + user, + organization, + project: projectA, + } = await createTestOrgProjectWithMember(prisma); + + const projectB = await prisma.project.create({ + data: { + name: "Project B", + slug: `proj-b-${Date.now()}`, + organizationId: organization.id, + externalRef: `ext-b-${Date.now()}`, + }, + }); + + const envA = await createRuntimeEnvironment(prisma, { + projectId: projectA.id, + organizationId: organization.id, + type: "PRODUCTION", + }); + const envB = await createRuntimeEnvironment(prisma, { + projectId: projectB.id, + organizationId: organization.id, + type: "PRODUCTION", + }); + + const repository = new EnvironmentVariablesRepository(prisma, prisma); + + // Caller scoped to projectA supplies a mixed array: an in-project env + // (envA) plus a foreign one (envB). The whole request must be refused. + const result = await repository.create(projectA.id, { + override: true, + environmentIds: [envA.id, envB.id], + variables: [{ key: "CROSS_TENANT", value: "x" }], + isSecret: false, + lastUpdatedBy: { type: "user", userId: user.id }, + }); + + expect(result.success).toBe(false); + + // No value row may have been written against the foreign environment. + const foreignValues = await prisma.environmentVariableValue.findMany({ + where: { environmentId: envB.id }, + }); + expect(foreignValues).toHaveLength(0); + } + ); + + postgresTest( + "create() still succeeds for an all-in-project environmentIds array", + async ({ prisma }) => { + const { user, organization, project } = await createTestOrgProjectWithMember(prisma); + + const environment = await createRuntimeEnvironment(prisma, { + projectId: project.id, + organizationId: organization.id, + type: "PRODUCTION", + }); + + const repository = new EnvironmentVariablesRepository(prisma, prisma); + + const result = await repository.create(project.id, { + override: true, + environmentIds: [environment.id], + variables: [{ key: "OK_KEY", value: "v" }], + isSecret: false, + lastUpdatedBy: { type: "user", userId: user.id }, + }); + + expect(result.success).toBe(true); + } + ); }); diff --git a/apps/webapp/test/registryConfig.test.ts b/apps/webapp/test/registryConfig.test.ts index 341761d0115..b9a8c966123 100644 --- a/apps/webapp/test/registryConfig.test.ts +++ b/apps/webapp/test/registryConfig.test.ts @@ -10,6 +10,9 @@ describe("getRegistryConfig", () => { MAGIC_LINK_SECRET: "test-magic-link-secret", ENCRYPTION_KEY: "test-encryption-keeeeey-32-bytes", CLICKHOUSE_URL: "http://localhost:8123", + PROVIDER_SECRET: "test-provider-secret", + COORDINATOR_SECRET: "test-coordinator-secret", + MANAGED_WORKER_SECRET: "test-managed-worker-secret", }; beforeEach(() => { diff --git a/apps/webapp/test/resolveProjectScopedEnvironments.test.ts b/apps/webapp/test/resolveProjectScopedEnvironments.test.ts new file mode 100644 index 00000000000..f04a623d676 --- /dev/null +++ b/apps/webapp/test/resolveProjectScopedEnvironments.test.ts @@ -0,0 +1,44 @@ +import { describe, expect, it } from "vitest"; +import { resolveProjectScopedEnvironments } from "../app/v3/services/resolveProjectScopedEnvironments.js"; + +const projectEnvs = [{ id: "env_prod" }, { id: "env_staging" }, { id: "env_dev" }]; + +// A submitted environment id that doesn't belong to the project must be +// rejected, not silently dropped. +describe("resolveProjectScopedEnvironments", () => { + it("resolves ids that all belong to the project", () => { + const r = resolveProjectScopedEnvironments(["env_prod", "env_dev"], projectEnvs); + expect(r.kind).toBe("ok"); + if (r.kind === "ok") expect(r.environments.map((e) => e.id)).toEqual(["env_prod", "env_dev"]); + }); + + it("rejects a foreign environment id (the cross-tenant vector)", () => { + const r = resolveProjectScopedEnvironments(["env_someone_elses"], projectEnvs); + expect(r.kind).toBe("foreign"); + if (r.kind === "foreign") expect(r.foreignEnvironmentId).toBe("env_someone_elses"); + }); + + it("rejects when a foreign id is mixed in with valid ones (not silently dropped)", () => { + const r = resolveProjectScopedEnvironments(["env_prod", "env_foreign"], projectEnvs); + expect(r.kind).toBe("foreign"); + if (r.kind === "foreign") expect(r.foreignEnvironmentId).toBe("env_foreign"); + }); + + it("returns an empty set for no ids", () => { + const r = resolveProjectScopedEnvironments([], projectEnvs); + expect(r.kind).toBe("ok"); + if (r.kind === "ok") expect(r.environments).toEqual([]); + }); + + it("rejects an empty-string id rather than dropping it (falsy edge case)", () => { + const r = resolveProjectScopedEnvironments([""], projectEnvs); + expect(r.kind).toBe("foreign"); + if (r.kind === "foreign") expect(r.foreignEnvironmentId).toBe(""); + }); + + it("rejects an empty-string id mixed in with valid ids", () => { + const r = resolveProjectScopedEnvironments(["env_prod", ""], projectEnvs); + expect(r.kind).toBe("foreign"); + if (r.kind === "foreign") expect(r.foreignEnvironmentId).toBe(""); + }); +}); diff --git a/apps/webapp/test/schedulesPutEnvScoping.test.ts b/apps/webapp/test/schedulesPutEnvScoping.test.ts new file mode 100644 index 00000000000..4bc937f6088 --- /dev/null +++ b/apps/webapp/test/schedulesPutEnvScoping.test.ts @@ -0,0 +1,261 @@ +import { containerTest } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import { describe, expect, vi } from "vitest"; +import { getScheduleEnvVisibility } from "~/models/schedules.server"; + +vi.setConfig({ testTimeout: 60_000 }); + +async function seedProjectWithEnvs(prisma: PrismaClient, slugBase: string) { + const slug = `${slugBase}_${Math.random().toString(36).slice(2, 10)}`; + const organization = await prisma.organization.create({ + data: { title: slug, slug }, + }); + const project = await prisma.project.create({ + data: { + name: slug, + slug, + organizationId: organization.id, + externalRef: slug, + }, + }); + const prodEnv = await prisma.runtimeEnvironment.create({ + data: { + slug: "prod", + type: "PRODUCTION", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_prod_${slug}`, + pkApiKey: `pk_prod_${slug}`, + shortcode: `p${slug.slice(0, 4)}`, + }, + }); + const stagingEnv = await prisma.runtimeEnvironment.create({ + data: { + slug: "staging", + type: "STAGING", + projectId: project.id, + organizationId: organization.id, + apiKey: `tr_staging_${slug}`, + pkApiKey: `pk_staging_${slug}`, + shortcode: `s${slug.slice(0, 4)}`, + }, + }); + return { organization, project, prodEnv, stagingEnv }; +} + +async function seedScheduleWithInstance( + prisma: PrismaClient, + projectId: string, + environmentId: string, + opts: { friendlyId?: string; deduplicationKey?: string } = {} +) { + const schedule = await prisma.taskSchedule.create({ + data: { + friendlyId: opts.friendlyId ?? `sched_${Math.random().toString(36).slice(2, 10)}`, + taskIdentifier: "my-task", + projectId, + generatorExpression: "0 * * * *", + generatorDescription: "every hour", + type: "IMPERATIVE", + ...(opts.deduplicationKey + ? { deduplicationKey: opts.deduplicationKey, userProvidedDeduplicationKey: true } + : {}), + }, + }); + await prisma.taskScheduleInstance.create({ + data: { + taskScheduleId: schedule.id, + environmentId, + projectId, + }, + }); + return schedule; +} + +describe("getScheduleEnvVisibility", () => { + containerTest( + "returns 'hidden' when an instance lives in a different env (by friendlyId)", + async ({ prisma }) => { + const env = await seedProjectWithEnvs(prisma, "orga"); + const schedule = await seedScheduleWithInstance(prisma, env.project.id, env.prodEnv.id); + + const visibility = await getScheduleEnvVisibility( + prisma, + env.project.id, + schedule.friendlyId, + env.stagingEnv.id + ); + expect(visibility.status).toBe("hidden"); + } + ); + + containerTest("returns 'visible' when every instance is in caller env", async ({ prisma }) => { + const env = await seedProjectWithEnvs(prisma, "orga"); + const schedule = await seedScheduleWithInstance(prisma, env.project.id, env.prodEnv.id); + + const visibility = await getScheduleEnvVisibility( + prisma, + env.project.id, + schedule.friendlyId, + env.prodEnv.id + ); + expect(visibility.status).toBe("visible"); + if (visibility.status === "visible") { + expect(visibility.schedule.id).toBe(schedule.id); + } + }); + + containerTest("returns 'missing' when no schedule exists", async ({ prisma }) => { + const env = await seedProjectWithEnvs(prisma, "orga"); + + const visibility = await getScheduleEnvVisibility( + prisma, + env.project.id, + "sched_does_not_exist", + env.prodEnv.id + ); + expect(visibility.status).toBe("missing"); + }); + + containerTest("returns 'visible' when no instances exist yet", async ({ prisma }) => { + const env = await seedProjectWithEnvs(prisma, "orga"); + const schedule = await prisma.taskSchedule.create({ + data: { + friendlyId: `sched_${Math.random().toString(36).slice(2, 10)}`, + taskIdentifier: "my-task", + projectId: env.project.id, + generatorExpression: "0 * * * *", + generatorDescription: "every hour", + type: "IMPERATIVE", + }, + }); + + const visibility = await getScheduleEnvVisibility( + prisma, + env.project.id, + schedule.friendlyId, + env.prodEnv.id + ); + expect(visibility.status).toBe("visible"); + }); + + containerTest( + "returns 'visible' from every environment a multi-env schedule spans", + async ({ prisma }) => { + const env = await seedProjectWithEnvs(prisma, "orga"); + const schedule = await prisma.taskSchedule.create({ + data: { + friendlyId: `sched_${Math.random().toString(36).slice(2, 10)}`, + taskIdentifier: "my-task", + projectId: env.project.id, + generatorExpression: "0 * * * *", + generatorDescription: "every hour", + type: "IMPERATIVE", + }, + }); + await prisma.taskScheduleInstance.createMany({ + data: [ + { taskScheduleId: schedule.id, environmentId: env.prodEnv.id, projectId: env.project.id }, + { + taskScheduleId: schedule.id, + environmentId: env.stagingEnv.id, + projectId: env.project.id, + }, + ], + }); + + // The schedule list surfaces a schedule for any environment it has an + // instance in, so per-schedule reads/mutations must resolve the same + // way. A multi-env schedule is visible from each environment it spans. + const fromProd = await getScheduleEnvVisibility( + prisma, + env.project.id, + schedule.friendlyId, + env.prodEnv.id + ); + expect(fromProd.status).toBe("visible"); + + const fromStaging = await getScheduleEnvVisibility( + prisma, + env.project.id, + schedule.friendlyId, + env.stagingEnv.id + ); + expect(fromStaging.status).toBe("visible"); + } + ); + + containerTest( + "returns 'hidden' from an environment the schedule has no instance in", + async ({ prisma }) => { + const env = await seedProjectWithEnvs(prisma, "orga"); + // A third environment with no instance of the schedule. + const devEnv = await prisma.runtimeEnvironment.create({ + data: { + slug: "dev", + type: "DEVELOPMENT", + projectId: env.project.id, + organizationId: env.organization.id, + apiKey: `tr_dev_${env.project.slug}`, + pkApiKey: `pk_dev_${env.project.slug}`, + shortcode: `d${env.project.slug.slice(0, 4)}`, + }, + }); + const schedule = await prisma.taskSchedule.create({ + data: { + friendlyId: `sched_${Math.random().toString(36).slice(2, 10)}`, + taskIdentifier: "my-task", + projectId: env.project.id, + generatorExpression: "0 * * * *", + generatorDescription: "every hour", + type: "IMPERATIVE", + }, + }); + await prisma.taskScheduleInstance.createMany({ + data: [ + { taskScheduleId: schedule.id, environmentId: env.prodEnv.id, projectId: env.project.id }, + { + taskScheduleId: schedule.id, + environmentId: env.stagingEnv.id, + projectId: env.project.id, + }, + ], + }); + + const fromDev = await getScheduleEnvVisibility( + prisma, + env.project.id, + schedule.friendlyId, + devEnv.id + ); + expect(fromDev.status).toBe("hidden"); + } + ); + + containerTest( + "resolves by user-provided deduplicationKey (the non-sched_ prefix branch)", + async ({ prisma }) => { + const env = await seedProjectWithEnvs(prisma, "orga"); + const dedupKey = `my-daily-cleanup-${Math.random().toString(36).slice(2, 8)}`; + await seedScheduleWithInstance(prisma, env.project.id, env.prodEnv.id, { + deduplicationKey: dedupKey, + }); + + const hiddenFromStaging = await getScheduleEnvVisibility( + prisma, + env.project.id, + dedupKey, + env.stagingEnv.id + ); + expect(hiddenFromStaging.status).toBe("hidden"); + + const visibleFromProd = await getScheduleEnvVisibility( + prisma, + env.project.id, + dedupKey, + env.prodEnv.id + ); + expect(visibleFromProd.status).toBe("visible"); + } + ); +}); diff --git a/apps/webapp/test/setActiveOnTaskSchedule.test.ts b/apps/webapp/test/setActiveOnTaskSchedule.test.ts new file mode 100644 index 00000000000..6c3b6e9ab2b --- /dev/null +++ b/apps/webapp/test/setActiveOnTaskSchedule.test.ts @@ -0,0 +1,149 @@ +import { containerTest } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import { describe, expect, vi } from "vitest"; +import { + getScheduleEnvVisibility, + scheduleUniqWhereClause, + scheduleWhereClause, +} from "~/models/schedules.server"; + +vi.setConfig({ testTimeout: 60_000 }); + +// Exercises the project-scoping primitives SetActiveOnTaskScheduleService relies +// on (`scheduleWhereClause` + `scheduleUniqWhereClause`) directly against a real +// database, to avoid importing `~/db.server` and its eager global-prisma connect. + +async function seedProject(prisma: PrismaClient, slugBase: string) { + const slug = `${slugBase}_${Math.random().toString(36).slice(2, 10)}`; + const organization = await prisma.organization.create({ data: { title: slug, slug } }); + const project = await prisma.project.create({ + data: { name: slug, slug, organizationId: organization.id, externalRef: slug }, + }); + return { organization, project }; +} + +function seedSchedule( + prisma: PrismaClient, + projectId: string, + friendlyId: string, + active: boolean = true +) { + return prisma.taskSchedule.create({ + data: { + friendlyId, + taskIdentifier: "my-task", + projectId, + generatorExpression: "0 * * * *", + generatorDescription: "every hour", + type: "IMPERATIVE", + active, + }, + }); +} + +describe("schedule scoping (enable/disable)", () => { + containerTest( + "toggling scoped to another project does not touch the victim schedule", + async ({ prisma }) => { + const a = await seedProject(prisma, "orga"); + const b = await seedProject(prisma, "orgb"); + const victim = await seedSchedule( + prisma, + b.project.id, + `sched_${Math.random().toString(36).slice(2, 10)}`, + true + ); + + // Project A cannot toggle B's schedule: the where pins projectId, so the + // update matches zero rows. + const result = await prisma.taskSchedule.updateMany({ + where: scheduleWhereClause(a.project.id, victim.friendlyId), + data: { active: false }, + }); + expect(result.count).toBe(0); + + const unchanged = await prisma.taskSchedule.findUnique({ where: { id: victim.id } }); + expect(unchanged?.active).toBe(true); + + // The owning project can toggle it. + const owned = await prisma.taskSchedule.updateMany({ + where: scheduleWhereClause(b.project.id, victim.friendlyId), + data: { active: false }, + }); + expect(owned.count).toBe(1); + } + ); + + containerTest("the unique-where pins projectId", async ({ prisma }) => { + const a = await seedProject(prisma, "orga"); + expect(scheduleUniqWhereClause(a.project.id, "sched_abc")).toMatchObject({ + friendlyId: "sched_abc", + projectId: a.project.id, + }); + }); + + // The public activate/deactivate endpoints gate on getScheduleEnvVisibility + // before toggling `active`. A key scoped to one environment must not be able + // to enable/disable a schedule that only runs in another environment of the + // same project. + containerTest( + "an env-scoped caller cannot toggle a schedule that only runs in another env", + async ({ prisma }) => { + const a = await seedProject(prisma, "orga"); + const prodEnv = await prisma.runtimeEnvironment.create({ + data: { + slug: "prod", + type: "PRODUCTION", + projectId: a.project.id, + organizationId: a.organization.id, + apiKey: `tr_prod_${a.project.slug}`, + pkApiKey: `pk_prod_${a.project.slug}`, + shortcode: `p${a.project.slug.slice(0, 4)}`, + }, + }); + const stagingEnv = await prisma.runtimeEnvironment.create({ + data: { + slug: "staging", + type: "STAGING", + projectId: a.project.id, + organizationId: a.organization.id, + apiKey: `tr_staging_${a.project.slug}`, + pkApiKey: `pk_staging_${a.project.slug}`, + shortcode: `s${a.project.slug.slice(0, 4)}`, + }, + }); + const schedule = await seedSchedule( + prisma, + a.project.id, + `sched_${Math.random().toString(36).slice(2, 10)}`, + true + ); + // The schedule only runs in staging. + await prisma.taskScheduleInstance.create({ + data: { + taskScheduleId: schedule.id, + environmentId: stagingEnv.id, + projectId: a.project.id, + }, + }); + + // A prod-scoped key is refused (the route returns 404 and never updates). + const fromProd = await getScheduleEnvVisibility( + prisma, + a.project.id, + schedule.friendlyId, + prodEnv.id + ); + expect(fromProd.status).toBe("hidden"); + + // The staging-scoped key that owns an instance can toggle it. + const fromStaging = await getScheduleEnvVisibility( + prisma, + a.project.id, + schedule.friendlyId, + stagingEnv.id + ); + expect(fromStaging.status).toBe("visible"); + } + ); +}); diff --git a/apps/webapp/test/setup.ts b/apps/webapp/test/setup.ts index db07cb1402a..521a9624b7d 100644 --- a/apps/webapp/test/setup.ts +++ b/apps/webapp/test/setup.ts @@ -13,6 +13,9 @@ config({ path: path.resolve(__dirname, "../.env") }); // the pair — the ioredis mock below forces lazyConnect, so nothing ever dials. process.env.REDIS_HOST ??= "localhost"; process.env.REDIS_PORT ??= "6379"; +process.env.PROVIDER_SECRET ??= "test-provider-secret"; +process.env.COORDINATOR_SECRET ??= "test-coordinator-secret"; +process.env.MANAGED_WORKER_SECRET ??= "test-managed-worker-secret"; // Worker singletons construct a RedisWorker at import time whose ioredis client // connects eagerly, so any test importing the service graph opens real Redis diff --git a/apps/webapp/test/spanTraceRoutes.replicaLag.test.ts b/apps/webapp/test/spanTraceRoutes.replicaLag.test.ts index f3df2d2ac7d..6b8dac27932 100644 --- a/apps/webapp/test/spanTraceRoutes.replicaLag.test.ts +++ b/apps/webapp/test/spanTraceRoutes.replicaLag.test.ts @@ -475,7 +475,7 @@ describe("run-trace/span-detail route loaders under a lagging replica", () => { const seed = await seedTenant(prisma14, suffix); const runId = `run_${CUID_25}`; const friendlyId = `run_${suffix}`; - const traceId = `trace_${suffix}`; + const traceId = "a".repeat(32); const userId = `user_${suffix}`; // The dashboard user, joined to the org so the route's real orgMember check passes. @@ -538,7 +538,8 @@ describe("run-trace/span-detail route loaders under a lagging replica", () => { holder.resolvedEnv = { organizationId: seed.organization.id }; holder.replicaMarker = { orgMember: prisma14.orgMember }; - const res = (await syncTraceRunsLoader(syncRequest("trace_does_not_exist"))) as Response; + const res = (await syncTraceRunsLoader(syncRequest("b".repeat(32)))) as Response; + expect(lagged.legacyReplica.wasHit("taskRun")).toBe(true); expect(res.status).toBe(404); } ); diff --git a/apps/webapp/test/workerIdUnwrap.test.ts b/apps/webapp/test/workerIdUnwrap.test.ts new file mode 100644 index 00000000000..ea787a81108 --- /dev/null +++ b/apps/webapp/test/workerIdUnwrap.test.ts @@ -0,0 +1,71 @@ +import { describe, expect, it } from "vitest"; +import { mintWorkloadDeploymentToken, SemanticInternalAttributes } from "@trigger.dev/core/v3"; +import { unwrapWorkerId, unwrapWorkerIdInMetadata } from "~/v3/workerIdUnwrap.server"; + +const EXP = Math.floor(Date.UTC(2032, 0, 1) / 1000); +const WORKER_ID_KEY = `${SemanticInternalAttributes.METADATA}.${SemanticInternalAttributes.WORKER_ID}`; + +function mint(deployment: string) { + return mintWorkloadDeploymentToken( + { + deployment, + deployment_version: "20260709.1", + environment_id: "env_1", + environment_type: "PRODUCTION", + org_id: "org_1", + project_id: "proj_1", + }, + "any-secret", + EXP + ); +} + +describe("unwrapWorkerId", () => { + it("unwraps a real minted token to its deployment friendlyId", async () => { + // Decode is signature-independent (display-only), so the secret is irrelevant here. + expect(unwrapWorkerId(await mint("deployment_abc123"))).toBe("deployment_abc123"); + }); + + it("passes a legacy bare friendlyId through unchanged", () => { + expect(unwrapWorkerId("deployment_legacy")).toBe("deployment_legacy"); + }); + + it("passes undefined and garbage through unchanged (fail-safe)", () => { + expect(unwrapWorkerId(undefined)).toBeUndefined(); + expect(unwrapWorkerId("")).toBe(""); + expect(unwrapWorkerId("a.b.c")).toBe("a.b.c"); + expect(unwrapWorkerId("not-a-token")).toBe("not-a-token"); + }); + + it("is stable across repeated calls (memoized)", async () => { + const token = await mint("deployment_memo"); + expect(unwrapWorkerId(token)).toBe("deployment_memo"); + expect(unwrapWorkerId(token)).toBe("deployment_memo"); + }); +}); + +describe("unwrapWorkerIdInMetadata", () => { + it("unwraps a token worker.id in the metadata bag, leaving other keys untouched", async () => { + const metadata = { + [WORKER_ID_KEY]: await mint("deployment_span"), + "$metadata.custom": "keep-me", + "$metadata.worker.version": "20260709.1", + }; + + const result = unwrapWorkerIdInMetadata(metadata); + + expect(result?.[WORKER_ID_KEY]).toBe("deployment_span"); + expect(result?.["$metadata.custom"]).toBe("keep-me"); + expect(result?.["$metadata.worker.version"]).toBe("20260709.1"); + }); + + it("leaves a legacy bare friendlyId worker.id unchanged", () => { + const metadata = { [WORKER_ID_KEY]: "deployment_legacy" }; + expect(unwrapWorkerIdInMetadata(metadata)?.[WORKER_ID_KEY]).toBe("deployment_legacy"); + }); + + it("handles absent worker.id and undefined metadata", () => { + expect(unwrapWorkerIdInMetadata(undefined)).toBeUndefined(); + expect(unwrapWorkerIdInMetadata({ "$metadata.other": "x" })?.["$metadata.other"]).toBe("x"); + }); +}); diff --git a/apps/webapp/test/workloadTokenAuthorization.test.ts b/apps/webapp/test/workloadTokenAuthorization.test.ts new file mode 100644 index 00000000000..205db263255 --- /dev/null +++ b/apps/webapp/test/workloadTokenAuthorization.test.ts @@ -0,0 +1,26 @@ +import { describe, expect, it } from "vitest"; +import { evaluateCreatedAtGate } from "~/v3/services/worker/workloadTokenAuthorization.server"; + +const cutoff = new Date("2026-07-09T00:00:00.000Z"); +const before = new Date("2026-07-01T00:00:00.000Z"); +const after = new Date("2026-07-10T00:00:00.000Z"); + +describe("evaluateCreatedAtGate", () => { + it("grandfathers a run created before the cutoff", () => { + const result = evaluateCreatedAtGate({ runCreatedAt: before, cutoff }); + expect(result.outcome).toBe("grandfathered"); + expect(result.allow).toBe(true); + }); + + it("suppresses a run created after the cutoff", () => { + const result = evaluateCreatedAtGate({ runCreatedAt: after, cutoff }); + expect(result.outcome).toBe("suppressed"); + expect(result.allow).toBe(false); + }); + + it("treats a run created exactly at the cutoff as grandfathered (not after)", () => { + const result = evaluateCreatedAtGate({ runCreatedAt: cutoff, cutoff }); + expect(result.outcome).toBe("grandfathered"); + expect(result.allow).toBe(true); + }); +}); diff --git a/apps/webapp/test/workloadTokenGate.integration.test.ts b/apps/webapp/test/workloadTokenGate.integration.test.ts new file mode 100644 index 00000000000..ea6458b5760 --- /dev/null +++ b/apps/webapp/test/workloadTokenGate.integration.test.ts @@ -0,0 +1,88 @@ +import { postgresTest } from "@internal/testcontainers"; +import type { PrismaClient } from "@trigger.dev/database"; +import { describe, expect } from "vitest"; + +// The env-scoped snapshot read the platform now performs for worker actions. Mirrors the where clause +// in getLatestExecutionSnapshot so this exercises the real select against the schema. +async function readLatestSnapshot(prisma: PrismaClient, runId: string, environmentId?: string) { + return prisma.taskRunExecutionSnapshot.findFirst({ + where: { runId, isValid: true, ...(environmentId ? { environmentId } : {}) }, + orderBy: { createdAt: "desc" }, + }); +} + +async function seed(prisma: PrismaClient) { + const org = await prisma.organization.create({ + data: { title: "Org", slug: `org-${Date.now()}` }, + }); + const project = await prisma.project.create({ + data: { + name: "Project", + slug: `proj-${Date.now()}`, + externalRef: `proj_${Date.now()}`, + organizationId: org.id, + }, + }); + const env = await prisma.runtimeEnvironment.create({ + data: { + type: "PRODUCTION", + slug: "prod", + projectId: project.id, + organizationId: org.id, + apiKey: "api_key", + pkApiKey: "pk_api_key", + shortcode: "short", + }, + }); + + const run = await prisma.taskRun.create({ + data: { + friendlyId: `run_${Date.now()}`, + taskIdentifier: "test-task", + payload: "{}", + payloadType: "application/json", + traceId: "trace_1", + spanId: "span_1", + queue: "task/test-task", + runtimeEnvironmentId: env.id, + projectId: project.id, + organizationId: org.id, + }, + }); + + await prisma.taskRunExecutionSnapshot.create({ + data: { + engine: "V2", + executionStatus: "RUN_CREATED", + description: "seed", + runId: run.id, + runStatus: "PENDING", + environmentId: env.id, + environmentType: "PRODUCTION", + projectId: project.id, + organizationId: org.id, + }, + }); + + return { env, run }; +} + +describe("env-scoped snapshot read against a real DB row", () => { + postgresTest( + "returns the snapshot for the matching env and nothing for another", + async ({ prisma }) => { + const { env, run } = await seed(prisma as PrismaClient); + + // No env scoping -> found (internal callers) + expect(await readLatestSnapshot(prisma as PrismaClient, run.id)).not.toBeNull(); + + // Matching env -> found + expect(await readLatestSnapshot(prisma as PrismaClient, run.id, env.id)).not.toBeNull(); + + // Different env -> not found (rejected as a tenant boundary) + expect( + await readLatestSnapshot(prisma as PrismaClient, run.id, "clenvdoesnotexist000000000") + ).toBeNull(); + } + ); +}); diff --git a/apps/webapp/test/writableEnvironments.test.ts b/apps/webapp/test/writableEnvironments.test.ts new file mode 100644 index 00000000000..ed26fcde905 --- /dev/null +++ b/apps/webapp/test/writableEnvironments.test.ts @@ -0,0 +1,55 @@ +import { describe, expect, it } from "vitest"; +import { + findUnauthorizedEnvironmentId, + type WriteCheckEnvironment, +} from "../app/v3/writableEnvironments.js"; + +const prod: WriteCheckEnvironment = { id: "env_prod", type: "PRODUCTION", orgMember: null }; +const staging: WriteCheckEnvironment = { id: "env_staging", type: "STAGING", orgMember: null }; +const myDev: WriteCheckEnvironment = { + id: "env_dev_me", + type: "DEVELOPMENT", + orgMember: { userId: "user_me" }, +}; +const otherDev: WriteCheckEnvironment = { + id: "env_dev_other", + type: "DEVELOPMENT", + orgMember: { userId: "user_other" }, +}; +const projectEnvs = [prod, staging, myDev, otherDev]; + +// Shared env types are writable by any project member; a DEVELOPMENT env only by +// its owner; an id not in the project is never writable. +describe("findUnauthorizedEnvironmentId", () => { + it("allows shared env types for any member", () => { + expect( + findUnauthorizedEnvironmentId(projectEnvs, ["env_prod", "env_staging"], "user_me") + ).toBeNull(); + }); + + it("allows a caller's own DEV env", () => { + expect(findUnauthorizedEnvironmentId(projectEnvs, ["env_dev_me"], "user_me")).toBeNull(); + }); + + it("rejects another user's DEV env (the cross-user injection vector)", () => { + expect(findUnauthorizedEnvironmentId(projectEnvs, ["env_dev_other"], "user_me")).toBe( + "env_dev_other" + ); + }); + + it("rejects a foreign DEV env even when mixed with allowed ones", () => { + expect( + findUnauthorizedEnvironmentId( + projectEnvs, + ["env_prod", "env_dev_me", "env_dev_other"], + "user_me" + ) + ).toBe("env_dev_other"); + }); + + it("rejects an id that isn't one of the project's environments", () => { + expect(findUnauthorizedEnvironmentId(projectEnvs, ["env_not_in_project"], "user_me")).toBe( + "env_not_in_project" + ); + }); +}); diff --git a/docs/self-hosting/docker.mdx b/docs/self-hosting/docker.mdx index ead4281a6df..6c8de292f17 100644 --- a/docs/self-hosting/docker.mdx +++ b/docs/self-hosting/docker.mdx @@ -75,12 +75,23 @@ git clone --depth=1 https://github.com/triggerdotdev/trigger.dev cd trigger.dev/hosting/docker ``` -2. Create a `.env` file +2. Create a `.env` file and generate secrets ```bash cp .env.example .env + +# Fills the required secrets in .env with strong, unique values. +# Safe to re-run - it never overwrites a secret you've already set. +./generate-secrets.sh ``` + + The stack ships no working default credentials. `generate-secrets.sh` fills the + application secrets and the bundled datastore passwords with strong, unique values. + Keep them safe - rotating the encryption key or session secret later will invalidate + existing sessions and encrypted data. + + 3. Start the webapp ```bash @@ -130,6 +141,13 @@ docker compose up -d 4. Configure the supervisor using the [environment variables](/self-hosting/env/supervisor) in your `.env` file, including the [worker token](#worker-token). + + For a split webapp/worker setup, set `MANAGED_WORKER_SECRET` on the worker to + the **same** value as the webapp's `MANAGED_WORKER_SECRET`. Don't run + `generate-secrets.sh` on the worker host - it would create a mismatched value + and the worker would fail to authenticate. + + 5. Apply the changes: ```bash diff --git a/docs/self-hosting/env/webapp.mdx b/docs/self-hosting/env/webapp.mdx index 1f629718b52..09416f0f38f 100644 --- a/docs/self-hosting/env/webapp.mdx +++ b/docs/self-hosting/env/webapp.mdx @@ -11,7 +11,8 @@ mode: "wide" | `SESSION_SECRET` | Yes | — | Session encryption secret. Run: `openssl rand -hex 16` | | `MAGIC_LINK_SECRET` | Yes | — | Magic link encryption secret. Run: `openssl rand -hex 16` | | `ENCRYPTION_KEY` | Yes | — | Secret store encryption key. Run: `openssl rand -hex 16` | -| `MANAGED_WORKER_SECRET` | No | managed-secret | Managed worker secret. Should be changed and match supervisor. | +| `MANAGED_WORKER_SECRET` | Yes | — | Managed worker secret. Must be set and match supervisor. Run: `openssl rand -hex 32` | +| `ALLOW_INSECURE_DEFAULT_SECRETS` | No | false | Boot even if a secret is still a known-insecure published default. Temporary escape hatch for values you can't safely rotate yet (see [Secret generation and rotation](/self-hosting/kubernetes#secret-generation-and-rotation)). | | **Domains & ports** | | | | | `REMIX_APP_PORT` | No | 3030 | Remix app port. | | `APP_ORIGIN` | Yes | http://localhost:3030 | App origin URL. | diff --git a/docs/self-hosting/kubernetes.mdx b/docs/self-hosting/kubernetes.mdx index 5e24b675f4a..33d06f3086c 100644 --- a/docs/self-hosting/kubernetes.mdx +++ b/docs/self-hosting/kubernetes.mdx @@ -121,8 +121,10 @@ The default values are insecure and are only suitable for testing. You will need Create a `values-custom.yaml` file to override the defaults. For example: ```yaml -# Generate new secrets with `openssl rand -hex 16` -# WARNING: You should probably use an existingSecret instead +# Leave these unset to have the chart auto-generate strong values on first +# install (retained across upgrades). Set them explicitly only if you need to +# control the value - e.g. sharing MANAGED_WORKER_SECRET with an external +# supervisor - or use an existingSecret. secrets: enabled: true sessionSecret: "your-32-char-hex-secret-1" @@ -133,6 +135,8 @@ secrets: # - SESSION_SECRET # - MAGIC_LINK_SECRET # - ENCRYPTION_KEY +# - PROVIDER_SECRET +# - COORDINATOR_SECRET # - MANAGED_WORKER_SECRET # - OBJECT_STORE_ACCESS_KEY_ID # - OBJECT_STORE_SECRET_ACCESS_KEY @@ -176,6 +180,24 @@ helm upgrade -n trigger --install trigger \ -f values-custom.yaml ``` +### Secret generation and rotation + +Application, control-plane, and bundled-datastore secrets left unset are generated on +first install and **retained across `helm upgrade`** - they are never rotated +automatically, so sessions, encrypted data, and datastore volumes survive upgrades. + + + GitOps tools that render with `helm template` (e.g. Argo CD) cannot read the existing + secret, so they regenerate these values on every sync - which rotates them. If you + deploy via GitOps, always supply your own `secrets.existingSecret` (and datastore + credentials) so nothing is generated in-cluster. + + +There is no clean migration for a compromised `ENCRYPTION_KEY`: changing it makes +existing encrypted data unreadable. If a deployment is still running a previously +published default and cannot rotate yet, set `ALLOW_INSECURE_DEFAULT_SECRETS=true` on +the webapp to keep booting while you plan a migration. + ### Extra env You can set extra environment variables on all services. For example: diff --git a/hosting/docker/.env.example b/hosting/docker/.env.example index 383b6f79dfe..4c7cf11bc70 100644 --- a/hosting/docker/.env.example +++ b/hosting/docker/.env.example @@ -3,13 +3,16 @@ # - You should change them to suit your needs, especially the secrets # - See the docs for more information: https://trigger.dev/docs/self-hosting/overview -# Secrets -# - Do NOT use these defaults in production -# - Generate your own by running `openssl rand -hex 16` for each secret -SESSION_SECRET=2818143646516f6fffd707b36f334bbb -MAGIC_LINK_SECRET=44da78b7bbb0dfe709cf38931d25dcdd -ENCRYPTION_KEY=f686147ab967943ebbe9ed3b496e465a -MANAGED_WORKER_SECRET=447c29678f9eaf289e9c4b70d3dd8a7f +# Secrets — REQUIRED, no defaults. The stack will not boot until each is set to a unique value. +# Generate each with: openssl rand -hex 16 +SESSION_SECRET= +MAGIC_LINK_SECRET= +ENCRYPTION_KEY= +# These authenticate the internal control-plane connections. Generate each with: openssl rand -hex 16 +# COORDINATOR_SECRET must match the coordinator's PLATFORM_SECRET; MANAGED_WORKER_SECRET the supervisor's. +PROVIDER_SECRET= +COORDINATOR_SECRET= +MANAGED_WORKER_SECRET= # Worker token # - This is the token for the worker to connect to the webapp @@ -24,13 +27,14 @@ MANAGED_WORKER_SECRET=447c29678f9eaf289e9c4b70d3dd8a7f # OTEL_EXPORTER_OTLP_ENDPOINT=https://trigger.example.com/otel # Postgres -# - Do NOT use these defaults in production -# - Especially if you decide to expose the database to the internet +# - Password is REQUIRED, no default. Run ./generate-secrets.sh to fill it (or openssl rand -hex 16). +# - DATABASE_URL / DIRECT_URL are derived from POSTGRES_PASSWORD automatically - only set them +# below to point at an external Postgres (POSTGRES_PASSWORD is then unused). # POSTGRES_USER=postgres -POSTGRES_PASSWORD=unsafe-postgres-pw +POSTGRES_PASSWORD= # POSTGRES_DB=postgres -DATABASE_URL=postgresql://postgres:unsafe-postgres-pw@postgres:5432/main?schema=public&sslmode=disable -DIRECT_URL=postgresql://postgres:unsafe-postgres-pw@postgres:5432/main?schema=public&sslmode=disable +# DATABASE_URL=postgresql://user:password@host:5432/main?schema=public&sslmode=disable +# DIRECT_URL=postgresql://user:password@host:5432/main?schema=public&sslmode=disable # Trigger image tag # - This is the version of the webapp and worker images to use, they should be locked to a specific version in production @@ -59,19 +63,21 @@ DEV_OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:8030/otel # NODE_MAX_OLD_SPACE_SIZE=8192 # ClickHouse -# - Do NOT use these defaults in production +# - Password is REQUIRED, no default. Run ./generate-secrets.sh to fill it. +# - CLICKHOUSE_URL / RUN_REPLICATION_CLICKHOUSE_URL are derived from CLICKHOUSE_PASSWORD +# automatically - only set them below to point at an external ClickHouse. CLICKHOUSE_USER=default -CLICKHOUSE_PASSWORD=password -CLICKHOUSE_URL=http://default:password@clickhouse:8123?secure=false -RUN_REPLICATION_CLICKHOUSE_URL=http://default:password@clickhouse:8123 +CLICKHOUSE_PASSWORD= +# CLICKHOUSE_URL=http://user:password@host:8123?secure=false +# RUN_REPLICATION_CLICKHOUSE_URL=http://user:password@host:8123 # Docker Registry -# - When testing locally, the default values should be fine -# - When deploying to production, you will have to change these, especially the password and URL +# - Password is REQUIRED, no default. Run ./generate-secrets.sh to fill it - it also writes +# the matching registry/auth.htpasswd (bcrypt) the bundled registry authenticates against. # - See the docs for more information: https://trigger.dev/docs/self-hosting/docker#registry-setup DOCKER_REGISTRY_URL=localhost:5000 DOCKER_REGISTRY_USERNAME=registry-user -DOCKER_REGISTRY_PASSWORD=very-secure-indeed +DOCKER_REGISTRY_PASSWORD= # When using an external registry you will have to change this # On Docker Hub it should generally be the same as your username DOCKER_REGISTRY_NAMESPACE=trigger @@ -80,8 +86,10 @@ DOCKER_REGISTRY_NAMESPACE=trigger # - You need to log into the Minio dashboard and create a bucket called "packets" # - See the docs for more information: https://trigger.dev/docs/self-hosting/docker#object-storage # Default provider (backward compatible - no protocol prefix) +# - Secret access key is REQUIRED, no default. Run ./generate-secrets.sh to fill it. +# - For the bundled MinIO, these ARE its root credentials (MINIO_ROOT_USER/PASSWORD derive from them). OBJECT_STORE_ACCESS_KEY_ID=admin -OBJECT_STORE_SECRET_ACCESS_KEY=very-safe-password +OBJECT_STORE_SECRET_ACCESS_KEY= # You will have to uncomment and configure this for production # OBJECT_STORE_BASE_URL=http://localhost:9000 # OBJECT_STORE_REGION=auto @@ -100,11 +108,11 @@ OBJECT_STORE_SECRET_ACCESS_KEY=very-safe-password # OBJECT_STORE_R2_SECRET_ACCESS_KEY= # OBJECT_STORE_R2_REGION=auto # OBJECT_STORE_R2_SERVICE=s3 -# Credentials to access the Minio dashboard at http://localhost:9001 -# - You should change these credentials and not use them for the `OBJECT_STORE_` env vars above -# - Instead, setup a non-root user with access the "packets" bucket +# Minio dashboard at http://localhost:9001 +# - The bundled Minio's root credentials default to OBJECT_STORE_ACCESS_KEY_ID / OBJECT_STORE_SECRET_ACCESS_KEY. +# - For production, set a separate root user here and create a non-root user scoped to the "packets" bucket for OBJECT_STORE_*. # MINIO_ROOT_USER=admin -# MINIO_ROOT_PASSWORD=very-safe-password +# MINIO_ROOT_PASSWORD= # Realtime streams # - Realtime streams power AI-agent token streaming and run streams diff --git a/hosting/docker/generate-secrets.sh b/hosting/docker/generate-secrets.sh new file mode 100755 index 00000000000..679dc129cfb --- /dev/null +++ b/hosting/docker/generate-secrets.sh @@ -0,0 +1,110 @@ +#!/bin/sh +# Generate strong, unique secrets and datastore passwords into the self-hosting .env. +# +# Safe to re-run: only fills values that are missing or empty, and never overwrites +# one that is already set. Rotating a live secret would orphan encrypted data, log +# everyone out, or break an already-initialised datastore volume - so rotation is +# opt-in only, via --force (which WILL break existing data/sessions). +set -eu + +FORCE=0 +for arg in "$@"; do + case "$arg" in + -f | --force) FORCE=1 ;; + -h | --help) + echo "Usage: $0 [--force] [env-file]" + echo " --force Regenerate every secret, overwriting existing values." + echo " WARNING: rotates live secrets - breaks encrypted data, sessions," + echo " and already-initialised datastore volumes." + exit 0 + ;; + *) env_file="$arg" ;; + esac +done + +script_dir=$(CDPATH= cd -- "$(dirname -- "$0")" && pwd) +env_file="${env_file:-$script_dir/.env}" +env_example="$script_dir/.env.example" +htpasswd_file="$script_dir/registry/auth.htpasswd" + +# openssl rand -hex 16 -> 32 hex chars: URL-safe (no @ : / etc. to break connection +# strings) and satisfies the webapp's exact-32-byte ENCRYPTION_KEY check. +gen() { openssl rand -hex 16; } + +if [ ! -f "$env_file" ]; then + cp "$env_example" "$env_file" + echo "Created $(basename "$env_file") from $(basename "$env_example")" +fi + +sed_inplace() { + if [ "$(uname)" = "Darwin" ]; then + sed -i '' "$@" + else + sed -i "$@" + fi +} + +# Value of KEY= in the env file, trailing inline comment/whitespace stripped, so +# "KEY=" and "KEY= # todo" both read as unset. +current_value() { + grep -E "^$1=" "$env_file" | head -n1 | cut -d= -f2- \ + | sed -e 's/[[:space:]]*#.*$//' -e 's/[[:space:]]*$//' +} + +set_var() { + key="$1" + value="$2" + if grep -qE "^$key=" "$env_file"; then + sed_inplace -e "s|^$key=.*|$key=$value|" "$env_file" + else + printf '%s=%s\n' "$key" "$value" >>"$env_file" + fi +} + +# Fill KEY with a fresh secret unless it already has a value (respecting --force). +# Returns 0 if it wrote, 1 if it skipped. +fill() { + key="$1" + if [ "$FORCE" -eq 0 ] && [ -n "$(current_value "$key")" ]; then + return 1 + fi + set_var "$key" "$(gen)" + echo "Generated $key" + return 0 +} + +generated=0 + +# App + control-plane secrets, and the bundled-datastore passwords. All plain +# values consumed directly (or, for datastores, woven into the connection URLs by +# docker-compose interpolation - see .env.example). +for key in \ + SESSION_SECRET MAGIC_LINK_SECRET ENCRYPTION_KEY \ + PROVIDER_SECRET COORDINATOR_SECRET MANAGED_WORKER_SECRET \ + POSTGRES_PASSWORD CLICKHOUSE_PASSWORD OBJECT_STORE_SECRET_ACCESS_KEY; do + if fill "$key"; then generated=$((generated + 1)); fi +done + +# Registry is special: the bundled registry authenticates against a bcrypt htpasswd +# file, so when we set its password we must regenerate that file to match. bcrypt is +# the only hash the registry accepts, and openssl can't produce it - use httpd's +# htpasswd via docker (already required for this stack). +if [ "$FORCE" -eq 1 ] || [ -z "$(current_value DOCKER_REGISTRY_PASSWORD)" ]; then + registry_user=$(current_value DOCKER_REGISTRY_USERNAME) + registry_user=${registry_user:-registry-user} + registry_pass=$(gen) + if ! command -v docker >/dev/null 2>&1; then + echo "ERROR: docker is required to hash the registry password (bcrypt). Install docker and re-run." >&2 + exit 1 + fi + docker run --rm httpd:2 htpasswd -Bbn "$registry_user" "$registry_pass" >"$htpasswd_file" + set_var DOCKER_REGISTRY_PASSWORD "$registry_pass" + echo "Generated DOCKER_REGISTRY_PASSWORD (and wrote $(basename "$htpasswd_file"))" + generated=$((generated + 1)) +fi + +if [ "$generated" -eq 0 ]; then + echo "All secrets already set in $(basename "$env_file"); nothing to do. Use --force to rotate." +else + echo "Wrote $generated secret(s) to $(basename "$env_file")." +fi diff --git a/hosting/docker/registry/auth.htpasswd b/hosting/docker/registry/auth.htpasswd index 3f659261e48..e69de29bb2d 100644 --- a/hosting/docker/registry/auth.htpasswd +++ b/hosting/docker/registry/auth.htpasswd @@ -1 +0,0 @@ -registry-user:$2y$05$6ingYqw0.3j13dxHY4w3neMSvKhF3pvRmc0AFifScWsVA9JpuLwNK diff --git a/hosting/docker/webapp/docker-compose.yml b/hosting/docker/webapp/docker-compose.yml index 611ce4939e3..8d133cbeea5 100644 --- a/hosting/docker/webapp/docker-compose.yml +++ b/hosting/docker/webapp/docker-compose.yml @@ -54,11 +54,13 @@ services: REALTIME_STREAMS_S2_ENDPOINT: ${REALTIME_STREAMS_S2_ENDPOINT:-http://s2/v1} REALTIME_STREAMS_S2_SKIP_ACCESS_TOKENS: ${REALTIME_STREAMS_S2_SKIP_ACCESS_TOKENS:-true} REALTIME_STREAMS_S2_ACCESS_TOKEN: ${REALTIME_STREAMS_S2_ACCESS_TOKEN:-} - DATABASE_URL: ${DATABASE_URL:-postgresql://postgres:postgres@postgres:5432/main?schema=public&sslmode=disable} - DIRECT_URL: ${DIRECT_URL:-postgresql://postgres:postgres@postgres:5432/main?schema=public&sslmode=disable} + DATABASE_URL: ${DATABASE_URL:-postgresql://postgres:${POSTGRES_PASSWORD}@postgres:5432/main?schema=public&sslmode=disable} + DIRECT_URL: ${DIRECT_URL:-postgresql://postgres:${POSTGRES_PASSWORD}@postgres:5432/main?schema=public&sslmode=disable} SESSION_SECRET: ${SESSION_SECRET} MAGIC_LINK_SECRET: ${MAGIC_LINK_SECRET} ENCRYPTION_KEY: ${ENCRYPTION_KEY} + PROVIDER_SECRET: ${PROVIDER_SECRET} + COORDINATOR_SECRET: ${COORDINATOR_SECRET} MANAGED_WORKER_SECRET: ${MANAGED_WORKER_SECRET} REDIS_HOST: redis REDIS_PORT: 6379 @@ -78,11 +80,11 @@ services: TRIGGER_BOOTSTRAP_WORKER_GROUP_NAME: bootstrap TRIGGER_BOOTSTRAP_WORKER_TOKEN_PATH: /home/node/shared/worker_token # ClickHouse configuration - CLICKHOUSE_URL: ${CLICKHOUSE_URL:-http://default:password@clickhouse:8123?secure=false} + CLICKHOUSE_URL: ${CLICKHOUSE_URL:-http://default:${CLICKHOUSE_PASSWORD}@clickhouse:8123?secure=false} CLICKHOUSE_LOG_LEVEL: ${CLICKHOUSE_LOG_LEVEL:-info} # Run replication RUN_REPLICATION_ENABLED: ${RUN_REPLICATION_ENABLED:-1} - RUN_REPLICATION_CLICKHOUSE_URL: ${RUN_REPLICATION_CLICKHOUSE_URL:-http://default:password@clickhouse:8123} + RUN_REPLICATION_CLICKHOUSE_URL: ${RUN_REPLICATION_CLICKHOUSE_URL:-http://default:${CLICKHOUSE_PASSWORD}@clickhouse:8123} RUN_REPLICATION_LOG_LEVEL: ${RUN_REPLICATION_LOG_LEVEL:-info} # Limits # TASK_PAYLOAD_OFFLOAD_THRESHOLD: 524288 # 512KB @@ -109,7 +111,7 @@ services: - wal_level=logical environment: POSTGRES_USER: ${POSTGRES_USER:-postgres} - POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-postgres} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in .env - run ./generate-secrets.sh} POSTGRES_DB: ${POSTGRES_DB:-postgres} healthcheck: test: ["CMD", "pg_isready", "-U", "postgres"] @@ -144,7 +146,7 @@ services: networks: - webapp environment: - DATABASE_URL: ${DATABASE_URL:-postgresql://postgres:postgres@postgres:5432/main?schema=public&sslmode=disable} + DATABASE_URL: ${DATABASE_URL:-postgresql://postgres:${POSTGRES_PASSWORD}@postgres:5432/main?schema=public&sslmode=disable} ELECTRIC_INSECURE: true ELECTRIC_USAGE_REPORTING: false healthcheck: @@ -163,7 +165,7 @@ services: - ${CLICKHOUSE_PUBLISH_IP:-127.0.0.1}:9090:9000 environment: CLICKHOUSE_ADMIN_USER: ${CLICKHOUSE_USER:-default} - CLICKHOUSE_ADMIN_PASSWORD: ${CLICKHOUSE_PASSWORD:-password} + CLICKHOUSE_ADMIN_PASSWORD: ${CLICKHOUSE_PASSWORD:?Set CLICKHOUSE_PASSWORD in .env - run ./generate-secrets.sh} volumes: - clickhouse:/bitnami/clickhouse - ../clickhouse/override.xml:/bitnami/clickhouse/etc/config.d/override.xml:ro @@ -181,7 +183,7 @@ services: "--user", "${CLICKHOUSE_USER:-default}", "--password", - "${CLICKHOUSE_PASSWORD:-password}", + "${CLICKHOUSE_PASSWORD}", "--query", "SELECT 1", ] @@ -224,8 +226,8 @@ services: volumes: - minio:/bitnami/minio/data environment: - MINIO_ROOT_USER: ${MINIO_ROOT_USER:-admin} - MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:-very-safe-password} + MINIO_ROOT_USER: ${MINIO_ROOT_USER:-${OBJECT_STORE_ACCESS_KEY_ID:-admin}} + MINIO_ROOT_PASSWORD: ${MINIO_ROOT_PASSWORD:-${OBJECT_STORE_SECRET_ACCESS_KEY:?Set OBJECT_STORE_SECRET_ACCESS_KEY in .env - run ./generate-secrets.sh}} MINIO_DEFAULT_BUCKETS: packets MINIO_BROWSER: "on" healthcheck: diff --git a/hosting/k8s/helm/README.md b/hosting/k8s/helm/README.md index 33e64a93a23..79c49138215 100644 --- a/hosting/k8s/helm/README.md +++ b/hosting/k8s/helm/README.md @@ -20,7 +20,10 @@ helm template trigger . --dependency-update ### Installation ```bash -# Deploy with default values (testing/development only) +# A bare install works: application, control-plane, and bundled-datastore +# secrets are auto-generated on first install (and retained across upgrades) +# when you don't set them. Provide a values file to pin any of them or to +# point at external datastores. helm install trigger . # Deploy to specific namespace @@ -59,24 +62,28 @@ npx trigger.dev@latest deploy --push ### Secrets Configuration -**IMPORTANT**: The default secrets are for **TESTING ONLY** and must be changed for production. +**IMPORTANT**: The chart ships no working secret defaults. Application, control-plane, and bundled-datastore secrets (postgres/clickhouse/minio/registry) are **auto-generated per install** (and retained across upgrades) when left unset. You can still set them explicitly - e.g. to share `managedWorkerSecret` with an external supervisor, or to manage everything via `secrets.existingSecret`. Explicit values always win over generation. -#### Required Secrets +#### Auto-generated secrets -All secrets must be exactly **32 hexadecimal characters** (16 bytes): +Left unset, these are generated on first install and preserved on `helm upgrade` (rotating them would invalidate sessions and orphan encrypted data, so they are never regenerated once set): - `sessionSecret` - User authentication sessions -- `magicLinkSecret` - Passwordless login tokens +- `magicLinkSecret` - Passwordless login tokens - `encryptionKey` - Sensitive data encryption -- `managedWorkerSecret` - Worker authentication +- `managedWorkerSecret` - Worker authentication (set explicitly if an external supervisor must share it) +- `providerSecret` - Provider control-plane socket authentication +- `coordinatorSecret` - Coordinator control-plane socket authentication -#### Generate Production Secrets +#### Generating secrets yourself + +If you prefer to set them explicitly (e.g. to reuse across clusters): ```bash -for i in {1..4}; do openssl rand -hex 16; done +for i in {1..6}; do openssl rand -hex 16; done ``` -#### Configure Production Secrets +#### Configure secrets explicitly ```yaml # values-production.yaml @@ -85,6 +92,8 @@ secrets: magicLinkSecret: "your-generated-secret-2" encryptionKey: "your-generated-secret-3" managedWorkerSecret: "your-generated-secret-4" + providerSecret: "your-generated-secret-5" + coordinatorSecret: "your-generated-secret-6" objectStore: accessKeyId: "your-s3-access-key" secretAccessKey: "your-s3-secret-key" diff --git a/hosting/k8s/helm/ci/lint-values.yaml b/hosting/k8s/helm/ci/lint-values.yaml new file mode 100644 index 00000000000..967cc6c503a --- /dev/null +++ b/hosting/k8s/helm/ci/lint-values.yaml @@ -0,0 +1,26 @@ +# Throwaway credentials used ONLY to render the chart during CI (helm lint / helm template). +# The shipped values.yaml no longer contains working default credentials for the bundled +# datastores (see SEC-70): an unconfigured install now fails closed. CI must therefore +# supply dummy secrets here so template rendering can be validated. These values are never +# deployed and must never be used in a real environment. +secrets: + sessionSecret: "ci-render-only-session" + magicLinkSecret: "ci-render-only-magic-link" + encryptionKey: "ci-render-only-encryption-key" + providerSecret: "ci-render-only-provider" + coordinatorSecret: "ci-render-only-coordinator" + managedWorkerSecret: "ci-render-only-managed-worker" +postgres: + auth: + postgresPassword: "ci-render-only-postgres" + password: "ci-render-only-postgres" +clickhouse: + auth: + password: "ci-render-only-clickhouse" +s3: + auth: + rootPassword: "ci-render-only-minio" +registry: + deploy: true + auth: + password: "ci-render-only-registry" diff --git a/hosting/k8s/helm/templates/NOTES.txt b/hosting/k8s/helm/templates/NOTES.txt index e70d18dd74a..4571eb9f474 100644 --- a/hosting/k8s/helm/templates/NOTES.txt +++ b/hosting/k8s/helm/templates/NOTES.txt @@ -2,21 +2,11 @@ Thank you for installing {{ .Chart.Name }}. Your release is named {{ .Release.Name }}. -🔐 SECURITY WARNING: -{{- if or (eq .Values.secrets.sessionSecret "2818143646516f6fffd707b36f334bbb") (eq .Values.secrets.magicLinkSecret "44da78b7bbb0dfe709cf38931d25dcdd") (eq .Values.secrets.encryptionKey "f686147ab967943ebbe9ed3b496e465a") (eq .Values.secrets.managedWorkerSecret "447c29678f9eaf289e9c4b70d3dd8a7f") }} - You are using DEFAULT SECRETS which are NOT SECURE for production! - - For production deployments, generate new secrets: - 1. Run: openssl rand -hex 16 (repeat for each secret) - 2. Override in your values.yaml: - secrets: - sessionSecret: "your-new-32-char-hex-secret" - magicLinkSecret: "your-new-32-char-hex-secret" - encryptionKey: "your-new-32-char-hex-secret" - managedWorkerSecret: "your-new-32-char-hex-secret" -{{- else }} - Custom secrets detected - good for production deployment! -{{- end }} +🔐 SECURITY: + This chart ships no working secret defaults. Application and control-plane + secrets are auto-generated per install (and retained across upgrades) unless + you set them explicitly or supply secrets.existingSecret. Bundled-datastore + passwords must still be provided. To get started: diff --git a/hosting/k8s/helm/templates/_helpers.tpl b/hosting/k8s/helm/templates/_helpers.tpl index 2ce273c0171..0ecc17b9893 100644 --- a/hosting/k8s/helm/templates/_helpers.tpl +++ b/hosting/k8s/helm/templates/_helpers.tpl @@ -141,7 +141,7 @@ PostgreSQL connection string (fallback when not using secrets) {{- if .Values.postgres.external.databaseUrl -}} {{ .Values.postgres.external.databaseUrl }} {{- else if .Values.postgres.deploy -}} -postgresql://{{ .Values.postgres.auth.username }}:{{ .Values.postgres.auth.password }}@{{ include "trigger-v4.postgres.hostname" . }}:5432/{{ .Values.postgres.auth.database }}?schema={{ .Values.postgres.connection.schema | default "public" }}&sslmode={{ .Values.postgres.connection.sslMode | default "prefer" }} +postgresql://{{ .Values.postgres.auth.username }}:$(POSTGRES_PASSWORD)@{{ include "trigger-v4.postgres.hostname" . }}:5432/{{ .Values.postgres.auth.database }}?schema={{ .Values.postgres.connection.schema | default "public" }}&sslmode={{ .Values.postgres.connection.sslMode | default "prefer" }} {{- end -}} {{- end }} @@ -439,7 +439,7 @@ hex-encoded password or percent-encode before storing in the Secret. {{- if .Values.clickhouse.deploy -}} {{- $protocol := ternary "https" "http" .Values.clickhouse.secure -}} {{- $secure := ternary "true" "false" .Values.clickhouse.secure -}} -{{ $protocol }}://{{ .Values.clickhouse.auth.username }}:{{ .Values.clickhouse.auth.password }}@{{ include "trigger-v4.clickhouse.hostname" . }}:8123?secure={{ $secure }} +{{ $protocol }}://{{ .Values.clickhouse.auth.username }}:$(CLICKHOUSE_PASSWORD)@{{ include "trigger-v4.clickhouse.hostname" . }}:8123?secure={{ $secure }} {{- else if .Values.clickhouse.external.host -}} {{- $protocol := ternary "https" "http" .Values.clickhouse.external.secure -}} {{- $secure := ternary "true" "false" .Values.clickhouse.external.secure -}} @@ -460,7 +460,7 @@ applies to the replication URL. {{- define "trigger-v4.clickhouse.replication.url" -}} {{- if .Values.clickhouse.deploy -}} {{- $protocol := ternary "https" "http" .Values.clickhouse.secure -}} -{{ $protocol }}://{{ .Values.clickhouse.auth.username }}:{{ .Values.clickhouse.auth.password }}@{{ include "trigger-v4.clickhouse.hostname" . }}:8123 +{{ $protocol }}://{{ .Values.clickhouse.auth.username }}:$(CLICKHOUSE_PASSWORD)@{{ include "trigger-v4.clickhouse.hostname" . }}:8123 {{- else if .Values.clickhouse.external.host -}} {{- $protocol := ternary "https" "http" .Values.clickhouse.external.secure -}} {{- if .Values.clickhouse.external.existingSecret -}} @@ -511,6 +511,35 @@ Get the secrets name - either existing secret or generated name {{- end -}} {{- end }} +{{/* +Fixed name of the chart-managed datastore-credentials Secret. Fixed (not release- +scoped) because bundled Bitnami subcharts read it via `*.auth.existingSecret`, which +is set in values.yaml and cannot be templated - so the name must be a literal that +both this chart and those values agree on. One release per namespace. +*/}} +{{- define "trigger-v4.datastore.secretName" -}} +trigger-datastore +{{- end }} + +{{/* +Resolve a chart-managed secret: an explicit value wins; otherwise reuse the +value already stored in the cluster Secret so `helm upgrade` never rotates it +(rotating ENCRYPTION_KEY/SESSION_SECRET would orphan encrypted data and +invalidate every session); otherwise generate a fresh 32-hex-char value. +Note: `lookup` returns nothing during `helm template`/`--dry-run`, so those +render a throwaway value each time - only real installs/upgrades retain. +Args: dict "existingData" "key" "value" +*/}} +{{- define "trigger-v4.resolveSecret" -}} +{{- if .value -}} +{{- .value -}} +{{- else if (index .existingData .key) -}} +{{- index .existingData .key | b64dec -}} +{{- else -}} +{{- sha256sum (randAlphaNum 32) | trunc 32 -}} +{{- end -}} +{{- end }} + {{/* Registry connection details */}} diff --git a/hosting/k8s/helm/templates/datastore-secret.yaml b/hosting/k8s/helm/templates/datastore-secret.yaml new file mode 100644 index 00000000000..392a2e7af76 --- /dev/null +++ b/hosting/k8s/helm/templates/datastore-secret.yaml @@ -0,0 +1,39 @@ +{{/* +Chart-managed credentials for the bundled datastores (postgres/clickhouse/minio). +Each password is generated ONCE here (retained across upgrades via lookup) and +consumed in two places from this single source: the Bitnami subchart reads it via +its `auth.existingSecret`, and the webapp reads it via secretKeyRef + `$(VAR)` +runtime interpolation in the connection URL. Generating it once and reading it back +is what keeps the server credential and the app's URL identical. + +The registry password is NOT here - it is consumed at template render time +(htpasswd + dockerconfigjson) so it is generated and retained inside secrets.yaml. +*/}} +{{- if or .Values.postgres.deploy .Values.clickhouse.deploy .Values.s3.deploy }} +{{- $name := include "trigger-v4.datastore.secretName" . }} +{{- $existing := (lookup "v1" "Secret" .Release.Namespace $name) | default dict }} +{{- $existingData := (get $existing "data") | default dict }} +apiVersion: v1 +kind: Secret +metadata: + name: {{ $name }} + labels: + {{- include "trigger-v4.labels" . | nindent 4 }} + annotations: + # Never lose these - the password is the only copy the datastore volumes accept. + helm.sh/resource-policy: keep +type: Opaque +data: + {{- if .Values.postgres.deploy }} + {{- $pg := include "trigger-v4.resolveSecret" (dict "existingData" $existingData "key" "postgres-password" "value" .Values.postgres.auth.password) }} + postgres-password: {{ $pg | b64enc | quote }} + password: {{ $pg | b64enc | quote }} + {{- end }} + {{- if .Values.clickhouse.deploy }} + clickhouse-admin-password: {{ include "trigger-v4.resolveSecret" (dict "existingData" $existingData "key" "clickhouse-admin-password" "value" .Values.clickhouse.auth.password) | b64enc | quote }} + {{- end }} + {{- if .Values.s3.deploy }} + minio-root-user: {{ .Values.s3.auth.rootUser | default "admin" | b64enc | quote }} + minio-root-password: {{ include "trigger-v4.resolveSecret" (dict "existingData" $existingData "key" "minio-root-password" "value" (.Values.s3.auth.rootPassword | default .Values.s3.auth.secretAccessKey)) | b64enc | quote }} + {{- end }} +{{- end }} diff --git a/hosting/k8s/helm/templates/electric.yaml b/hosting/k8s/helm/templates/electric.yaml index ac2d7582f26..5bb1964a0e9 100644 --- a/hosting/k8s/helm/templates/electric.yaml +++ b/hosting/k8s/helm/templates/electric.yaml @@ -44,6 +44,13 @@ spec: name: {{ include "trigger-v4.postgres.external.secretName" . }} key: {{ include "trigger-v4.postgres.external.databaseUrlKey" . }} {{- else }} + {{- if .Values.postgres.deploy }} + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "trigger-v4.datastore.secretName" . }} + key: password + {{- end }} - name: DATABASE_URL value: {{ include "trigger-v4.postgres.connectionString" . | quote }} {{- end }} diff --git a/hosting/k8s/helm/templates/s2.yaml b/hosting/k8s/helm/templates/s2.yaml index 54e75b13bc2..eccfe4c14b8 100644 --- a/hosting/k8s/helm/templates/s2.yaml +++ b/hosting/k8s/helm/templates/s2.yaml @@ -56,7 +56,10 @@ spec: {{- end }} image: "{{ .Values.s2.image.registry }}/{{ .Values.s2.image.repository }}:{{ .Values.s2.image.tag }}{{ with .Values.s2.image.digest }}@{{ . }}{{ end }}" imagePullPolicy: {{ .Values.s2.image.pullPolicy }} - command: + # args, not command: the image ENTRYPOINT is ["./s2"] and these are its + # subcommand/flags. Using command would override the entrypoint and try to + # exec "lite" directly (StartError). + args: - "lite" - "--init-file" - "/config/s2-spec.json" diff --git a/hosting/k8s/helm/templates/secrets.yaml b/hosting/k8s/helm/templates/secrets.yaml index c122f89d0b3..3c19df0d3b4 100644 --- a/hosting/k8s/helm/templates/secrets.yaml +++ b/hosting/k8s/helm/templates/secrets.yaml @@ -1,4 +1,6 @@ {{- if and .Values.secrets.enabled (not .Values.secrets.existingSecret) }} +{{- $existing := (lookup "v1" "Secret" .Release.Namespace (printf "%s-secrets" (include "trigger-v4.fullname" .))) | default dict }} +{{- $existingData := (get $existing "data") | default dict }} apiVersion: v1 kind: Secret metadata: @@ -7,10 +9,12 @@ metadata: {{- include "trigger-v4.labels" . | nindent 4 }} type: Opaque data: - SESSION_SECRET: {{ .Values.secrets.sessionSecret | b64enc | quote }} - MAGIC_LINK_SECRET: {{ .Values.secrets.magicLinkSecret | b64enc | quote }} - ENCRYPTION_KEY: {{ .Values.secrets.encryptionKey | b64enc | quote }} - MANAGED_WORKER_SECRET: {{ .Values.secrets.managedWorkerSecret | b64enc | quote }} + SESSION_SECRET: {{ include "trigger-v4.resolveSecret" (dict "existingData" $existingData "key" "SESSION_SECRET" "value" .Values.secrets.sessionSecret) | b64enc | quote }} + MAGIC_LINK_SECRET: {{ include "trigger-v4.resolveSecret" (dict "existingData" $existingData "key" "MAGIC_LINK_SECRET" "value" .Values.secrets.magicLinkSecret) | b64enc | quote }} + ENCRYPTION_KEY: {{ include "trigger-v4.resolveSecret" (dict "existingData" $existingData "key" "ENCRYPTION_KEY" "value" .Values.secrets.encryptionKey) | b64enc | quote }} + PROVIDER_SECRET: {{ include "trigger-v4.resolveSecret" (dict "existingData" $existingData "key" "PROVIDER_SECRET" "value" .Values.secrets.providerSecret) | b64enc | quote }} + COORDINATOR_SECRET: {{ include "trigger-v4.resolveSecret" (dict "existingData" $existingData "key" "COORDINATOR_SECRET" "value" .Values.secrets.coordinatorSecret) | b64enc | quote }} + MANAGED_WORKER_SECRET: {{ include "trigger-v4.resolveSecret" (dict "existingData" $existingData "key" "MANAGED_WORKER_SECRET" "value" .Values.secrets.managedWorkerSecret) | b64enc | quote }} {{- if and .Values.s3.external.accessKeyId (not .Values.s3.external.existingSecret) }} s3-access-key-id: {{ .Values.s3.external.accessKeyId | b64enc | quote }} s3-secret-access-key: {{ .Values.s3.external.secretAccessKey | b64enc | quote }} @@ -36,15 +40,27 @@ data: {{- end }} --- {{- if and .Values.registry.deploy .Values.registry.auth.enabled }} +{{/* Generated here (not in datastore-secret) because it is consumed at render time by + htpasswd and the dockerconfigjson below. Retained via the plaintext `password` key + on this secret; htpasswd bcrypt is one-way so it can't be the retain source. */}} +{{- $regAuthName := printf "%s-registry-auth" (include "trigger-v4.fullname" .) }} +{{- $regExisting := (lookup "v1" "Secret" .Release.Namespace $regAuthName) | default dict }} +{{- $regExistingData := (get $regExisting "data") | default dict }} +{{- $regPass := include "trigger-v4.resolveSecret" (dict "existingData" $regExistingData "key" "password" "value" .Values.registry.auth.password) }} +{{- $regAuth := printf "%s:%s" .Values.registry.auth.username $regPass | b64enc }} +{{- $regConfig := dict "auths" (dict (include "trigger-v4.registry.host" .) (dict "username" .Values.registry.auth.username "password" $regPass "auth" $regAuth)) }} apiVersion: v1 kind: Secret metadata: - name: {{ include "trigger-v4.fullname" . }}-registry-auth + name: {{ $regAuthName }} labels: {{- include "trigger-v4.labels" . | nindent 4 }} + annotations: + helm.sh/resource-policy: keep type: Opaque data: - htpasswd: {{ htpasswd .Values.registry.auth.username .Values.registry.auth.password | trim | b64enc | quote }} + htpasswd: {{ htpasswd .Values.registry.auth.username $regPass | trim | b64enc | quote }} + password: {{ $regPass | b64enc | quote }} --- apiVersion: v1 kind: Secret @@ -54,7 +70,7 @@ metadata: {{- include "trigger-v4.labels" . | nindent 4 }} type: kubernetes.io/dockerconfigjson data: - .dockerconfigjson: {{ include "trigger-v4.imagePullSecret" . | b64enc }} + .dockerconfigjson: {{ $regConfig | toJson | b64enc }} {{- else if and (not .Values.registry.deploy) .Values.registry.external.auth.enabled }} apiVersion: v1 kind: Secret diff --git a/hosting/k8s/helm/templates/validate-external-config.yaml b/hosting/k8s/helm/templates/validate-external-config.yaml index 063aeecbcfd..939e699a423 100644 --- a/hosting/k8s/helm/templates/validate-external-config.yaml +++ b/hosting/k8s/helm/templates/validate-external-config.yaml @@ -7,6 +7,7 @@ This template will fail the Helm deployment if external config is missing for re {{- fail "PostgreSQL external configuration is required when postgres.deploy=false. Please provide either postgres.external.databaseUrl or postgres.external.existingSecret" }} {{- end }} {{- end }} +{{/* When postgres.deploy=true the password is auto-generated into the datastore Secret (templates/datastore-secret.yaml) and consumed by the subchart via auth.existingSecret; set postgres.auth.password to pin it. */}} {{- if not .Values.redis.deploy }} {{- if not .Values.redis.external.host }} @@ -19,12 +20,9 @@ This template will fail the Helm deployment if external config is missing for re {{- fail "ClickHouse external configuration is required when clickhouse.deploy=false. Please provide clickhouse.external.host and clickhouse.external.username" }} {{- end }} {{- end }} +{{/* When clickhouse.deploy=true the password is auto-generated into the datastore Secret and consumed via auth.existingSecret; set clickhouse.auth.password to pin it. */}} -{{- if .Values.s3.deploy }} -{{- if and (not .Values.s3.auth.existingSecret) (not .Values.s3.auth.accessKeyId) (not .Values.s3.auth.rootUser) }} -{{- fail "S3 auth credentials are required when s3.deploy=true. Please provide either s3.auth.accessKeyId, s3.auth.existingSecret, or s3.auth.rootUser" }} -{{- end }} -{{- else }} +{{- if not .Values.s3.deploy }} {{- if not .Values.s3.external.endpoint }} {{- fail "S3 external configuration is required when s3.deploy=false. Please provide s3.external.endpoint" }} {{- end }} @@ -32,6 +30,7 @@ This template will fail the Helm deployment if external config is missing for re {{- fail "S3 credentials are required when s3.deploy=false. Please provide either s3.external.existingSecret or both s3.external.accessKeyId and s3.external.secretAccessKey" }} {{- end }} {{- end }} +{{/* When s3.deploy=true the MinIO root password is auto-generated into the datastore Secret and consumed via auth.existingSecret; set s3.auth.rootPassword to pin it. */}} {{- if not .Values.electric.deploy }} {{- if not .Values.electric.external.url }} @@ -48,6 +47,14 @@ This template will fail the Helm deployment if external config is missing for re {{- fail "Registry external configuration is required when registry.deploy=false. Please provide registry.external.host" }} {{- end }} {{- end }} +{{/* When registry.deploy=true the password is auto-generated + retained inside secrets.yaml; set registry.auth.password to pin it. */}} + +{{/* +Application and control-plane secrets are auto-generated by templates/secrets.yaml +when left unset (retained across upgrades via lookup), so they need no fail-closed +guard here. The webapp still rejects previously published values at startup, even +when supplied via secrets.existingSecret. +*/}} {{/* This template produces no output but will fail the deployment if validation fails diff --git a/hosting/k8s/helm/templates/webapp.yaml b/hosting/k8s/helm/templates/webapp.yaml index f83ff0ecb84..49acee98f4a 100644 --- a/hosting/k8s/helm/templates/webapp.yaml +++ b/hosting/k8s/helm/templates/webapp.yaml @@ -221,6 +221,13 @@ spec: name: {{ include "trigger-v4.postgres.external.secretName" . }} key: {{ include "trigger-v4.postgres.external.directUrlKey" . }} {{- else }} + {{- if .Values.postgres.deploy }} + - name: POSTGRES_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "trigger-v4.datastore.secretName" . }} + key: password + {{- end }} - name: DATABASE_URL value: {{ include "trigger-v4.postgres.connectionString" . | quote }} - name: DIRECT_URL @@ -306,6 +313,16 @@ spec: secretKeyRef: name: {{ include "trigger-v4.secretsName" . }} key: ENCRYPTION_KEY + - name: PROVIDER_SECRET + valueFrom: + secretKeyRef: + name: {{ include "trigger-v4.secretsName" . }} + key: PROVIDER_SECRET + - name: COORDINATOR_SECRET + valueFrom: + secretKeyRef: + name: {{ include "trigger-v4.secretsName" . }} + key: COORDINATOR_SECRET - name: MANAGED_WORKER_SECRET valueFrom: secretKeyRef: @@ -401,6 +418,12 @@ spec: secretKeyRef: name: {{ include "trigger-v4.clickhouse.external.secretName" . }} key: {{ include "trigger-v4.clickhouse.external.passwordKey" . }} + {{- else if .Values.clickhouse.deploy }} + - name: CLICKHOUSE_PASSWORD + valueFrom: + secretKeyRef: + name: {{ include "trigger-v4.datastore.secretName" . }} + key: clickhouse-admin-password {{- end }} - name: CLICKHOUSE_URL value: {{ include "trigger-v4.clickhouse.url" . | quote }} diff --git a/hosting/k8s/helm/values-production-example.yaml b/hosting/k8s/helm/values-production-example.yaml index 65bdc0a6fc5..ee99de58a9b 100644 --- a/hosting/k8s/helm/values-production-example.yaml +++ b/hosting/k8s/helm/values-production-example.yaml @@ -7,6 +7,8 @@ secrets: magicLinkSecret: "YOUR_32_CHAR_HEX_SECRET_HERE_002" encryptionKey: "YOUR_32_CHAR_HEX_SECRET_HERE_003" managedWorkerSecret: "YOUR_32_CHAR_HEX_SECRET_HERE_004" + providerSecret: "YOUR_32_CHAR_HEX_SECRET_HERE_005" + coordinatorSecret: "YOUR_32_CHAR_HEX_SECRET_HERE_006" # Production webapp configuration webapp: @@ -42,6 +44,10 @@ webapp: # Production PostgreSQL (or use external) postgres: + auth: + # Required — no built-in default. The webapp connection string uses postgres.auth.password. + postgresPassword: "your-strong-postgres-password" + password: "your-strong-postgres-password" primary: persistence: enabled: true @@ -72,6 +78,9 @@ redis: # Production ClickHouse clickhouse: + auth: + # Required — no built-in default. The webapp connection string uses clickhouse.auth.password. + password: "your-strong-clickhouse-password" # Set to true to enable TLS/secure connections in production secure: true persistence: diff --git a/hosting/k8s/helm/values.yaml b/hosting/k8s/helm/values.yaml index 968419d9871..e76a921279c 100644 --- a/hosting/k8s/helm/values.yaml +++ b/hosting/k8s/helm/values.yaml @@ -10,11 +10,10 @@ nameOverride: "" fullnameOverride: "" # Secrets configuration -# IMPORTANT: The default values below are for TESTING ONLY and should NOT be used in production -# For production deployments: -# 1. Generate new secrets using: openssl rand -hex 16 -# 2. Override these values in your values.yaml or use external secret management -# 3. Each secret must be exactly 32 hex characters (16 bytes) +# No working defaults are shipped. Leave a secret empty and the chart generates a +# strong random value on first install, retained across upgrades (so sessions and +# encrypted data survive). Set a value explicitly to control it yourself, or use +# secrets.existingSecret / external secret management. Explicit values win. secrets: # Enable/disable creation of secrets # Set to false to use external secret management (Vault, Infisical, External Secrets, etc.) @@ -27,17 +26,23 @@ secrets: # - SESSION_SECRET # - MAGIC_LINK_SECRET # - ENCRYPTION_KEY + # - PROVIDER_SECRET + # - COORDINATOR_SECRET # - MANAGED_WORKER_SECRET existingSecret: "" - # Session secret for user authentication (32 hex chars) - sessionSecret: "2818143646516f6fffd707b36f334bbb" - # Magic link secret for passwordless login (32 hex chars) - magicLinkSecret: "44da78b7bbb0dfe709cf38931d25dcdd" - # Encryption key for sensitive data (32 hex chars) - encryptionKey: "f686147ab967943ebbe9ed3b496e465a" - # Worker secret for managed worker authentication (32 hex chars) - managedWorkerSecret: "447c29678f9eaf289e9c4b70d3dd8a7f" + # Session secret for user authentication + sessionSecret: "" # Leave empty to auto-generate, or set a strong value. + # Magic link secret for passwordless login + magicLinkSecret: "" # Leave empty to auto-generate, or set a strong value. + # Encryption key for sensitive data + encryptionKey: "" # Leave empty to auto-generate, or set a strong value. + # Provider socket secret + providerSecret: "" # Leave empty to auto-generate, or set a strong value. + # Coordinator socket secret + coordinatorSecret: "" # Leave empty to auto-generate, or set a strong value. + # Worker secret for managed worker authentication + managedWorkerSecret: "" # Leave empty to auto-generate, or set a strong value. # Object store credentials moved to s3.auth and s3.external section # Webapp configuration @@ -397,10 +402,16 @@ postgres: # Bitnami PostgreSQL chart configuration (when deploy: true) auth: enablePostgresUser: true - postgresPassword: "postgres" + postgresPassword: "" # Leave empty to auto-generate into the datastore secret, or set to pin. username: "postgres" - password: "postgres" + password: "" # Leave empty to auto-generate into the datastore secret, or set to pin. database: "main" + # Read the auto-generated password from the chart-managed datastore secret + # (templates/datastore-secret.yaml). Must equal trigger-v4.datastore.secretName. + existingSecret: "trigger-datastore" + secretKeys: + adminPasswordKey: "postgres-password" + userPasswordKey: "password" primary: persistence: @@ -632,7 +643,10 @@ clickhouse: # Bitnami ClickHouse chart configuration (when deploy: true) auth: username: "default" - password: "password" + password: "" # Leave empty to auto-generate into the datastore secret, or set to pin. + # Read the auto-generated password from the chart-managed datastore secret. + existingSecret: "trigger-datastore" + existingSecretKey: "clickhouse-admin-password" # Single-node configuration (disable clustering for dev/test) keeper: @@ -709,14 +723,18 @@ s3: # MinIO provides S3-compatible storage when deployed internally auth: rootUser: "admin" - rootPassword: "very-safe-password" + rootPassword: "" # Leave empty to auto-generate into the datastore secret, or set to pin. # Webapp credentials for S3 access (defaults to root credentials if not specified) accessKeyId: "" # Defaults to rootUser if empty secretAccessKey: "" # Defaults to rootPassword if empty - # Existing secret support for webapp credentials - existingSecret: "" # If set, accessKeyId/secretAccessKey will be ignored - accessKeyIdSecretKey: "access-key-id" # Key in existingSecret containing access key ID - secretAccessKeySecretKey: "secret-access-key" # Key in existingSecret containing secret access key + # Read the auto-generated root credentials from the chart-managed datastore secret. + # The same keys serve the MinIO subchart (rootUser/rootPassword) and the webapp's + # S3 credentials, so both stay in sync. Must equal trigger-v4.datastore.secretName. + existingSecret: "trigger-datastore" + rootUserSecretKey: "minio-root-user" + rootPasswordSecretKey: "minio-root-password" + accessKeyIdSecretKey: "minio-root-user" # Key in existingSecret containing access key ID + secretAccessKeySecretKey: "minio-root-password" # Key in existingSecret containing secret access key # The required "packets" bucket is created by default. defaultBuckets: "packets" @@ -729,8 +747,8 @@ s3: # External S3 connection (when deploy: false) external: endpoint: "" # e.g., "https://s3.amazonaws.com" or "https://your-minio.com:9000" - accessKeyId: "admin" # Default for internal MinIO - change for production - secretAccessKey: "very-safe-password" # Default for internal MinIO - change for production + accessKeyId: "" # Required when s3.deploy=false — no default. Set explicitly or use existingSecret. + secretAccessKey: "" # Required when s3.deploy=false — no default. Set explicitly or use existingSecret. # # Secure credential management existingSecret: "" # Name of existing secret containing S3 credentials @@ -769,7 +787,7 @@ registry: auth: enabled: true username: "registry-user" - password: "very-secure-indeed" + password: "" # Required when registry.deploy=true and auth.enabled — no default. Set a strong value. # External Registry connection (when deploy: false) external: diff --git a/internal-packages/run-engine/src/engine/index.ts b/internal-packages/run-engine/src/engine/index.ts index bf9612ba198..3c1f4330a0f 100644 --- a/internal-packages/run-engine/src/engine/index.ts +++ b/internal-packages/run-engine/src/engine/index.ts @@ -1497,6 +1497,7 @@ export class RunEngine { workerId, runnerId, isWarmStart, + environmentId, tx, }: { runId: string; @@ -1504,6 +1505,7 @@ export class RunEngine { workerId?: string; runnerId?: string; isWarmStart?: boolean; + environmentId?: string; tx?: PrismaClientOrTransaction; }): Promise { return this.runAttemptSystem.startRunAttempt({ @@ -1512,6 +1514,7 @@ export class RunEngine { workerId, runnerId, isWarmStart, + environmentId, tx, }); } @@ -1523,12 +1526,14 @@ export class RunEngine { completion, workerId, runnerId, + environmentId, }: { runId: string; snapshotId: string; completion: TaskRunExecutionResult; workerId?: string; runnerId?: string; + environmentId?: string; }): Promise { return this.runAttemptSystem.completeRunAttempt({ runId, @@ -1536,6 +1541,7 @@ export class RunEngine { completion, workerId, runnerId, + environmentId, }); } @@ -2014,12 +2020,14 @@ export class RunEngine { snapshotId, workerId, runnerId, + environmentId, tx, }: { runId: string; snapshotId: string; workerId?: string; runnerId?: string; + environmentId?: string; tx?: PrismaClientOrTransaction; }): Promise { return this.checkpointSystem.continueRunExecution({ @@ -2027,6 +2035,7 @@ export class RunEngine { snapshotId, workerId, runnerId, + environmentId, tx, }); } @@ -2062,14 +2071,21 @@ export class RunEngine { /** Get required data to execute the run */ async getRunExecutionData({ runId, + environmentId, tx, }: { runId: string; + environmentId?: string; tx?: PrismaClientOrTransaction; }): Promise { const prisma = tx ?? this.prisma; try { - const snapshot = await getLatestExecutionSnapshot(prisma, runId, this.runStore); + const snapshot = await getLatestExecutionSnapshot( + prisma, + runId, + this.runStore, + environmentId + ); return executionDataFromSnapshot(snapshot); } catch (e) { this.logger.error("Failed to getRunExecutionData", { @@ -2086,10 +2102,12 @@ export class RunEngine { async getSnapshotsSince({ runId, snapshotId, + environmentId, tx, }: { runId: string; snapshotId: string; + environmentId?: string; tx?: PrismaClientOrTransaction; }): Promise { const useReplica = @@ -2107,7 +2125,8 @@ export class RunEngine { runId, snapshotId, this.runStore, - repairClient + repairClient, + environmentId ); return snapshots.map(executionDataFromSnapshot); }; diff --git a/internal-packages/run-engine/src/engine/systems/checkpointSystem.ts b/internal-packages/run-engine/src/engine/systems/checkpointSystem.ts index 7bac40838f8..a8f4eab1fdd 100644 --- a/internal-packages/run-engine/src/engine/systems/checkpointSystem.ts +++ b/internal-packages/run-engine/src/engine/systems/checkpointSystem.ts @@ -271,18 +271,25 @@ export class CheckpointSystem { snapshotId, workerId, runnerId, + environmentId, tx, }: { runId: string; snapshotId: string; workerId?: string; runnerId?: string; + environmentId?: string; tx?: PrismaClientOrTransaction; }): Promise { const prisma = tx ?? this.$.prisma; return await this.$.runLock.lock("continueRunExecution", [runId], async () => { - const snapshot = await getLatestExecutionSnapshot(prisma, runId, this.$.runStore); + const snapshot = await getLatestExecutionSnapshot( + prisma, + runId, + this.$.runStore, + environmentId + ); if (snapshot.id !== snapshotId) { throw new ServiceValidationError( diff --git a/internal-packages/run-engine/src/engine/systems/executionSnapshotSystem.ts b/internal-packages/run-engine/src/engine/systems/executionSnapshotSystem.ts index b98f2920b39..dca4c66b2e7 100644 --- a/internal-packages/run-engine/src/engine/systems/executionSnapshotSystem.ts +++ b/internal-packages/run-engine/src/engine/systems/executionSnapshotSystem.ts @@ -11,7 +11,7 @@ import type { Waitpoint, } from "@trigger.dev/database"; import type { RunStore } from "@internal/run-store"; -import { ExecutionSnapshotNotFoundError } from "../errors.js"; +import { ExecutionSnapshotNotFoundError, ServiceValidationError } from "../errors.js"; import type { HeartbeatTimeouts } from "../types.js"; import type { SystemResources } from "./systems.js"; @@ -195,16 +195,21 @@ async function fetchWaitpointsInChunks( return allWaitpoints; } -/* Gets the most recent valid snapshot for a run */ +/** + * Gets the most recent valid snapshot for a run. When `environmentId` is provided the read is scoped + * to that environment (tenant boundary): a run in another environment reads as not-found and rejects + * with a 404 rather than leaking existence. Internal callers omit it to read regardless of env. + */ export async function getLatestExecutionSnapshot( prisma: PrismaClientOrTransaction, runId: string, - runStore?: RunStore + runStore?: RunStore, + environmentId?: string ): Promise { const snapshot = runStore - ? await runStore.findLatestExecutionSnapshot(runId, prisma) + ? await runStore.findLatestExecutionSnapshot(runId, prisma, environmentId) : await prisma.taskRunExecutionSnapshot.findFirst({ - where: { runId, isValid: true }, + where: { runId, isValid: true, ...(environmentId ? { environmentId } : {}) }, include: { completedWaitpoints: true, checkpoint: true, @@ -213,6 +218,9 @@ export async function getLatestExecutionSnapshot( }); if (!snapshot) { + if (environmentId) { + throw new ServiceValidationError(`No execution snapshot found for TaskRun ${runId}`, 404); + } throw new Error(`No execution snapshot found for TaskRun ${runId}`); } @@ -295,19 +303,24 @@ export async function getExecutionSnapshotsSince( runStore?: RunStore, // The primary, for read-repair when `prisma` is a lagging read replica (see Step 3). Omit when // `prisma` is already the primary. - repairClient?: PrismaClientOrTransaction + repairClient?: PrismaClientOrTransaction, + // When set, scopes both reads to this environment (tenant boundary): a run in another env reads as + // not-found. Omit to read regardless of environment (internal callers). + environmentId?: string ): Promise { + const envScope = environmentId ? { environmentId } : {}; + // Step 1: Find the createdAt of the sinceSnapshotId const sinceSnapshot = runStore ? await runStore.findExecutionSnapshot( { - where: { id: sinceSnapshotId, runId }, + where: { id: sinceSnapshotId, runId, ...envScope }, select: { createdAt: true }, }, prisma ) : await prisma.taskRunExecutionSnapshot.findFirst({ - where: { id: sinceSnapshotId, runId }, + where: { id: sinceSnapshotId, runId, ...envScope }, select: { createdAt: true }, }); @@ -323,6 +336,7 @@ export async function getExecutionSnapshotsSince( runId, isValid: true, createdAt: { gt: sinceSnapshot.createdAt }, + ...envScope, }, include: { checkpoint: true, @@ -338,6 +352,7 @@ export async function getExecutionSnapshotsSince( runId, isValid: true, createdAt: { gt: sinceSnapshot.createdAt }, + ...envScope, }, include: { checkpoint: true, diff --git a/internal-packages/run-engine/src/engine/systems/runAttemptSystem.ts b/internal-packages/run-engine/src/engine/systems/runAttemptSystem.ts index 922b41cd225..c7a331fe9de 100644 --- a/internal-packages/run-engine/src/engine/systems/runAttemptSystem.ts +++ b/internal-packages/run-engine/src/engine/systems/runAttemptSystem.ts @@ -300,6 +300,7 @@ export class RunAttemptSystem { workerId, runnerId, isWarmStart, + environmentId, tx, }: { runId: string; @@ -307,6 +308,7 @@ export class RunAttemptSystem { workerId?: string; runnerId?: string; isWarmStart?: boolean; + environmentId?: string; tx?: PrismaClientOrTransaction; }): Promise { const prisma = tx ?? this.$.prisma; @@ -316,7 +318,12 @@ export class RunAttemptSystem { "startRunAttempt", async (span) => { return this.$.runLock.lock("startRunAttempt", [runId], async () => { - const latestSnapshot = await getLatestExecutionSnapshot(prisma, runId, this.$.runStore); + const latestSnapshot = await getLatestExecutionSnapshot( + prisma, + runId, + this.$.runStore, + environmentId + ); if (latestSnapshot.id !== snapshotId) { //if there is a big delay between the snapshot and the attempt, the snapshot might have changed @@ -643,12 +650,14 @@ export class RunAttemptSystem { completion, workerId, runnerId, + environmentId, }: { runId: string; snapshotId: string; completion: TaskRunExecutionResult; workerId?: string; runnerId?: string; + environmentId?: string; }): Promise { await this.#notifyMetadataUpdated(runId, completion); @@ -661,6 +670,7 @@ export class RunAttemptSystem { tx: this.$.prisma, workerId, runnerId, + environmentId, }); } case false: { @@ -671,6 +681,7 @@ export class RunAttemptSystem { tx: this.$.prisma, workerId, runnerId, + environmentId, }); } } @@ -683,6 +694,7 @@ export class RunAttemptSystem { tx, workerId, runnerId, + environmentId, }: { runId: string; snapshotId: string; @@ -690,6 +702,7 @@ export class RunAttemptSystem { tx: PrismaClientOrTransaction; workerId?: string; runnerId?: string; + environmentId?: string; }): Promise { const prisma = tx ?? this.$.prisma; @@ -698,7 +711,12 @@ export class RunAttemptSystem { "#completeRunAttemptSuccess", async (span) => { return this.$.runLock.lock("attemptSucceeded", [runId], async () => { - const latestSnapshot = await getLatestExecutionSnapshot(prisma, runId, this.$.runStore); + const latestSnapshot = await getLatestExecutionSnapshot( + prisma, + runId, + this.$.runStore, + environmentId + ); if (latestSnapshot.id !== snapshotId) { throw new ServiceValidationError("Snapshot ID doesn't match the latest snapshot", 400); @@ -868,6 +886,7 @@ export class RunAttemptSystem { runnerId, completion, forceRequeue, + environmentId, tx, }: { runId: string; @@ -876,6 +895,7 @@ export class RunAttemptSystem { runnerId?: string; completion: TaskRunFailedExecutionResult; forceRequeue?: boolean; + environmentId?: string; tx: PrismaClientOrTransaction; }): Promise { const prisma = this.$.prisma; @@ -885,7 +905,12 @@ export class RunAttemptSystem { "completeRunAttemptFailure", async (span) => { return this.$.runLock.lock("attemptFailed", [runId], async () => { - const latestSnapshot = await getLatestExecutionSnapshot(prisma, runId, this.$.runStore); + const latestSnapshot = await getLatestExecutionSnapshot( + prisma, + runId, + this.$.runStore, + environmentId + ); if (latestSnapshot.id !== snapshotId) { throw new ServiceValidationError("Snapshot ID doesn't match the latest snapshot", 400); diff --git a/internal-packages/run-store/src/PostgresRunStore.ts b/internal-packages/run-store/src/PostgresRunStore.ts index 5f636ada987..1b0d2f89cee 100644 --- a/internal-packages/run-store/src/PostgresRunStore.ts +++ b/internal-packages/run-store/src/PostgresRunStore.ts @@ -1708,15 +1708,17 @@ export class PostgresRunStore implements RunStore { async findLatestExecutionSnapshot( runId: string, - client?: ReadClient + client?: ReadClient, + environmentId?: string ): Promise | null> { const prisma = client ?? this.readOnlyPrisma; + const where = { runId, isValid: true, ...(environmentId ? { environmentId } : {}) }; if (this.schemaVariant === "dedicated") { const snapshot = await prisma.taskRunExecutionSnapshot.findFirst({ - where: { runId, isValid: true }, + where, include: { checkpoint: true }, orderBy: { createdAt: "desc" }, }); @@ -1728,7 +1730,7 @@ export class PostgresRunStore implements RunStore { } return prisma.taskRunExecutionSnapshot.findFirst({ - where: { runId, isValid: true }, + where, include: { completedWaitpoints: true, checkpoint: true, diff --git a/internal-packages/run-store/src/runOpsStore.ts b/internal-packages/run-store/src/runOpsStore.ts index d14ae525224..ba496980768 100644 --- a/internal-packages/run-store/src/runOpsStore.ts +++ b/internal-packages/run-store/src/runOpsStore.ts @@ -841,14 +841,16 @@ export class RoutingRunStore implements RunStore { // OUTPUT is silently missing from the resume payload — re-resolve them across BOTH DBs. async findLatestExecutionSnapshot( runId: string, - client?: ReadClient + client?: ReadClient, + environmentId?: string ): Promise | null> { const owningStore = this.#routeOrNew(runId); const snapshot = await owningStore.findLatestExecutionSnapshot( runId, - RoutingRunStore.#ownPrimary(owningStore, client) + RoutingRunStore.#ownPrimary(owningStore, client), + environmentId ); if (snapshot) { await this.#reresolveCompletedWaitpointsCrossDb( diff --git a/internal-packages/run-store/src/types.ts b/internal-packages/run-store/src/types.ts index b55e8f968c2..374a550e419 100644 --- a/internal-packages/run-store/src/types.ts +++ b/internal-packages/run-store/src/types.ts @@ -673,7 +673,10 @@ export interface RunStore { // Snapshot group findLatestExecutionSnapshot( runId: string, - client?: ReadClient + client?: ReadClient, + // When set, scopes the read to this environment (tenant boundary); a run in another env reads as + // not-found. Omit to read regardless of environment (internal callers). + environmentId?: string ): Promise | null>; diff --git a/internal-packages/testcontainers/src/webapp.ts b/internal-packages/testcontainers/src/webapp.ts index 2d2774df420..fc587abd696 100644 --- a/internal-packages/testcontainers/src/webapp.ts +++ b/internal-packages/testcontainers/src/webapp.ts @@ -104,6 +104,11 @@ export async function startWebapp( SESSION_SECRET: "test-session-secret-for-e2e-tests", MAGIC_LINK_SECRET: "test-magic-link-secret-32chars!!", ENCRYPTION_KEY: "test-encryption-key-for-e2e!!!!!", // exactly 32 bytes + // Control-plane auth secrets are required (no defaults) and reject the + // known-insecure placeholder strings, so supply strong test values here. + PROVIDER_SECRET: "test-provider-secret-for-e2e-tests", + COORDINATOR_SECRET: "test-coordinator-secret-for-e2e-tests", + MANAGED_WORKER_SECRET: "test-managed-worker-secret-for-e2e-tests", CLICKHOUSE_URL: "http://localhost:19123", // dummy, auth paths never connect DEPLOY_REGISTRY_HOST: "registry.example.com", // dummy, not needed for auth tests ELECTRIC_ORIGIN: "http://localhost:3060", diff --git a/internal-packages/tsql/src/query/printer.ts b/internal-packages/tsql/src/query/printer.ts index 6f1af07d71c..00ff8eabdba 100644 --- a/internal-packages/tsql/src/query/printer.ts +++ b/internal-packages/tsql/src/query/printer.ts @@ -3127,8 +3127,16 @@ export class ClickHousePrinter { } private visitWindowFunction(node: WindowFunction): string { + // Validate the function name against the allowlist and resolve it to its + // safe ClickHouse name. Without this, an attacker-controlled name would be + // emitted raw into the query (mirrors the validation in visitCall). + const funcMeta = findTSQLFunction(node.name) ?? findTSQLAggregation(node.name); + if (!funcMeta) { + throw new QueryError(`Unknown function: ${node.name}`); + } + const args = node.args ? node.args.map((a) => this.visit(a)) : []; - const funcCall = `${node.name}(${args.join(", ")})`; + const funcCall = `${funcMeta.clickhouseName}(${args.join(", ")})`; if (node.over_identifier) { return `${funcCall} OVER ${this.printIdentifier(node.over_identifier)}`; diff --git a/packages/cli-v3/src/commands/login.ts b/packages/cli-v3/src/commands/login.ts index 0171286d416..83576b0d010 100644 --- a/packages/cli-v3/src/commands/login.ts +++ b/packages/cli-v3/src/commands/login.ts @@ -295,9 +295,10 @@ export async function login(options?: LoginOptions): Promise { const indexResult = await pRetry( () => getPersonalAccessToken(apiClient, authorizationCodeResult.authorizationCode), { - //this means we're polling, same distance between each attempt + //poll at a fixed 1s interval. ~5 min window so the user has time to + //approve the consent screen; stays within the code's 10-min validity. factor: 1, - retries: 60, + retries: 300, minTimeout: 1000, } ); @@ -404,6 +405,15 @@ export async function getPersonalAccessToken(apiClient: CliApiClient, authorizat const token = await apiClient.getPersonalAccessToken(authorizationCode); if (!token.success) { + // A 429 from the per-code poll rate limiter is transient: the auth code + // is still valid and the user may just not have approved the consent + // screen yet. Throw a regular (retryable) error so the poll loop backs + // off and keeps polling, rather than an AbortError, which pRetry treats + // as fatal and would abandon the whole login. + if (token.statusCode === 429) { + throw new Error(token.error); + } + throw new AbortError(token.error); } diff --git a/packages/cli-v3/src/entryPoints/managed/controller.ts b/packages/cli-v3/src/entryPoints/managed/controller.ts index 5c63a4082d7..ae1f1d22a4a 100644 --- a/packages/cli-v3/src/entryPoints/managed/controller.ts +++ b/packages/cli-v3/src/entryPoints/managed/controller.ts @@ -76,7 +76,7 @@ export class ManagedRunController { }); const properties = { - ...env.raw, + ...env.rawForLogging, TRIGGER_POD_SCHEDULED_AT_MS: env.TRIGGER_POD_SCHEDULED_AT_MS.toISOString(), TRIGGER_DEQUEUED_AT_MS: env.TRIGGER_DEQUEUED_AT_MS.toISOString(), }; diff --git a/packages/cli-v3/src/entryPoints/managed/env.ts b/packages/cli-v3/src/entryPoints/managed/env.ts index 4da8361e863..825e83e1572 100644 --- a/packages/cli-v3/src/entryPoints/managed/env.ts +++ b/packages/cli-v3/src/entryPoints/managed/env.ts @@ -27,6 +27,9 @@ const Env = z.object({ // Set at runtime TRIGGER_DEPLOYMENT_ID: z.string(), + // Plain deployment friendlyId for telemetry. Optional: older supervisors don't set it, in which + // case we fall back to TRIGGER_DEPLOYMENT_ID. + TRIGGER_DEPLOYMENT_FRIENDLY_ID: z.string().optional(), TRIGGER_DEPLOYMENT_VERSION: z.string(), TRIGGER_WORKLOAD_CONTROLLER_ID: z.string().default(`controller_${randomUUID()}`), TRIGGER_ENV_ID: z.string(), @@ -74,6 +77,11 @@ export class RunnerEnv { return this.env; } + // TRIGGER_DEPLOYMENT_ID carries the deployment token; redact it before logging. + get rawForLogging() { + return { ...this.env, TRIGGER_DEPLOYMENT_ID: "[redacted]" }; + } + // Base environment variables get NODE_ENV() { return this.env.NODE_ENV; @@ -93,6 +101,9 @@ export class RunnerEnv { get TRIGGER_DEPLOYMENT_ID() { return this.env.TRIGGER_DEPLOYMENT_ID; } + get TRIGGER_DEPLOYMENT_FRIENDLY_ID() { + return this.env.TRIGGER_DEPLOYMENT_FRIENDLY_ID; + } get TRIGGER_DEPLOYMENT_VERSION() { return this.env.TRIGGER_DEPLOYMENT_VERSION; } diff --git a/packages/cli-v3/src/entryPoints/managed/execution.ts b/packages/cli-v3/src/entryPoints/managed/execution.ts index 8dca48fe279..c5bb4875bce 100644 --- a/packages/cli-v3/src/entryPoints/managed/execution.ts +++ b/packages/cli-v3/src/entryPoints/managed/execution.ts @@ -951,7 +951,7 @@ export class RunExecution { this.sendDebugLog(`[override] processing: ${reason}`, { overrides, - currentEnv: this.env.raw, + currentEnv: this.env.rawForLogging, }); // Override the env with the new values diff --git a/packages/cli-v3/src/entryPoints/managed/taskRunProcessProvider.ts b/packages/cli-v3/src/entryPoints/managed/taskRunProcessProvider.ts index 4e0a561220b..aaf09862f1d 100644 --- a/packages/cli-v3/src/entryPoints/managed/taskRunProcessProvider.ts +++ b/packages/cli-v3/src/entryPoints/managed/taskRunProcessProvider.ts @@ -252,11 +252,14 @@ export class TaskRunProcessProvider { workerManifest: this.workerManifest, env: processEnv, serverWorker: { - id: this.env.TRIGGER_DEPLOYMENT_ID, + // Telemetry-only (becomes OTel worker.id). Prefer the plain friendlyId; fall back to + // DEPLOYMENT_ID (which may be an opaque token) only when an older supervisor didn't set it. + id: this.env.TRIGGER_DEPLOYMENT_FRIENDLY_ID ?? this.env.TRIGGER_DEPLOYMENT_ID, contentHash: this.env.TRIGGER_CONTENT_HASH, version: this.env.TRIGGER_DEPLOYMENT_VERSION, engine: "V2", }, + machineResources: { cpu: Number(this.env.TRIGGER_MACHINE_CPU), memory: Number(this.env.TRIGGER_MACHINE_MEMORY), diff --git a/packages/cli-v3/src/mcp/auth.ts b/packages/cli-v3/src/mcp/auth.ts index b948f2cb889..fa5f62708ec 100644 --- a/packages/cli-v3/src/mcp/auth.ts +++ b/packages/cli-v3/src/mcp/auth.ts @@ -123,9 +123,10 @@ export async function mcpAuth(options: McpAuthOptions): Promise { const indexResult = await pRetry( () => getPersonalAccessToken(apiClient, authorizationCodeResult.authorizationCode), { - //this means we're polling, same distance between each attempt + //poll at a fixed 1s interval. ~5 min window so the user has time to + //approve the consent screen; stays within the code's 10-min validity. factor: 1, - retries: 60, + retries: 300, minTimeout: 1000, } ); diff --git a/packages/core/src/v3/apiClient/core.ts b/packages/core/src/v3/apiClient/core.ts index 3c9ae47cfb1..c6425e7ee37 100644 --- a/packages/core/src/v3/apiClient/core.ts +++ b/packages/core/src/v3/apiClient/core.ts @@ -725,6 +725,13 @@ export type ApiResult = | { success: false; error: string; + /** + * HTTP status code, when the failure originated from an API response + * (e.g. 429 for rate limiting). Undefined for connection/transport + * errors that never reached the server. Lets callers distinguish + * transient failures worth retrying from fatal ones. + */ + statusCode?: number; }; export async function wrapZodFetch( @@ -754,6 +761,7 @@ export async function wrapZodFetch( return { success: false, error: error.message, + statusCode: error.status, }; } else if (error instanceof Error) { return { diff --git a/packages/core/src/v3/index.ts b/packages/core/src/v3/index.ts index 9052884bbd6..35dbfa287f9 100644 --- a/packages/core/src/v3/index.ts +++ b/packages/core/src/v3/index.ts @@ -30,6 +30,7 @@ export * from "./resource-catalog-api.js"; export * from "./types/index.js"; export { links } from "./links.js"; export * from "./jwt.js"; +export * from "./workloadDeploymentToken.js"; export * from "./idempotencyKeys.js"; export * from "./streams/asyncIterableStream.js"; export * from "./utils/getEnv.js"; diff --git a/packages/core/src/v3/jwt.ts b/packages/core/src/v3/jwt.ts index d71f1e7f1e8..6845225be8a 100644 --- a/packages/core/src/v3/jwt.ts +++ b/packages/core/src/v3/jwt.ts @@ -4,6 +4,10 @@ export type GenerateJWTOptions = { secretKey: string; payload: Record; expirationTime?: number | Date | string; + // Skip the `iat` claim. Combined with an absolute `expirationTime`, this makes the signed token a + // pure function of its payload — the same claims mint byte-identical tokens. Off by default so + // ordinary short-lived tokens keep their issued-at. + omitIssuedAt?: boolean; }; export const JWT_ALGORITHM = "HS256"; @@ -15,13 +19,17 @@ export async function generateJWT(options: GenerateJWTOptions): Promise const secret = new TextEncoder().encode(options.secretKey); - return new SignJWT(options.payload) + const jwt = new SignJWT(options.payload) .setIssuer(JWT_ISSUER) .setAudience(JWT_AUDIENCE) .setProtectedHeader({ alg: JWT_ALGORITHM }) - .setIssuedAt() - .setExpirationTime(options.expirationTime ?? "15m") - .sign(secret); + .setExpirationTime(options.expirationTime ?? "15m"); + + if (!options.omitIssuedAt) { + jwt.setIssuedAt(); + } + + return jwt.sign(secret); } export type ValidationResult = diff --git a/packages/core/src/v3/runEngineWorker/consts.ts b/packages/core/src/v3/runEngineWorker/consts.ts index 29dc76b5358..e04e83875b2 100644 --- a/packages/core/src/v3/runEngineWorker/consts.ts +++ b/packages/core/src/v3/runEngineWorker/consts.ts @@ -3,6 +3,9 @@ export const WORKER_HEADERS = { DEPLOYMENT_ID: "x-trigger-worker-deployment-id", MANAGED_SECRET: "x-trigger-worker-managed-secret", RUNNER_ID: "x-trigger-worker-runner-id", + // Verified environment id recovered from the deployment token. Set by the supervisor only when + // enforcing; the platform scopes the worker-action's snapshot read by it. + ENVIRONMENT_ID: "x-trigger-worker-environment-id", }; export const WORKLOAD_HEADERS = { diff --git a/packages/core/src/v3/runEngineWorker/supervisor/http.ts b/packages/core/src/v3/runEngineWorker/supervisor/http.ts index bbb1fe227de..d6d51b1c2b2 100644 --- a/packages/core/src/v3/runEngineWorker/supervisor/http.ts +++ b/packages/core/src/v3/runEngineWorker/supervisor/http.ts @@ -143,7 +143,8 @@ export class SupervisorHttpClient { runId: string, snapshotId: string, body: WorkerApiRunAttemptStartRequestBody, - runnerId?: string + runnerId?: string, + environmentId?: string ) { return wrapZodFetch( WorkerApiRunAttemptStartResponseBody, @@ -153,6 +154,7 @@ export class SupervisorHttpClient { headers: { ...this.defaultHeaders, ...this.runnerIdHeader(runnerId), + ...this.environmentIdHeader(environmentId), }, body: JSON.stringify(body), } @@ -163,7 +165,8 @@ export class SupervisorHttpClient { runId: string, snapshotId: string, body: WorkerApiRunAttemptCompleteRequestBody, - runnerId?: string + runnerId?: string, + environmentId?: string ) { return wrapZodFetch( WorkerApiRunAttemptCompleteResponseBody, @@ -173,13 +176,14 @@ export class SupervisorHttpClient { headers: { ...this.defaultHeaders, ...this.runnerIdHeader(runnerId), + ...this.environmentIdHeader(environmentId), }, body: JSON.stringify(body), } ); } - async getLatestSnapshot(runId: string, runnerId?: string) { + async getLatestSnapshot(runId: string, runnerId?: string, environmentId?: string) { return wrapZodFetch( WorkerApiRunLatestSnapshotResponseBody, `${this.apiUrl}/engine/v1/worker-actions/runs/${runId}/snapshots/latest`, @@ -188,12 +192,18 @@ export class SupervisorHttpClient { headers: { ...this.defaultHeaders, ...this.runnerIdHeader(runnerId), + ...this.environmentIdHeader(environmentId), }, } ); } - async getSnapshotsSince(runId: string, snapshotId: string, runnerId?: string) { + async getSnapshotsSince( + runId: string, + snapshotId: string, + runnerId?: string, + environmentId?: string + ) { return wrapZodFetch( WorkerApiRunSnapshotsSinceResponseBody, `${this.apiUrl}/engine/v1/worker-actions/runs/${runId}/snapshots/since/${snapshotId}`, @@ -202,6 +212,7 @@ export class SupervisorHttpClient { headers: { ...this.defaultHeaders, ...this.runnerIdHeader(runnerId), + ...this.environmentIdHeader(environmentId), }, } ); @@ -235,7 +246,12 @@ export class SupervisorHttpClient { } } - async continueRunExecution(runId: string, snapshotId: string, runnerId?: string) { + async continueRunExecution( + runId: string, + snapshotId: string, + runnerId?: string, + environmentId?: string + ) { return wrapZodFetch( WorkerApiContinueRunExecutionRequestBody, `${this.apiUrl}/engine/v1/worker-actions/runs/${runId}/snapshots/${snapshotId}/continue`, @@ -244,6 +260,7 @@ export class SupervisorHttpClient { headers: { ...this.defaultHeaders, ...this.runnerIdHeader(runnerId), + ...this.environmentIdHeader(environmentId), }, }, { @@ -295,4 +312,10 @@ export class SupervisorHttpClient { [WORKER_HEADERS.RUNNER_ID]: runnerId, }); } + + private environmentIdHeader(environmentId?: string): Record { + return createHeaders({ + [WORKER_HEADERS.ENVIRONMENT_ID]: environmentId, + }); + } } diff --git a/packages/core/src/v3/runEngineWorker/types.ts b/packages/core/src/v3/runEngineWorker/types.ts index b54d23ccde7..7a76af3f4cb 100644 --- a/packages/core/src/v3/runEngineWorker/types.ts +++ b/packages/core/src/v3/runEngineWorker/types.ts @@ -29,4 +29,6 @@ export type WorkloadClientSocketData = { runnerId: string; runFriendlyId?: string; snapshotId?: string; + // Deployment friendlyId resolved from the verified token; prefer over the raw deploymentId. + deploymentFriendlyId?: string; }; diff --git a/packages/core/src/v3/runMetadata/operations.ts b/packages/core/src/v3/runMetadata/operations.ts index b1cc68672c2..fff98c50bc9 100644 --- a/packages/core/src/v3/runMetadata/operations.ts +++ b/packages/core/src/v3/runMetadata/operations.ts @@ -1,5 +1,5 @@ import { JSONHeroPath } from "@jsonhero/path"; -import type { RunMetadataChangeOperation } from "../schemas/common.js"; +import { isSafeMetadataKey, type RunMetadataChangeOperation } from "../schemas/common.js"; import { dequal } from "dequal"; export type ApplyOperationResult = { @@ -16,6 +16,13 @@ export function applyMetadataOperations( let newMetadata: Record = structuredClone(currentMetadata); for (const operation of Array.isArray(operations) ? operations : [operations]) { + // Prevent unsafe JSON paths and direct __proto__ assignments from changing Object.prototype. + // ("update" carries no key.) + if (operation.type !== "update" && !isSafeMetadataKey(operation.key)) { + unappliedOperations.push(operation); + continue; + } + switch (operation.type) { case "set": { if (operation.key.startsWith("$.")) { diff --git a/packages/core/src/v3/schemas/common.ts b/packages/core/src/v3/schemas/common.ts index 4de2ddb5802..a6ec4b1002d 100644 --- a/packages/core/src/v3/schemas/common.ts +++ b/packages/core/src/v3/schemas/common.ts @@ -4,6 +4,25 @@ import type { RuntimeEnvironmentType as DBRuntimeEnvironmentType } from "@trigge export type Enum = { [K in T]: K }; +const DANGEROUS_METADATA_KEY_SEGMENTS = new Set(["__proto__", "constructor", "prototype"]); + +/** + * Prototype-pollution guard for run metadata operation keys. JSON paths are applied via + * JSONHeroPath, so dangerous path segments must be rejected. Literal keys are assigned directly, + * where only __proto__ can change the target object's prototype. + */ +export function isSafeMetadataKey(key: string): boolean { + if (!key.startsWith("$.")) { + return key !== "__proto__"; + } + + return !key.split(/[.[\]'"]+/).some((segment) => DANGEROUS_METADATA_KEY_SEGMENTS.has(segment)); +} + +const MetadataOperationKey = z.string().refine(isSafeMetadataKey, { + message: "Metadata key may not reference __proto__, constructor, or prototype", +}); + export const RunMetadataUpdateOperation = z.object({ type: z.literal("update"), value: z.record(z.unknown()), @@ -13,7 +32,7 @@ export type RunMetadataUpdateOperation = z.infer; export const RunMetadataAppendKeyOperation = z.object({ type: z.literal("append"), - key: z.string(), + key: MetadataOperationKey, value: DeserializedJsonSchema, }); @@ -36,7 +55,7 @@ export type RunMetadataAppendKeyOperation = z.infer /^\d+$/.test(k))) { + // Convert the result to an array if all top-level keys are numeric indices. + // Guard against an empty result (e.g. every key was skipped as unsafe), which + // would otherwise produce Array(-Infinity) and throw. + if (Object.keys(result).length > 0 && Object.keys(result).every((k) => /^\d+$/.test(k))) { const maxIndex = Math.max(...Object.keys(result).map((k) => parseInt(k))); const arrayResult = Array(maxIndex + 1); for (const key in result) { diff --git a/packages/core/src/v3/workloadDeploymentToken.test.ts b/packages/core/src/v3/workloadDeploymentToken.test.ts new file mode 100644 index 00000000000..4a807b002ce --- /dev/null +++ b/packages/core/src/v3/workloadDeploymentToken.test.ts @@ -0,0 +1,127 @@ +import { describe, expect, it } from "vitest"; +import { generateJWT } from "./jwt.js"; +import { + classifyDeploymentIdHeader, + looksLikeWorkloadDeploymentToken, + mintWorkloadDeploymentToken, + verifyWorkloadDeploymentToken, + WORKLOAD_DEPLOYMENT_TOKEN_VERSION, + type WorkloadDeploymentTokenInput, +} from "./workloadDeploymentToken.js"; + +const SECRET = "test-workload-token-secret"; +const EXP = Math.floor(Date.UTC(2032, 0, 1) / 1000); + +const claims: WorkloadDeploymentTokenInput = { + deployment: "deployment_abc123", + deployment_version: "20260709.1", + environment_id: "env_1", + environment_type: "PRODUCTION", + org_id: "org_1", + project_id: "proj_1", +}; + +describe("workloadDeploymentToken", () => { + it("mints a token that verifies and round-trips every claim", async () => { + const token = await mintWorkloadDeploymentToken(claims, SECRET, EXP); + + const result = await verifyWorkloadDeploymentToken(token, SECRET); + expect(result.ok).toBe(true); + if (!result.ok) return; + + expect(result.claims.ver).toBe(WORKLOAD_DEPLOYMENT_TOKEN_VERSION); + expect(result.claims.deployment).toBe(claims.deployment); + expect(result.claims.deployment_version).toBe(claims.deployment_version); + expect(result.claims.environment_id).toBe(claims.environment_id); + expect(result.claims.environment_type).toBe(claims.environment_type); + expect(result.claims.org_id).toBe(claims.org_id); + expect(result.claims.project_id).toBe(claims.project_id); + }); + + it("sets the caller-supplied absolute exp", async () => { + const token = await mintWorkloadDeploymentToken(claims, SECRET, EXP); + const result = await verifyWorkloadDeploymentToken(token, SECRET); + expect(result.ok).toBe(true); + if (!result.ok) return; + + expect(result.claims.exp).toBe(EXP); + }); + + it("mints byte-identical tokens for identical claims (deterministic, no iat)", async () => { + const a = await mintWorkloadDeploymentToken(claims, SECRET, EXP); + const b = await mintWorkloadDeploymentToken(claims, SECRET, EXP); + expect(a).toBe(b); + + const result = await verifyWorkloadDeploymentToken(a, SECRET); + expect(result.ok).toBe(true); + if (!result.ok) return; + expect((result.claims as Record).iat).toBeUndefined(); + }); + + it("rejects a token signed with a different secret", async () => { + const token = await mintWorkloadDeploymentToken(claims, SECRET, EXP); + const result = await verifyWorkloadDeploymentToken(token, "wrong-secret"); + expect(result.ok).toBe(false); + if (result.ok) return; + expect(result.reason).toBe("invalid_signature"); + }); + + it("rejects a tampered token", async () => { + const token = await mintWorkloadDeploymentToken(claims, SECRET, EXP); + const [header, payload, signature] = token.split("."); + const tampered = `${header}.${payload}x.${signature}`; + const result = await verifyWorkloadDeploymentToken(tampered, SECRET); + expect(result.ok).toBe(false); + }); + + it("rejects a valid JWT that is missing required claims", async () => { + // Signed with the right secret, but not a workload deployment token. + const token = await generateJWT({ + secretKey: SECRET, + payload: { foo: "bar" }, + expirationTime: "1825d", + }); + const result = await verifyWorkloadDeploymentToken(token, SECRET); + expect(result.ok).toBe(false); + if (result.ok) return; + expect(result.reason).toBe("malformed_claims"); + }); + + describe("looksLikeWorkloadDeploymentToken", () => { + it("is true for a minted token", async () => { + const token = await mintWorkloadDeploymentToken(claims, SECRET, EXP); + expect(looksLikeWorkloadDeploymentToken(token)).toBe(true); + }); + + it("is false for a legacy bare friendlyId", () => { + expect(looksLikeWorkloadDeploymentToken("deployment_abc123")).toBe(false); + }); + + it("is false for empty and malformed values", () => { + expect(looksLikeWorkloadDeploymentToken("")).toBe(false); + expect(looksLikeWorkloadDeploymentToken("a.b")).toBe(false); + expect(looksLikeWorkloadDeploymentToken("a..c")).toBe(false); + }); + }); + + describe("classifyDeploymentIdHeader", () => { + it("classifies a minted token as jwt_valid and returns claims", async () => { + const token = await mintWorkloadDeploymentToken(claims, SECRET, EXP); + const result = await classifyDeploymentIdHeader(token, SECRET); + expect(result.outcome).toBe("jwt_valid"); + expect(result.claims?.deployment).toBe(claims.deployment); + }); + + it("classifies a JWT with a bad signature as jwt_invalid", async () => { + const token = await mintWorkloadDeploymentToken(claims, SECRET, EXP); + const result = await classifyDeploymentIdHeader(token, "wrong-secret"); + expect(result.outcome).toBe("jwt_invalid"); + expect(result.claims).toBeUndefined(); + }); + + it("classifies a bare friendlyId as legacy_bare", async () => { + const result = await classifyDeploymentIdHeader("deployment_abc123", SECRET); + expect(result.outcome).toBe("legacy_bare"); + }); + }); +}); diff --git a/packages/core/src/v3/workloadDeploymentToken.ts b/packages/core/src/v3/workloadDeploymentToken.ts new file mode 100644 index 00000000000..4f0d367038f --- /dev/null +++ b/packages/core/src/v3/workloadDeploymentToken.ts @@ -0,0 +1,99 @@ +import { z } from "zod"; +import { generateJWT, validateJWT } from "./jwt.js"; + +/** + * Signed, deployment-scoped token carried in the TRIGGER_DEPLOYMENT_ID env var to identify a run + * controller. Long-lived identity token, not a rotating credential; the `ver` claim allows a future + * format revision. + */ +export const WORKLOAD_DEPLOYMENT_TOKEN_VERSION = 1 as const; + +export const WorkloadDeploymentTokenClaims = z.object({ + ver: z.literal(WORKLOAD_DEPLOYMENT_TOKEN_VERSION), + /** Deployment friendlyId (deployment_xxxx). */ + deployment: z.string(), + deployment_version: z.string(), + environment_id: z.string(), + environment_type: z.string(), + org_id: z.string(), + project_id: z.string(), + exp: z.number(), +}); + +export type WorkloadDeploymentTokenClaims = z.infer; + +/** Claims the caller supplies; `ver` and `exp` are set by the minter. */ +export type WorkloadDeploymentTokenInput = Omit; + +/** + * `expiresAtSeconds` is an absolute epoch (caller-supplied — the supervisor drives it). Combined with + * the omitted `iat`, the token is byte-deterministic per deployment: identical claims mint identical + * bytes, so the OTel `worker.id` the runner derives from it stays one value per deployment. + */ +export async function mintWorkloadDeploymentToken( + claims: WorkloadDeploymentTokenInput, + secret: string, + expiresAtSeconds: number +): Promise { + return generateJWT({ + secretKey: secret, + payload: { ver: WORKLOAD_DEPLOYMENT_TOKEN_VERSION, ...claims }, + expirationTime: expiresAtSeconds, + omitIssuedAt: true, + }); +} + +export type WorkloadDeploymentTokenVerification = + | { ok: true; claims: WorkloadDeploymentTokenClaims } + | { ok: false; reason: "invalid_signature" | "malformed_claims"; error: string }; + +export async function verifyWorkloadDeploymentToken( + token: string, + secret: string +): Promise { + const result = await validateJWT(token, secret); + + if (!result.ok) { + return { ok: false, reason: "invalid_signature", error: result.error }; + } + + const parsed = WorkloadDeploymentTokenClaims.safeParse(result.payload); + + if (!parsed.success) { + return { ok: false, reason: "malformed_claims", error: parsed.error.message }; + } + + return { ok: true, claims: parsed.data }; +} + +/** + * A legacy TRIGGER_DEPLOYMENT_ID is a bare friendlyId (deployment_xxxx); a minted token is a JWT of + * three non-empty base64url segments. Used by the dry-run to tell a pre-upgrade runner from a token. + */ +export function looksLikeWorkloadDeploymentToken(value: string): boolean { + const parts = value.split("."); + return parts.length === 3 && parts.every((part) => part.length > 0); +} + +export type DeploymentIdHeaderOutcome = "jwt_valid" | "jwt_invalid" | "legacy_bare"; + +/** + * Classify the value carried in the deployment-id header for the rollout metric, verifying the + * signature when it is JWT-shaped. Returns the claims on a valid token so callers avoid a second verify. + */ +export async function classifyDeploymentIdHeader( + value: string, + secret: string +): Promise<{ outcome: DeploymentIdHeaderOutcome; claims?: WorkloadDeploymentTokenClaims }> { + if (!looksLikeWorkloadDeploymentToken(value)) { + return { outcome: "legacy_bare" }; + } + + const result = await verifyWorkloadDeploymentToken(value, secret); + + if (result.ok) { + return { outcome: "jwt_valid", claims: result.claims }; + } + + return { outcome: "jwt_invalid" }; +} diff --git a/packages/core/test/taskExecutor.test.ts b/packages/core/test/taskExecutor.test.ts index 9bbda935f6d..d46a3fbe67e 100644 --- a/packages/core/test/taskExecutor.test.ts +++ b/packages/core/test/taskExecutor.test.ts @@ -1802,8 +1802,10 @@ describe("TaskExecutor", () => { // Rate limit errors should use the rate limit retry delay expect((result.result as any).retry.delay).toBeGreaterThan(0); } else { - // Other retryable errors should use the exponential backoff - expect((result.result as any).retry.delay).toBeGreaterThan(1000); + // Other retryable errors should use the exponential backoff. The first retry uses + // minDelay (1000ms) as its base, and jitter can land exactly on the floor, so this + // is >= rather than > to avoid a flaky boundary failure. + expect((result.result as any).retry.delay).toBeGreaterThanOrEqual(1000); expect((result.result as any).retry.delay).toBeLessThan(5000); } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index b0e328c76a4..29450d4b162 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -585,6 +585,15 @@ importers: json-stable-stringify: specifier: ^1.3.0 version: 1.3.0 + jsonpointer: + specifier: ^5.0.1 + version: 5.0.1 + lodash.omit: + specifier: ^4.5.0 + version: 4.5.0 + lru-cache: + specifier: ^11.2.4 + version: 11.2.4 lucide-react: specifier: ^0.229.0 version: 0.229.0(react@18.3.1) @@ -11391,6 +11400,10 @@ packages: engines: {node: '>=18.0.0'} hasBin: true + jsonpointer@5.0.1: + resolution: {integrity: sha512-p/nXbhSEcu3pZRdkW1OfJhpsVtW1gd4Wa1fnQc9YLiTfAjn0312eMKimbdIQzuZl9aa9xUGaRlP9T/CJE/ditQ==} + engines: {node: '>=0.10.0'} + jsonwebtoken@9.0.2: resolution: {integrity: sha512-PRp66vJ865SSqOlgqS8hujT5U4AOgMfhrwYIuIhfKaoSCZcirrmASQr8CX7cUg+RMih+hgznrjp99o+W4pJLHQ==} engines: {node: '>=12', npm: '>=6'} @@ -26634,6 +26647,8 @@ snapshots: '@jsep-plugin/regex': 1.0.4(jsep@1.4.0) jsep: 1.4.0 + jsonpointer@5.0.1: {} + jsonwebtoken@9.0.2: dependencies: jws: 3.2.3