diff --git a/.github/workflows/containers.yml b/.github/workflows/containers.yml index 97d27857..593b3a94 100644 --- a/.github/workflows/containers.yml +++ b/.github/workflows/containers.yml @@ -1,5 +1,4 @@ name: Container matrix - on: pull_request: paths: @@ -10,6 +9,9 @@ on: - "starter-memory/**" - "Dockerfile" - "package-lock.json" + - "scripts/container-image-trim.mjs" + - "scripts/container-image-size-budget.mjs" + - "scripts/container-smoke/**" push: branches: - main @@ -21,6 +23,9 @@ on: - "starter-memory/**" - "Dockerfile" - "package-lock.json" + - "scripts/container-image-trim.mjs" + - "scripts/container-image-size-budget.mjs" + - "scripts/container-smoke/**" workflow_dispatch: permissions: @@ -44,7 +49,24 @@ jobs: - amd64 - arm64 runs-on: ubuntu-latest - timeout-minutes: 50 + # arm64 builds under QEMU need far longer than native amd64 ones. + timeout-minutes: ${{ matrix.arch == 'arm64' && 150 || 75 }} + services: + # The same pinned pgvector image the launcher compose file uses; the + # core and browser smoke checks run their migrations against it. + postgres: + image: pgvector/pgvector:0.8.3-pg17-trixie@sha256:3a3147ccd9a0aee952bb69e1acea1eb3cc7183544185b87a662f872cfa47ae3b + ports: + - 5432:5432 + env: + POSTGRES_USER: spaceapp + POSTGRES_PASSWORD: spaceapp + POSTGRES_DB: spaceapp + options: >- + --health-cmd "pg_isready -U spaceapp" + --health-interval 5s + --health-timeout 5s + --health-retries 20 steps: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: @@ -71,3 +93,13 @@ jobs: node scripts/container-image-size-budget.mjs --target "${{ matrix.target }}" --image "spaceapp-ci:${{ matrix.target }}-${{ matrix.arch }}" + + - name: Smoke check + run: >- + node scripts/container-smoke/${{ matrix.target }}-smoke.mjs + "spaceapp-ci:${{ matrix.target }}-${{ matrix.arch }}" + env: + SMOKE_DATABASE_URL: postgres://spaceapp:spaceapp@127.0.0.1:5432/spaceapp + # Chromium cannot start under qemu-user emulation; on an amd64 runner + # the arm64 job asserts the binary instead of launching it. + SMOKE_SKIP_CHROMIUM_LAUNCH: "${{ matrix.arch == 'arm64' && runner.arch == 'X64' }}" diff --git a/Dockerfile b/Dockerfile index 9100f79f..0178a991 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,10 +6,52 @@ RUN apt-get update \ && apt-get install -y --no-install-recommends build-essential liblzma5 python3 pkg-config \ && rm -rf /var/lib/apt/lists/* COPY . . +# The production dependency trees are built per target in the deps-* stages, +# so the build stage keeps its full dev install (typecheck, bundlers). RUN npm ci --no-audit --no-fund \ - && npm run build \ - && npm prune --omit=dev \ - && find node_modules -type f \( -name '*.js.map' -o -name '*.mjs.map' -o -name '*.cjs.map' -o -name '*.css.map' \) -delete + && npm run build + +# Per-target production dependency closures (#64): each stage reinstalls from +# the lockfile with --omit=dev limited to the workspaces that target runs, so +# its node_modules contains only that target's production closure. The trim +# step removes source maps and the prebuilt native libraries for platforms the +# image never runs on. Workspace build outputs (apps/*/dist, packages/*/dist) +# are not part of node_modules and survive the reinstall. +FROM build AS deps-core +ARG TARGETARCH +RUN npm ci --no-audit --no-fund --omit=dev -w apps/api -w apps/worker \ + && node scripts/container-image-trim.mjs --target core --arch "${TARGETARCH}" \ + && find apps packages \ + -type d \( -name node_modules -o -name dist \) -prune \ + -o -type d \( -name src -o -name __tests__ \) -prune -exec rm -rf {} + \ + && find apps packages \ + -type d \( -name node_modules -o -name dist \) -prune \ + -o -type f \( -name '*.test.js' -o -name '*.test.mjs' -o -name '*.test-d.ts' \) -exec rm -f {} + \ + && find apps packages -maxdepth 3 -name node_modules -type d -print + +FROM build AS deps-browser +ARG TARGETARCH +RUN npm ci --no-audit --no-fund --omit=dev -w apps/api \ + && node scripts/container-image-trim.mjs --target browser --arch "${TARGETARCH}" \ + && find apps packages \ + -type d \( -name node_modules -o -name dist \) -prune \ + -o -type d \( -name src -o -name __tests__ \) -prune -exec rm -rf {} + \ + && find apps packages \ + -type d \( -name node_modules -o -name dist \) -prune \ + -o -type f \( -name '*.test.js' -o -name '*.test.mjs' -o -name '*.test-d.ts' \) -exec rm -f {} + \ + && find apps packages -maxdepth 3 -name node_modules -type d -print + +FROM build AS deps-cli +ARG TARGETARCH +RUN npm ci --no-audit --no-fund --omit=dev -w packages/cli-host \ + && node scripts/container-image-trim.mjs --target cli --arch "${TARGETARCH}" \ + && find apps packages \ + -type d \( -name node_modules -o -name dist \) -prune \ + -o -type d \( -name src -o -name __tests__ \) -prune -exec rm -rf {} + \ + && find apps packages \ + -type d \( -name node_modules -o -name dist \) -prune \ + -o -type f \( -name '*.test.js' -o -name '*.test.mjs' -o -name '*.test-d.ts' \) -exec rm -f {} + \ + && find apps packages -maxdepth 3 -name node_modules -type d -print FROM node:22.23.0-trixie-slim@sha256:cfd8f2a5bc50526aee08e88970979f92722828e7dcc6d8983607fb8bff4bdb82 AS runtime-base ARG TARGETARCH @@ -55,15 +97,7 @@ RUN npm install --global --ignore-scripts --no-audit --no-fund "npm@${NPM_VERSIO && useradd --uid 10001 --gid 10001 --create-home --home-dir /var/lib/spaceapp-cli --shell /bin/bash spaceapp WORKDIR /app COPY --from=build --chown=spaceapp:spaceapp /app/package.json /app/package-lock.json ./ -COPY --from=build --chown=spaceapp:spaceapp /app/node_modules ./node_modules -COPY --from=build --chown=spaceapp:spaceapp /app/apps ./apps -COPY --from=build --chown=spaceapp:spaceapp /app/packages ./packages -COPY --from=build --chown=spaceapp:spaceapp /app/agent-skills ./agent-skills COPY --from=build --chown=spaceapp:spaceapp /app/LICENSE /app/NOTICE /app/THIRD_PARTY_NOTICES.md ./ -COPY --from=build --chown=spaceapp:spaceapp /app/scripts/reset-owner-password.mjs ./scripts/reset-owner-password.mjs -COPY --from=build --chown=spaceapp:spaceapp /app/scripts/rotate-owner-setup-token.mjs ./scripts/rotate-owner-setup-token.mjs -COPY --from=build --chown=spaceapp:spaceapp /app/scripts/portable-backup.mjs ./scripts/portable-backup.mjs -COPY --from=build --chown=spaceapp:spaceapp /app/scripts/portable-restore.mjs ./scripts/portable-restore.mjs COPY --chown=spaceapp:spaceapp deploy/docker ./deploy/docker RUN chmod 0755 deploy/docker/core-entrypoint.sh deploy/docker/browser-entrypoint.sh deploy/docker/cli-entrypoint.sh \ && install -m 0755 deploy/docker/public-cli-wrapper.sh /usr/local/bin/spaceapp-cli-wrapper \ @@ -71,7 +105,43 @@ RUN chmod 0755 deploy/docker/core-entrypoint.sh deploy/docker/browser-entrypoint ln -s /usr/local/bin/spaceapp-cli-wrapper "/usr/local/bin/${name}-vscode-parity"; \ done +# Core: API, Temporal worker, static web server and migrations. The image +# carries the apps/api + apps/worker production closure, the workspace build +# outputs it runs, the owner/backup scripts and the starter memory. FROM runtime-base AS core +COPY --from=deps-core --chown=spaceapp:spaceapp /app/node_modules ./node_modules +COPY --from=deps-core --chown=spaceapp:spaceapp \ + /app/apps/api ./apps/api +COPY --from=deps-core --chown=spaceapp:spaceapp \ + /app/apps/worker ./apps/worker +COPY --from=deps-core --chown=spaceapp:spaceapp /app/apps/web/dist ./apps/web/dist +COPY --from=deps-core --chown=spaceapp:spaceapp /app/apps/web/server.mjs ./apps/web/server.mjs +COPY --from=deps-core --chown=spaceapp:spaceapp /app/apps/web/package.json ./apps/web/package.json +COPY --from=deps-core --chown=spaceapp:spaceapp \ + /app/packages/browser-host ./packages/browser-host +COPY --from=deps-core --chown=spaceapp:spaceapp \ + /app/packages/cli-host ./packages/cli-host +COPY --from=deps-core --chown=spaceapp:spaceapp \ + /app/packages/codex-app-server ./packages/codex-app-server +COPY --from=deps-core --chown=spaceapp:spaceapp \ + /app/packages/contracts ./packages/contracts +COPY --from=deps-core --chown=spaceapp:spaceapp \ + /app/packages/db ./packages/db +COPY --from=deps-core --chown=spaceapp:spaceapp \ + /app/packages/gateway ./packages/gateway +COPY --from=deps-core --chown=spaceapp:spaceapp \ + /app/packages/memory-graph ./packages/memory-graph +COPY --from=deps-core --chown=spaceapp:spaceapp \ + /app/packages/opencode-control ./packages/opencode-control +COPY --from=deps-core --chown=spaceapp:spaceapp \ + /app/packages/runtime ./packages/runtime +COPY --from=deps-core --chown=spaceapp:spaceapp \ + /app/packages/streaming ./packages/streaming +COPY --from=build --chown=spaceapp:spaceapp /app/agent-skills ./agent-skills +COPY --from=build --chown=spaceapp:spaceapp /app/scripts/reset-owner-password.mjs ./scripts/reset-owner-password.mjs +COPY --from=build --chown=spaceapp:spaceapp /app/scripts/rotate-owner-setup-token.mjs ./scripts/rotate-owner-setup-token.mjs +COPY --from=build --chown=spaceapp:spaceapp /app/scripts/portable-backup.mjs ./scripts/portable-backup.mjs +COPY --from=build --chown=spaceapp:spaceapp /app/scripts/portable-restore.mjs ./scripts/portable-restore.mjs COPY --chown=spaceapp:spaceapp starter-memory /opt/spaceapp/starter-memory RUN install -d -o spaceapp -g spaceapp -m 0700 \ /var/lib/spaceapp \ @@ -83,6 +153,9 @@ HEALTHCHECK --interval=15s --timeout=5s --start-period=45s --retries=5 \ CMD ["node", "-e", "fetch('http://127.0.0.1:4911/readyz').then(async r=>{const body=await r.json();if(!r.ok||body.ok !== true)process.exit(1)}).catch(()=>process.exit(1))"] ENTRYPOINT ["/app/deploy/docker/core-entrypoint.sh"] +# Browser: Browser Host plus Chromium. The image carries the apps/api +# production closure (the Browser Host entry point lives in apps/api) and +# only the workspaces that entry point imports. FROM runtime-base AS browser RUN apt-get update \ && apt-get install -y --no-install-recommends chromium fonts-liberation liblzma5 \ @@ -93,8 +166,21 @@ RUN apt-get update \ /run/spaceapp-browser ENV SPACE_BROWSER_CHROME_PATH=/usr/bin/chromium \ SPACE_BROWSER_SESSIONS_CHROME_PATH=/usr/bin/chromium +COPY --from=deps-browser --chown=spaceapp:spaceapp /app/node_modules ./node_modules +COPY --from=deps-browser --chown=spaceapp:spaceapp \ + /app/apps/api ./apps/api +COPY --from=deps-browser --chown=spaceapp:spaceapp \ + /app/packages/browser-host ./packages/browser-host +COPY --from=deps-browser --chown=spaceapp:spaceapp \ + /app/packages/contracts ./packages/contracts +COPY --from=deps-browser --chown=spaceapp:spaceapp \ + /app/packages/db ./packages/db +COPY --from=deps-browser --chown=spaceapp:spaceapp \ + /app/packages/runtime ./packages/runtime ENTRYPOINT ["/app/deploy/docker/browser-entrypoint.sh"] +# CLI: the CLI host that runs the AI coding CLIs in PTYs. The image carries +# the packages/cli-host production closure, the agent skills and OpenCode. FROM runtime-base AS cli COPY --chown=root:root starter-memory/AGENTS.md /etc/AGENTS.md COPY --chown=root:root starter-memory/installation-guide.md /etc/spaceapp-installation-guide.md @@ -111,6 +197,10 @@ RUN npm install --global --no-audit --no-fund opencode-ai@1.18.4 \ /workspaces \ /run/spaceapp-cli \ && printf '%s\n' "$TARGETARCH" > /usr/local/share/spaceapp-cli-architecture +COPY --from=deps-cli --chown=spaceapp:spaceapp /app/node_modules ./node_modules +COPY --from=deps-cli --chown=spaceapp:spaceapp \ + /app/packages/cli-host ./packages/cli-host +COPY --from=build --chown=spaceapp:spaceapp /app/agent-skills ./agent-skills # The application's default room project is /opt/spaceapp. CLI sessions run in /etc, as # /etc/AGENTS.md documents, so that path resolves to /etc in this image. Without it, the # Getting Started OpenCode pane exits with "chdir(2) failed" before OpenCode starts. diff --git a/scripts/container-image-size-budget.mjs b/scripts/container-image-size-budget.mjs index fe006877..ecb7eb58 100644 --- a/scripts/container-image-size-budget.mjs +++ b/scripts/container-image-size-budget.mjs @@ -4,16 +4,25 @@ import { pathToFileURL } from "node:url"; const mebibyte = 1024 * 1024; -// 1.1.3 raised core and browser (998.7 / 1670.6 MiB measured): the current -// application adds production dependencies pdfjs-dist with @napi-rs/canvas -// (81 MiB, API PDF handling), typescript (23 MiB, peer of @t3-oss/env-core) and -// js-tiktoken (22 MiB, Space Gateway token counting). -// 1.1.5 raised core again: linux/arm64 measured 1066.9 MiB and linux/amd64 1045.0 MiB (sqlite3 in the image, -// the 1.1.5 application); the first public release run failed the arm64 build at 1050 MiB. +// #64 split the monolithic image into per-target production dependency +// closures, so each budget tracks only what its target runs. The budgets +// are UNCOMPRESSED image sizes: what `docker image inspect .Size` reports +// on the classic overlay2 store used by the containers.yml runners, which +// is where the budget step runs. Measured by the containers.yml matrix: +// linux/amd64 core 724.9 MiB, browser 1218.8 MiB, cli 758.2 MiB (the old +// monolithic image measured core 1045.0 MiB); linux/arm64 tracks amd64 +// within ~25 MiB (browser 1242.9 MiB). Each budget keeps room for the +// arm64 delta and dependency growth, while staying under the ceilings from +// the issue: core 850 MiB, browser 1350 MiB, cli 1000 MiB. +// Caveat: docker hosts running the containerd image store (Docker Desktop; +// docker 29 with the containerd snapshotter) report the COMPRESSED size in +// `docker image inspect .Size`, roughly 2.5-3x smaller for these images +// (core ~252 MiB, browser ~446 MiB, cli ~261 MiB on the wire). The CI +// matrix numbers are authoritative for these budgets. export const containerImageSizeBudgets = Object.freeze({ - core: 1100 * mebibyte, - browser: 1750 * mebibyte, - cli: 2800 * mebibyte + core: 800 * mebibyte, + browser: 1300 * mebibyte, + cli: 900 * mebibyte }); export function evaluateContainerImageSize({ target, sizeBytes }) { diff --git a/scripts/container-image-trim.mjs b/scripts/container-image-trim.mjs new file mode 100644 index 00000000..f1207c9a --- /dev/null +++ b/scripts/container-image-trim.mjs @@ -0,0 +1,151 @@ +#!/usr/bin/env node +// Removes files a production container target cannot load at runtime: source +// maps, the TypeScript package npm installs as an auto-installed peer +// dependency (nothing in a production image imports it), native prebuilds for +// platforms the image never runs on, and the Rust sources of the Temporal core +// bridge (the JavaScript loads only releases//index.node). +// +// Usage: container-image-trim.mjs --target --arch +import { existsSync, readdirSync, rmSync, statSync } from "node:fs"; +import { join } from "node:path"; +import { pathToFileURL } from "node:url"; + +const mebibyte = 1024 * 1024; + +export const temporalCoreBridgeReleaseTargetByArch = Object.freeze({ + amd64: "x86_64-unknown-linux-gnu", + arm64: "aarch64-unknown-linux-gnu" +}); + +export const napiRsCanvasDirectoryByArch = Object.freeze({ + amd64: "canvas-linux-x64-gnu", + arm64: "canvas-linux-arm64-gnu" +}); + +const validTargets = new Set(["core", "browser", "cli"]); +const validArchs = new Set(["amd64", "arm64"]); + +function optionValue(args, name) { + const index = args.indexOf(name); + return index === -1 ? undefined : args[index + 1]; +} + +function pathSizeBytes(path) { + const stats = statSync(path); + if (!stats.isDirectory()) return stats.size; + let total = 0; + for (const entry of readdirSync(path, { withFileTypes: true })) { + total += pathSizeBytes(join(path, entry.name)); + } + return total; +} + +function listFilesRecursive(root) { + const files = []; + for (const entry of readdirSync(root, { withFileTypes: true, recursive: true })) { + if (entry.isFile()) files.push(join(entry.parentPath, entry.name)); + } + return files; +} + +function removePath(path, removed, label) { + if (!existsSync(path)) return; + const bytes = pathSizeBytes(path); + rmSync(path, { recursive: true, force: true }); + removed.push({ label, path, bytes }); +} + +/** + * Trims the production node_modules tree of the given container target in the + * given root directory. The trim is safe for the target only when the + * node_modules tree was installed for the same architecture. + */ +export function trimContainerNodeModules({ target, arch, root = process.cwd() }) { + if (!validTargets.has(target)) { + throw new Error(`Unknown container target: ${target}`); + } + if (!validArchs.has(arch)) { + throw new Error(`Unknown container architecture: ${arch}`); + } + const removed = []; + const nodeModules = join(root, "node_modules"); + + // 1. Source maps: nothing imports them at runtime. + let sourceMapBytes = 0; + let sourceMapCount = 0; + if (existsSync(nodeModules)) { + for (const file of listFilesRecursive(nodeModules)) { + if (!file.endsWith(".map")) continue; + const bytes = statSync(file).size; + rmSync(file, { force: true }); + sourceMapBytes += bytes; + sourceMapCount += 1; + } + } + + // 2. TypeScript: npm installs it as an auto-installed peer dependency + // (peer of @t3-oss/env-core). No compiled entry point imports it: the + // build compiles TypeScript to JavaScript in the build stage. + removePath(join(nodeModules, "typescript"), removed, "typescript auto-installed peer"); + + // 3. Temporal core bridge: keep only the prebuilt library for this image's + // architecture and drop the Rust sources the JavaScript never reads. + const bridgeReleases = join(nodeModules, "@temporalio", "core-bridge", "releases"); + const bridgeKeep = temporalCoreBridgeReleaseTargetByArch[arch]; + if (existsSync(bridgeReleases)) { + for (const entry of readdirSync(bridgeReleases, { withFileTypes: true })) { + if (entry.name !== bridgeKeep) { + removePath(join(bridgeReleases, entry.name), removed, "temporal core-bridge platform prebuild"); + } + } + } + removePath(join(nodeModules, "@temporalio", "core-bridge", "sdk-core"), removed, "temporal core-bridge Rust sources"); + + // 4. node-pty ships Windows and macOS prebuilds; on Linux the install + // script builds the PTY from source (scripts/prebuild.js has no Linux + // prebuilds), and third_party/conpty is Windows-only console tooling. + removePath(join(nodeModules, "node-pty", "prebuilds"), removed, "node-pty non-Linux prebuilds"); + removePath(join(nodeModules, "node-pty", "third_party"), removed, "node-pty Windows console tooling"); + + // 5. @napi-rs/canvas is installed for every libc of the architecture; the + // images are glibc, so the musl builds are dead weight. + const napiRsRoot = join(nodeModules, "@napi-rs"); + const canvasKeep = napiRsCanvasDirectoryByArch[arch]; + if (existsSync(napiRsRoot)) { + for (const entry of readdirSync(napiRsRoot, { withFileTypes: true })) { + if (entry.name.startsWith("canvas-") && entry.name !== canvasKeep) { + removePath(join(napiRsRoot, entry.name), removed, "@napi-rs/canvas libc or architecture variant"); + } + } + } + + return { + target, + arch, + sourceMapCount, + sourceMapBytes, + removedBytes: removed.reduce((total, item) => total + item.bytes, 0), + removed + }; +} + +function main() { + const target = optionValue(process.argv.slice(2), "--target"); + const arch = optionValue(process.argv.slice(2), "--arch"); + if (target === undefined || arch === undefined) { + process.stderr.write("Usage: container-image-trim.mjs --target --arch \n"); + process.exitCode = 1; + return; + } + const summary = trimContainerNodeModules({ target, arch }); + const reclaimedMiB = (summary.removedBytes + summary.sourceMapBytes) / mebibyte; + process.stdout.write( + `container-image-trim: target ${summary.target} arch ${summary.arch}; ` + + `removed ${summary.removed.length + summary.sourceMapCount} items ` + + `(${summary.sourceMapCount} source maps), reclaimed ${reclaimedMiB.toFixed(1)} MiB\n` + ); +} + +if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) { + main(); +} diff --git a/scripts/container-smoke/browser-smoke.mjs b/scripts/container-smoke/browser-smoke.mjs new file mode 100644 index 00000000..25ab856f --- /dev/null +++ b/scripts/container-smoke/browser-smoke.mjs @@ -0,0 +1,116 @@ +#!/usr/bin/env node +// Browser image smoke check (#64): the image runs the database migrations, +// starts the Browser Host, answers a framed health ping on its socket and +// launches headless Chromium once. +// Usage: node scripts/container-smoke/browser-smoke.mjs +// Environment: SMOKE_DATABASE_URL (default spaceapp user on 127.0.0.1:5432), +// SMOKE_TIMEOUT_MS (default 240000), +// SMOKE_SKIP_CHROMIUM_LAUNCH ("true": assert the shipped Chromium binary is +// present and executable instead of launching it - used for arm64 jobs +// emulated under QEMU, where Chromium crashes during startup). +import { execFileSync } from "node:child_process"; +import { chmodSync, mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +const containerName = "spaceapp-smoke-browser"; +const socketPath = "/run/spaceapp-browser/browser-host.sock"; + +function sh(args, options = {}) { + // "input" only reaches the child when stdin is a pipe: with stdio[0] + // forced to "ignore", a piped program (docker exec -i node) reads EOF. + return execFileSync(args[0], args.slice(1), { stdio: [options.input ? "pipe" : "ignore", "pipe", "pipe"], encoding: "utf8", ...options }); +} + +function sleep(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +async function main() { + const image = process.argv[2]; + if (!image) { + throw new Error("Usage: node scripts/container-smoke/browser-smoke.mjs "); + } + const databaseUrl = process.env.SMOKE_DATABASE_URL ?? "postgres://spaceapp:spaceapp@127.0.0.1:5432/spaceapp"; + const timeoutMs = Number(process.env.SMOKE_TIMEOUT_MS ?? 240_000); + const secrets = mkdtempSync(join(tmpdir(), "spaceapp-smoke-browser-secrets-")); + const socketDir = mkdtempSync(join(tmpdir(), "spaceapp-smoke-browser-socket-")); + // The Browser Host runs as the image's spaceapp user (uid 10001); a bind-mounted + // host directory is owned by whoever runs the smoke, so make it writable. + chmodSync(socketDir, 0o777); + try { + writeFileSync(join(secrets, "database-url"), databaseUrl); + try { + sh(["docker", "rm", "-f", containerName]); + } catch {} + // Migrations first: the Browser Host opens the store when it starts. + // migrate.js resolves its database URL from the environment only. + sh(["docker", "run", "--rm", "--network", "host", + "-v", `${secrets}:/run/secrets:ro`, + "-e", `SPACE_DATABASE_URL=${databaseUrl}`, + "--entrypoint", "node", + image, "packages/db/dist/migrate.js"]); + sh(["docker", "run", "-d", "--name", containerName, "--network", "host", + "-v", `${secrets}:/run/secrets:ro`, + "-v", `${socketDir}:/run/spaceapp-browser`, + "-e", "SPACE_PUBLIC_DISTRIBUTION=true", + "-e", "SPACE_RUNTIME_STORE=postgres", + "-e", "SPACE_ARTIFACT_ROOT=/var/lib/spaceapp/artifacts", + "-e", "SPACE_BROWSER_SESSIONS_ENABLED=true", + "-e", `SPACE_BROWSER_HOST_SOCKET=${socketPath}`, + "-e", "SPACE_BROWSER_SESSIONS_DEFAULT_URL=about:blank", + image]); + const deadline = Date.now() + timeoutMs; + let socketUp = false; + while (Date.now() < deadline) { + await sleep(1_000); + try { + sh(["docker", "exec", containerName, "test", "-S", socketPath]); + socketUp = true; + break; + } catch {} + } + if (!socketUp) { + let logs = ""; + try { + // Merge stderr: a crashed entrypoint reports on stderr, stdout alone stays empty. + logs = sh(["bash", "-c", `docker logs --tail 60 ${containerName} 2>&1`]); + } catch {} + throw new Error(`Browser smoke failed: Browser Host socket did not appear\nContainer logs:\n${logs}`); + } + const healthScript = "const net = require(\"node:net\");\nconst socket = net.createConnection(\"/run/spaceapp-browser/browser-host.sock\");\nconst payload = Buffer.from(JSON.stringify({ kind: \"request\", requestId: \"smoke-health\", method: \"health\", params: {} }), \"utf8\");\nconst frame = Buffer.allocUnsafe(4 + payload.length);\nframe.writeUInt32BE(payload.length, 0);\npayload.copy(frame, 4);\nlet buffer = Buffer.alloc(0);\nconst timeout = setTimeout(() => { console.error(\"SMOKE-TIMEOUT\"); process.exit(1); }, 20000);\nsocket.on(\"connect\", () => socket.write(frame));\nsocket.on(\"data\", (chunk) => {\n buffer = Buffer.concat([buffer, chunk]);\n if (buffer.length < 4) return;\n const length = buffer.readUInt32BE(0);\n if (buffer.length < 4 + length) return;\n const value = JSON.parse(buffer.subarray(4, 4 + length).toString(\"utf8\"));\n if (value.kind === \"response\" && value.requestId === \"smoke-health\") {\n if (value.ok !== true) { console.error(\"SMOKE-ERROR \" + JSON.stringify(value.error)); process.exit(1); }\n console.log(\"SMOKE-OK\");\n clearTimeout(timeout);\n socket.end();\n process.exit(0);\n }\n});\nsocket.on(\"error\", (error) => { console.error(\"SMOKE-SOCKET-ERROR \" + error.message); process.exit(1); });"; + const healthOutput = sh(["docker", "exec", "-i", containerName, "node"], { input: healthScript }); + if (!healthOutput.includes("SMOKE-OK")) { + throw new Error(`Browser smoke failed: health response was not ok\n${healthOutput}`); + } + // Headless Chromium once: proves the shipped browser binary starts in + // this image (same user, same paths the Browser Host uses). Under QEMU + // user-mode emulation (arm64 job on an amd64 runner) Chromium dies + // during startup with SIGTRAP (crashpad prctl fails), which reflects + // the emulator, not the image, so SMOKE_SKIP_CHROMIUM_LAUNCH=true only + // asserts the binary is present and executable. + if (process.env.SMOKE_SKIP_CHROMIUM_LAUNCH === "true") { + sh(["docker", "exec", containerName, "test", "-x", "/usr/bin/chromium"]); + process.stdout.write("browser smoke: Browser Host healthy; headless Chromium launch skipped under QEMU (binary present and executable at /usr/bin/chromium)\n"); + } else { + const dom = sh(["docker", "exec", containerName, + "chromium", "--headless=new", "--no-sandbox", "--disable-gpu", + "--disable-dev-shm-usage", "--dump-dom", "about:blank"]); + if (!dom.includes(" +// Environment: SMOKE_TIMEOUT_MS (default 120000). +import { execFileSync } from "node:child_process"; +import { chmodSync, mkdtempSync, rmSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; + +const containerName = "spaceapp-smoke-cli"; +const socketPath = "/run/space-codex-pane-host/pane-host.sock"; + +function sh(args, options = {}) { + // "input" only reaches the child when stdin is a pipe: with stdio[0] + // forced to "ignore", a piped program (docker exec -i node) reads EOF. + return execFileSync(args[0], args.slice(1), { stdio: [options.input ? "pipe" : "ignore", "pipe", "pipe"], encoding: "utf8", ...options }); +} + +function sleep(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +async function main() { + const image = process.argv[2]; + if (!image) { + throw new Error("Usage: node scripts/container-smoke/cli-smoke.mjs "); + } + const timeoutMs = Number(process.env.SMOKE_TIMEOUT_MS ?? 120_000); + const socketDir = mkdtempSync(join(tmpdir(), "spaceapp-smoke-cli-socket-")); + // The pane host runs as the image's spaceapp user (uid 10001); a bind-mounted + // host directory is owned by whoever runs the smoke, so make it writable. + chmodSync(socketDir, 0o777); + try { + try { + sh(["docker", "rm", "-f", containerName]); + } catch {} + sh(["docker", "run", "-d", "--name", containerName, + "-v", `${socketDir}:/run/space-codex-pane-host`, + image]); + const deadline = Date.now() + timeoutMs; + let socketUp = false; + while (Date.now() < deadline) { + await sleep(1_000); + try { + sh(["docker", "exec", containerName, "test", "-S", socketPath]); + socketUp = true; + break; + } catch {} + } + if (!socketUp) { + let logs = ""; + try { + // Merge stderr: a crashed entrypoint reports on stderr, stdout alone stays empty. + logs = sh(["bash", "-c", `docker logs --tail 60 ${containerName} 2>&1`]); + } catch {} + throw new Error(`CLI smoke failed: pane host socket did not appear\nContainer logs:\n${logs}`); + } + const pingScript = "const net = require(\"node:net\");\nconst socket = net.createConnection(\"/run/space-codex-pane-host/pane-host.sock\");\nconst payload = Buffer.from(JSON.stringify({ kind: \"request\", requestId: \"smoke-ping\", method: \"ping\", params: {} }), \"utf8\");\nconst frame = Buffer.allocUnsafe(4 + payload.length);\nframe.writeUInt32BE(payload.length, 0);\npayload.copy(frame, 4);\nlet buffer = Buffer.alloc(0);\nconst timeout = setTimeout(() => { console.error(\"SMOKE-TIMEOUT\"); process.exit(1); }, 20000);\nsocket.on(\"connect\", () => socket.write(frame));\nsocket.on(\"data\", (chunk) => {\n buffer = Buffer.concat([buffer, chunk]);\n if (buffer.length < 4) return;\n const length = buffer.readUInt32BE(0);\n if (buffer.length < 4 + length) return;\n const value = JSON.parse(buffer.subarray(4, 4 + length).toString(\"utf8\"));\n if (value.kind === \"response\" && value.requestId === \"smoke-ping\") {\n if (value.ok !== true) { console.error(\"SMOKE-ERROR \" + JSON.stringify(value.error)); process.exit(1); }\n console.log(\"SMOKE-OK\");\n clearTimeout(timeout);\n socket.end();\n process.exit(0);\n }\n});\nsocket.on(\"error\", (error) => { console.error(\"SMOKE-SOCKET-ERROR \" + error.message); process.exit(1); });"; + const output = sh(["docker", "exec", "-i", containerName, "node"], { input: pingScript }); + if (!output.includes("SMOKE-OK")) { + throw new Error(`CLI smoke failed: ping response was not ok\n${output}`); + } + process.stdout.write("cli smoke: pane host accepted a framed ping\n"); + } finally { + try { + sh(["docker", "rm", "-f", containerName]); + } catch {} + // Best effort: the pane host owns the bind-mounted socket directory and may + // retighten its permissions, so an unremovable temp directory must not fail + // an otherwise passed smoke check. + try { rmSync(socketDir, { recursive: true, force: true }); } catch {} + } +} + +await main(); diff --git a/scripts/container-smoke/core-smoke.mjs b/scripts/container-smoke/core-smoke.mjs new file mode 100644 index 00000000..237183c5 --- /dev/null +++ b/scripts/container-smoke/core-smoke.mjs @@ -0,0 +1,90 @@ +#!/usr/bin/env node +// Core image smoke check (#64): the image starts against a pgvector +// PostgreSQL service, runs its migrations and answers /readyz with ok:true. +// Usage: node scripts/container-smoke/core-smoke.mjs +// Environment: SMOKE_DATABASE_URL (default spaceapp user on 127.0.0.1:5432), +// SMOKE_TIMEOUT_MS (default 300000), SMOKE_PORT (default 4911). +import { execFileSync } from "node:child_process"; +import { mkdtempSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { randomBytes } from "node:crypto"; + +const containerName = "spaceapp-smoke-core"; + +function sh(args) { + return execFileSync(args[0], args.slice(1), { stdio: ["ignore", "pipe", "pipe"], encoding: "utf8" }); +} + +function sleep(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +async function main() { + const image = process.argv[2]; + if (!image) { + throw new Error("Usage: node scripts/container-smoke/core-smoke.mjs "); + } + const databaseUrl = process.env.SMOKE_DATABASE_URL ?? "postgres://spaceapp:spaceapp@127.0.0.1:5432/spaceapp"; + const timeoutMs = Number(process.env.SMOKE_TIMEOUT_MS ?? 300_000); + const port = process.env.SMOKE_PORT ?? "4911"; + const secrets = mkdtempSync(join(tmpdir(), "spaceapp-smoke-core-secrets-")); + try { + writeFileSync(join(secrets, "database-url"), databaseUrl); + writeFileSync(join(secrets, "session-secret"), randomBytes(32).toString("hex")); + writeFileSync(join(secrets, "setup-token"), randomBytes(24).toString("hex")); + try { + sh(["docker", "rm", "-f", containerName]); + } catch {} + sh(["docker", "run", "-d", "--name", containerName, "--network", "host", + "-v", `${secrets}:/run/secrets:ro`, + "-e", "SPACE_RUNTIME_STORE=postgres", + "-e", "SPACE_PUBLIC_DISTRIBUTION=true", + // Workflows, CLI panes and browser panes are disabled so the check + // exercises this image (API + web server + migrations), not the other + // containers the launcher runs beside it. + "-e", "SPACEAPP_WORKFLOWS_ENABLED=false", + "-e", "SPACE_CLI_ENABLED=false", + "-e", "SPACE_BROWSER_SESSIONS_ENABLED=false", + "-e", "SPACE_API_HOST=127.0.0.1", + "-e", "SPACE_API_PORT=4910", + "-e", "SPACE_WEB_HOST=127.0.0.1", + "-e", `SPACE_WEB_PORT=${port}`, + "-e", "SPACE_API_ORIGIN=http://127.0.0.1:4910", + image]); + const deadline = Date.now() + timeoutMs; + let lastError = "no response"; + let lastBody = ""; + while (Date.now() < deadline) { + await sleep(2_000); + try { + const response = await fetch(`http://127.0.0.1:${port}/readyz`); + lastBody = await response.text(); + const body = JSON.parse(lastBody); + if (response.ok && body.ok === true) { + const logs = sh(["docker", "logs", containerName]); + if (!logs.includes("applied")) { + throw new Error("Core answered /readyz ok:true, but the migrations did not report applying."); + } + process.stdout.write("core smoke: /readyz ok:true, migrations applied\n"); + return; + } + lastError = `ok:${body.ok} dependencies:${JSON.stringify(body.dependencies)}`; + } catch (error) { + lastError = error instanceof Error ? error.message : String(error); + } + } + let logs = ""; + try { + logs = sh(["docker", "logs", "--tail", "80", containerName]); + } catch {} + throw new Error(`Core smoke failed: ${lastError}\nLast body: ${lastBody}\nContainer logs:\n${logs}`); + } finally { + try { + sh(["docker", "rm", "-f", containerName]); + } catch {} + rmSync(secrets, { recursive: true, force: true }); + } +} + +await main(); diff --git a/scripts/tests/container-image-size-budget.test.mjs b/scripts/tests/container-image-size-budget.test.mjs index 35146497..5ace6c0b 100644 --- a/scripts/tests/container-image-size-budget.test.mjs +++ b/scripts/tests/container-image-size-budget.test.mjs @@ -7,14 +7,24 @@ import { const mebibyte = 1024 * 1024; -test("public container size budgets preserve the lightweight release ceilings", () => { +test("public container size budgets preserve the per-service uncompressed sizes with headroom", () => { assert.deepEqual(containerImageSizeBudgets, { - core: 1100 * mebibyte, - browser: 1750 * mebibyte, - cli: 2800 * mebibyte + core: 800 * mebibyte, + browser: 1300 * mebibyte, + cli: 900 * mebibyte }); }); +test("per-service budgets stay under the #64 release ceilings", () => { + const ceilings = { core: 850, browser: 1350, cli: 1000 }; + for (const [target, ceiling] of Object.entries(ceilings)) { + assert.ok( + containerImageSizeBudgets[target] <= ceiling * mebibyte, + target + " budget exceeds the " + ceiling + " MiB ceiling" + ); + } +}); + test("container size budget accepts an image exactly at its target ceiling", () => { assert.deepEqual( evaluateContainerImageSize({ diff --git a/scripts/tests/container-image-trim.test.mjs b/scripts/tests/container-image-trim.test.mjs new file mode 100644 index 00000000..1e690e5c --- /dev/null +++ b/scripts/tests/container-image-trim.test.mjs @@ -0,0 +1,98 @@ +import assert from "node:assert/strict"; +import test from "node:test"; +import { existsSync, mkdirSync, mkdtempSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join } from "node:path"; +import { + napiRsCanvasDirectoryByArch, + temporalCoreBridgeReleaseTargetByArch, + trimContainerNodeModules +} from "../container-image-trim.mjs"; + +function makeFakeNodeModules() { + const root = mkdtempSync(join(tmpdir(), "container-image-trim-")); + const nodeModules = join(root, "node_modules"); + const file = (path, content = "x") => { + const absolute = join(nodeModules, path); + mkdirSync(dirname(absolute), { recursive: true }); + writeFileSync(absolute, content); + }; + // Source map and its compiled file. + file("example/lib/index.js", "module.exports = 1;"); + file("example/lib/index.js.map", "{}"); + // Auto-installed TypeScript peer. + file("typescript/lib/typescript.js"); + // Temporal core bridge with prebuilds for every platform and Rust sources. + file("@temporalio/core-bridge/index.js"); + for (const release of ["x86_64-apple-darwin", "x86_64-pc-windows-msvc", "x86_64-unknown-linux-gnu", "aarch64-unknown-linux-gnu"]) { + file(`@temporalio/core-bridge/releases/${release}/index.node`, "native"); + } + file("@temporalio/core-bridge/sdk-core/src/lib.rs"); + // node-pty with Windows prebuilds and Linux source build. + file("node-pty/build/Release/pty.node", "native"); + file("node-pty/prebuilds/win32-x64/pty.node", "native"); + file("node-pty/third_party/conpty/conpty.dll"); + // @napi-rs/canvas glibc and musl variants for both architectures. + file("@napi-rs/canvas-linux-x64-gnu/index.js"); + file("@napi-rs/canvas-linux-x64-musl/index.js"); + file("@napi-rs/canvas-linux-arm64-gnu/index.js"); + file("@napi-rs/canvas-linux-arm64-musl/index.js"); + return { root, nodeModules }; +} + +test("trim keeps only the target architecture Temporal prebuild and drops dead weight", () => { + const { root, nodeModules } = makeFakeNodeModules(); + try { + const summary = trimContainerNodeModules({ target: "core", arch: "amd64", root }); + assert.equal(summary.target, "core"); + assert.equal(summary.arch, "amd64"); + + // Source maps are gone, compiled files stay. + assert.equal(existsSync(join(nodeModules, "example/lib/index.js.map")), false); + assert.equal(existsSync(join(nodeModules, "example/lib/index.js")), true); + assert.equal(summary.sourceMapCount, 1); + + // The TypeScript peer dependency is removed. + assert.equal(existsSync(join(nodeModules, "typescript")), false); + + // Only the amd64 Linux Temporal prebuild stays; the Rust sources are gone. + const releases = join(nodeModules, "@temporalio/core-bridge/releases"); + assert.equal(existsSync(join(releases, temporalCoreBridgeReleaseTargetByArch.amd64, "index.node")), true); + assert.equal(existsSync(join(releases, temporalCoreBridgeReleaseTargetByArch.arm64)), false); + assert.equal(existsSync(join(releases, "x86_64-pc-windows-msvc")), false); + assert.equal(existsSync(join(nodeModules, "@temporalio/core-bridge/sdk-core")), false); + assert.equal(existsSync(join(nodeModules, "@temporalio/core-bridge/index.js")), true); + + // node-pty keeps its Linux build, drops Windows tooling and prebuilds. + assert.equal(existsSync(join(nodeModules, "node-pty/build/Release/pty.node")), true); + assert.equal(existsSync(join(nodeModules, "node-pty/prebuilds")), false); + assert.equal(existsSync(join(nodeModules, "node-pty/third_party")), false); + + // Only the glibc canvas variant of the architecture stays. + assert.equal(existsSync(join(nodeModules, `@napi-rs/${napiRsCanvasDirectoryByArch.amd64}`)), true); + assert.equal(existsSync(join(nodeModules, "@napi-rs/canvas-linux-x64-musl")), false); + assert.equal(existsSync(join(nodeModules, "@napi-rs/canvas-linux-arm64-gnu")), false); + assert.equal(existsSync(join(nodeModules, "@napi-rs/canvas-linux-arm64-musl")), false); + + // Every removed entry is reported with its size. + assert.ok(summary.removed.every((item) => Number.isSafeInteger(item.bytes) && item.bytes >= 0)); + } finally { + // mkdtempSync cleanup is not needed; the OS removes its tmpdir. + } +}); + +test("trim keeps the arm64 Temporal prebuild and canvas variant for arm64 images", () => { + const { root, nodeModules } = makeFakeNodeModules(); + trimContainerNodeModules({ target: "browser", arch: "arm64", root }); + const releases = join(nodeModules, "@temporalio/core-bridge/releases"); + assert.equal(existsSync(join(releases, temporalCoreBridgeReleaseTargetByArch.arm64, "index.node")), true); + assert.equal(existsSync(join(releases, temporalCoreBridgeReleaseTargetByArch.amd64)), false); + assert.equal(existsSync(join(nodeModules, `@napi-rs/${napiRsCanvasDirectoryByArch.arm64}`)), true); + assert.equal(existsSync(join(nodeModules, "@napi-rs/canvas-linux-x64-gnu")), false); +}); + +test("trim rejects unknown targets and architectures", () => { + const { root } = makeFakeNodeModules(); + assert.throws(() => trimContainerNodeModules({ target: "sidecar", arch: "amd64", root }), /Unknown container target/); + assert.throws(() => trimContainerNodeModules({ target: "core", arch: "riscv64", root }), /Unknown container architecture/); +});