diff --git a/.github/workflows/dev-stack.yml b/.github/workflows/dev-stack.yml new file mode 100644 index 000000000..825b970d1 --- /dev/null +++ b/.github/workflows/dev-stack.yml @@ -0,0 +1,340 @@ +name: Dev stack from source + +on: + pull_request: + paths: + - ".github/workflows/dev-stack.yml" + - ".nvmrc" + - "Dockerfile" + - "package.json" + - "package-lock.json" + - "CONTRIBUTING.md" + - "apps/api/**" + - "apps/web/**" + - "apps/worker/**" + - "packages/**" + - "deploy/docker/**" + workflow_dispatch: + +permissions: + contents: read + +concurrency: + group: dev-stack-${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + source-stack-linux: + name: Real source stack lifecycle (Linux) + runs-on: ubuntu-latest + timeout-minutes: 35 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version-file: .nvmrc + cache: npm + + - name: Install dependencies + env: + ELECTRON_SKIP_BINARY_DOWNLOAD: "1" + run: npm ci + + - name: Unit-test dev-stack contracts + run: npm run test:dev-stack + + - name: Start full stack without a TTY + shell: bash + run: | + set -euo pipefail + rm -f "$RUNNER_TEMP/dev-stack-initial.exit" + ( + set +e + npm run dev:stack >"$RUNNER_TEMP/dev-stack.log" 2>&1 + code=$? + printf '%s\n' "$code" >"$RUNNER_TEMP/dev-stack-initial.exit.tmp" + mv "$RUNNER_TEMP/dev-stack-initial.exit.tmp" "$RUNNER_TEMP/dev-stack-initial.exit" + exit "$code" + ) & + echo $! >"$RUNNER_TEMP/dev-stack-shell.pid" + echo "source stack launched in background" + + - name: Verify readiness, owner setup, and OpenCode pane + run: npm run verify:dev-stack + + - name: Verify a running shared-package rebuild + shell: bash + timeout-minutes: 10 + run: | + set -euo pipefail + probe="dev_stack_rebuild_${GITHUB_RUN_ID}_${GITHUB_RUN_ATTEMPT}" + before_pid="$(node -e 'const fs=require("node:fs"); process.stdout.write(String(JSON.parse(fs.readFileSync(".dev-stack/supervisor.pid","utf8")).pid))')" + export SPACE_DEV_EXPECTED_PACKAGE_PROBE="$probe" + echo "SPACE_DEV_EXPECTED_PACKAGE_PROBE=$probe" >> "$GITHUB_ENV" + git diff --exit-code -- packages/contracts/src/index.ts + node --input-type=module <<'NODE' + import { mkdtemp } from "node:fs/promises"; + import { join } from "node:path"; + import { editSource } from "./deploy/docker/dev-stack-hotreload.mjs"; + const backupRoot = await mkdtemp(join(process.env.RUNNER_TEMP, "dev-stack-source-backup-")); + const receiptRoot = join(process.env.RUNNER_TEMP, "dev-stack-source-edit-receipts"); + await editSource("packages/contracts/src/index.ts", source => + source + `\nexport const __devStackRebuildProbe = ${JSON.stringify(process.env.SPACE_DEV_EXPECTED_PACKAGE_PROBE)};\n`, + backupRoot, process.cwd(), receiptRoot); + NODE + api_url="$(node -e 'const fs=require("node:fs"); process.stdout.write(`http://127.0.0.1:${JSON.parse(fs.readFileSync(".dev-stack/config.json","utf8")).ports.api}`)')" + + npm run dev:stack -- rebuild + + rebuilt=0 + for attempt in $(seq 1 240); do + ready=0 + if curl -fsS "$api_url/readyz" | node -e ' + let input=""; + process.stdin.on("data", (chunk) => { input += chunk; }); + process.stdin.on("end", () => { + try { + const body = JSON.parse(input); + process.exit(body.ok === true && body.dependencies?.database === "RUNNING" && body.dependencies?.worker === "RUNNING" && body.dependencies?.cliHost === "RUNNING" ? 0 : 1); + } catch { process.exit(1); } + }); + '; then + ready=1 + fi + if grep -Fq "$probe" packages/contracts/dist/index.js && [ "$ready" -eq 1 ] && grep -Fq "Rebuild complete; source services are ready." "$RUNNER_TEMP/dev-stack.log"; then + rebuilt=1 + break + fi + sleep 1 + done + + if [ "$rebuilt" -ne 1 ]; then + echo "shared-package rebuild did not become observable in the running stack" >&2 + tail -200 "$RUNNER_TEMP/dev-stack.log" >&2 || true + exit 1 + fi + after_pid="$(node -e 'const fs=require("node:fs"); process.stdout.write(String(JSON.parse(fs.readFileSync(".dev-stack/supervisor.pid","utf8")).pid))')" + test "$before_pid" = "$after_pid" + echo "shared-package rebuild was picked up by supervisor $after_pid" + + - name: Verify API watch and real browser hot reload + timeout-minutes: 6 + env: + SPACE_DEV_ACCEPTANCE_MUTATIONS: "1" + run: node deploy/docker/dev-stack-hotreload.mjs --repo "$GITHUB_WORKSPACE" + + - name: Verify worker failure and explicit normal restart + timeout-minutes: 10 + env: + SPACE_DEV_ACCEPTANCE_MUTATIONS: "1" + SPACE_DEV_INITIAL_EXIT_FILE: ${{ runner.temp }}/dev-stack-initial.exit + run: node deploy/docker/dev-stack-worker-probe.mjs --repo "$GITHUB_WORKSPACE" + + - name: Stop and reset owned resources + if: always() + shell: bash + run: | + set +e + node --input-type=module <<'NODE' + import { existsSync, readFileSync, writeFileSync } from "node:fs"; + import { join } from "node:path"; + import { resolveConfig } from "./deploy/docker/dev-stack-lib.mjs"; + const config = await resolveConfig(); + const records = existsSync(config.processFile) ? JSON.parse(readFileSync(config.processFile, "utf8")).groups : []; + writeFileSync(join(process.env.RUNNER_TEMP, "dev-stack-cleanup-inventory.json"), JSON.stringify({ + stateRoot: config.stateRoot, socketPath: config.socketPath, groups: records.map(record => record.pgid) + }), { mode: 0o600 }); + NODE + inventory_status=$? + npm run dev:stack -- down + down_status=$? + npm run dev:stack -- reset + reset_status=$? + + process_status=0 + for pid_file in dev-stack-shell.pid dev-stack-recovery-shell.pid; do + shell_pid="$(cat "$RUNNER_TEMP/$pid_file" 2>/dev/null || true)" + attempt=0 + while [ "$attempt" -lt 50 ] && [ -n "$shell_pid" ]; do + state="$(ps -p "$shell_pid" -o stat= 2>/dev/null | xargs)" + if [ -z "$state" ] || [[ "$state" = Z* ]]; then break; fi + attempt=$((attempt + 1)) + sleep 0.1 + done + if [ -n "$shell_pid" ] && [ -n "$state" ] && [[ "$state" != Z* ]]; then + echo "dev-stack launcher $shell_pid is still alive after reset" >&2 + process_status=1 + fi + done + + npm run --silent dev:stack -- status >"$RUNNER_TEMP/dev-stack-final-status.json" + status_status=$? + cat "$RUNNER_TEMP/dev-stack-final-status.json" + node --input-type=module <<'NODE' + import assert from "node:assert/strict"; + import { execFileSync } from "node:child_process"; + import { existsSync, readFileSync } from "node:fs"; + import { join } from "node:path"; + const inventory = JSON.parse(readFileSync(join(process.env.RUNNER_TEMP, "dev-stack-cleanup-inventory.json"), "utf8")); + const status = JSON.parse(readFileSync(join(process.env.RUNNER_TEMP, "dev-stack-final-status.json"), "utf8")); + assert.equal(status.running, false); + for (const type of ["containers", "volumes", "networks"]) assert.deepEqual(status.docker[type], []); + assert(!existsSync(inventory.stateRoot), "Owned state directory remains after reset."); + assert(!existsSync(inventory.socketPath), "Owned Unix socket remains after reset."); + const groups = new Set(inventory.groups); + for (const line of execFileSync("ps", ["-axo", "pid=,pgid=,stat="], { encoding: "utf8" }).trim().split(/\r?\n/)) { + const match = line.trim().match(/^(\d+)\s+(\d+)\s+(\S+)$/); + assert(match, "Could not parse final process inventory."); + assert(!groups.has(Number(match[2])) || match[3].startsWith("Z"), `Owned group ${match[2]} still contains live PID ${match[1]}.`); + } + console.log("PASS no owned processes, containers, volumes, networks, state or Unix socket remain"); + NODE + resource_status=$? + + # Recover all source mutations, including interrupted probe processes, + # after services stop so normal worker recovery need not rebuild again. + node --input-type=module <<'NODE' + import assert from "node:assert/strict"; + import { existsSync, readFileSync, readdirSync } from "node:fs"; + import { basename, dirname, join, resolve } from "node:path"; + import { restoreSourceEdit } from "./deploy/docker/dev-stack-hotreload.mjs"; + const receiptRoot = join(process.env.RUNNER_TEMP, "dev-stack-source-edit-receipts"); + const sources = ["apps/api/src/health-routes.ts", "apps/web/src/features/auth/LoginScreen.tsx", "packages/contracts/src/index.ts"]; + for (const name of existsSync(receiptRoot) ? readdirSync(receiptRoot) : []) { + const source = sources.find(candidate => `${candidate.replaceAll("/", "__")}.json` === name); + assert(source, "Unexpected source recovery receipt; refusing overwrite."); + const file = join(receiptRoot, name); + const receipt = JSON.parse(readFileSync(file, "utf8")); + assert.equal(receipt.receiptFile, file); + assert.equal(receipt.path, resolve(source)); + assert.equal(dirname(dirname(receipt.backup)), process.env.RUNNER_TEMP); + assert(basename(dirname(receipt.backup)).startsWith("dev-stack-source-backup-")); + assert.equal(basename(receipt.backup), source.replaceAll("/", "__")); + await restoreSourceEdit(receipt); + console.log(`PASS interrupted source edit restored sha256=${receipt.originalHash}`); + } + NODE + restore_status=$? + git diff --exit-code -- packages/contracts/src/index.ts apps/api/src/health-routes.ts apps/web/src/features/auth/LoginScreen.tsx + diff_status=$? + echo "--- dev-stack log ---" + cat "$RUNNER_TEMP/dev-stack.log" 2>/dev/null || true + for result in "$inventory_status" "$down_status" "$reset_status" "$process_status" "$status_status" "$resource_status" "$restore_status" "$diff_status"; do + if [ "$result" -ne 0 ]; then exit 1; fi + done + + - name: Preserve limited acceptance evidence + if: always() + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: dev-stack-linux-acceptance + if-no-files-found: ignore + retention-days: 14 + path: | + ${{ runner.temp }}/dev-stack-acceptance-evidence/hmr-*.png + ${{ runner.temp }}/dev-stack-acceptance-evidence/*-receipt.json + + source-stack-macos: + name: Real source stack lifecycle (macOS, Colima) + runs-on: macos-15-intel + timeout-minutes: 90 + steps: + - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + persist-credentials: false + + - uses: actions/setup-node@49933ea5288caeca8642d1e84afbd3f7d6820020 # v4.4.0 + with: + node-version-file: .nvmrc + cache: npm + + - name: Install Docker CLI and Colima + env: + HOMEBREW_NO_AUTO_UPDATE: "1" + HOMEBREW_NO_INSTALL_CLEANUP: "1" + run: | + brew install colima docker docker-compose + mkdir -p "$HOME/.docker/cli-plugins" + ln -sfn "$(brew --prefix docker-compose)/bin/docker-compose" "$HOME/.docker/cli-plugins/docker-compose" + + - name: Start Colima + timeout-minutes: 25 + run: | + colima start --cpu 2 --memory 6 + docker version + docker compose version + + - name: Install dependencies + env: + ELECTRON_SKIP_BINARY_DOWNLOAD: "1" + run: npm ci + + - name: Unit-test dev-stack contracts + run: npm run test:dev-stack + + - name: Start full stack without a TTY + shell: bash + run: | + set -euo pipefail + npm run dev:stack >"$RUNNER_TEMP/dev-stack.log" 2>&1 & + echo $! >"$RUNNER_TEMP/dev-stack-shell.pid" + echo "source stack launched in background" + + - name: Verify readiness, owner setup, and OpenCode pane + env: + # Cold Colima/image startup plus one full source build can exceed 15m. + # Keep fresh-checkout startup under test; readiness assertions stay intact. + SPACE_DEV_VERIFY_READY_TIMEOUT_MS: "1800000" + SPACE_DEV_VERIFY_CLI_TIMEOUT_MS: "300000" + run: npm run verify:dev-stack + + - name: Stop and reset owned resources + if: always() + shell: bash + run: | + set +e + npm run dev:stack -- down + down_status=$? + npm run dev:stack -- reset + reset_status=$? + + shell_pid="$(cat "$RUNNER_TEMP/dev-stack-shell.pid" 2>/dev/null || true)" + attempt=0 + while [ "$attempt" -lt 50 ]; do + if [ -z "$shell_pid" ] || ! kill -0 "$shell_pid" 2>/dev/null; then + break + fi + attempt=$((attempt + 1)) + sleep 0.1 + done + process_status=0 + if [ -n "$shell_pid" ] && kill -0 "$shell_pid" 2>/dev/null; then + echo "dev-stack launcher process $shell_pid is still alive after reset" >&2 + process_status=1 + kill "$shell_pid" 2>/dev/null || true + fi + + status_json="$(npm run --silent dev:stack -- status 2>/dev/null)" + echo "$status_json" + if echo "$status_json" | grep -q '"running": true'; then + echo "dev-stack supervisor still reports running after reset" >&2 + process_status=1 + fi + if ! echo "$status_json" | grep -q '"containers": \[\]'; then + echo "dev-stack containers remain after reset" >&2 + process_status=1 + fi + if ! echo "$status_json" | grep -q '"volumes": \[\]'; then + echo "dev-stack volumes remain after reset" >&2 + process_status=1 + fi + + echo "--- dev-stack log ---" + cat "$RUNNER_TEMP/dev-stack.log" 2>/dev/null || true + colima stop >/dev/null 2>&1 || true + if [ "$down_status" -ne 0 ] || [ "$reset_status" -ne 0 ] || [ "$process_status" -ne 0 ]; then + exit 1 + fi diff --git a/.gitignore b/.gitignore index 81e68b26a..604f0123c 100644 --- a/.gitignore +++ b/.gitignore @@ -37,6 +37,8 @@ bin/ var/ projects/ /lost+found/ +.dev-stack/ +.dev-stack*/ # Local credentials and machine-specific environment. .env diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 03bcdad05..3c9b71062 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -50,6 +50,108 @@ select a short temporary directory you own and set `TMPDIR` for the command. On Windows run this particular smoke check inside WSL or a Linux development container. It does not certify the Windows/macOS installer. +## Run the complete app from source + +With Docker Engine or Docker Desktop available, start the contributor stack: + +```sh +npm run dev:stack +``` + +The command starts digest-pinned PostgreSQL with pgvector and Temporal in an +isolated Compose project, builds the workspace before migrations on a fresh +checkout, installs the `opencode-ai` version pinned by `Dockerfile` into local +state, and launches the CLI host, worker, API and Vite web server from this +checkout. The API uses normal single-owner setup: `SPACE_DEV_LOGIN` stays off. +When `/readyz` reports the database, worker and CLI host as running, the command +prints the local Web URL. It queries the API's owner setup status and prints the +one-time setup token **only** if an owner has not been claimed and that token is +still valid. On an ordinary restart after owner creation, it prints a sign-in +instruction instead; an expired or unavailable setup token is not displayed. +If the API setup status cannot be verified, startup fails rather than logging a +possibly consumed token. Do not use `reset` merely to recover a setup token: +`reset` deletes this checkout's persistent dev data. + +The default source-stack ports are deliberately separate from an installed +SpaceApp: API `5910`, Web `5911`, PostgreSQL `55432`, and Temporal `57233`. +Docker-published ports bind to `127.0.0.1`. Runtime state and generated secrets +live under the git-ignored `.dev-stack/`; secret files are mode `0600`. Provider +API keys and GitHub tokens inherited by the shell are not passed to source +services. + +Useful lifecycle commands are: + +```sh +npm run dev:stack -- status +npm run dev:stack -- rebuild +npm run dev:stack -- down +npm run dev:stack -- reset +``` + +`rebuild` rebuilds workspace packages and restarts the source services under the +same supervisor. `down` stops only this checkout's supervised processes and +Compose project. `reset` additionally removes this checkout's Compose volumes +and state directory; if Docker cannot remove an owned volume, reset fails and +keeps the state needed to retry cleanup instead of reporting success. Lifecycle +commands verify the recorded supervisor start identity and checkout hash before +sending a signal, so a stale PID file cannot target a reused unrelated PID. + +The isolated ports can be overridden with `SPACE_DEV_API_PORT`, +`SPACE_DEV_WEB_PORT`, `SPACE_DEV_POSTGRES_PORT`, and +`SPACE_DEV_TEMPORAL_PORT`. The state directory and Compose project identity are +derived from this checkout and intentionally cannot be overridden, so lifecycle +commands cannot target another checkout's resources. The runner also rejects +installed-instance ports `4910` and `4911`, duplicate ports, unsafe state roots, +and symlink state roots. + +For an automated end-to-end proof after the stack is ready, run: + +```sh +npm run verify:dev-stack +``` + +That proof performs normal owner setup, points the Getting Started room at the +current checkout rather than `/opt/spaceapp`, starts its OpenCode terminal pane +through the real CLI host, attaches the same terminal WebSocket transport used +by the web UI, requires a running CLI session plus real terminal output, and +verifies that the stack remains ready. The dedicated GitHub Actions workflow +runs this lifecycle on Linux and on an Intel macOS runner using Colima. Windows +11 contributors should use WSL2 with Docker Desktop's WSL integration or a +Docker engine running inside WSL2; that path follows the Linux host model. +These host paths are contributor environments, not installer-release +certification. + +The Linux lifecycle job also edits and restores source files in its disposable +checkout: it checks API watch restarts, imports a rebuilt package marker through +the running API, and verifies Vite updates a real Chrome page without navigation +or losing form state. It then stops the owned worker, requires a clear failure +and nonzero supervisor exit, and exercises an explicit normal restart with the +same owner data. These mutation probes are CI-only and are not needed for normal +contributor startup. Their screenshots include only the anonymous login heading; +state directories, credentials, browser profiles and source backups are not +uploaded. The final cleanup restores original source bytes and checks owned +processes, containers, volumes, networks, state and the Unix socket are gone. + +### Troubleshooting the source stack + +- If Docker is unavailable, run `docker compose version` first and start Docker + Desktop, Colima, or the Docker daemon before `npm run dev:stack`. +- If a default dev port is occupied, set the corresponding + `SPACE_DEV_*_PORT` variable rather than reusing installed ports `4910/4911`. +- If a package change is not reflected in a running source service, run + `npm run dev:stack -- rebuild`; Vite handles `apps/web/src` edits directly + and the API runs under `tsx watch`. +- If startup is interrupted, `npm run dev:stack -- down` stops the owned + supervisor and Compose project. Use `npm run dev:stack -- reset` for a clean + database and state directory; a failed volume removal is reported and leaves + the state in place so cleanup can be retried safely. +- On macOS, `npm run dev:stack` repairs the execute bits on node-pty's published + `prebuilds/darwin-*/spawn-helper` before the CLI host starts. If you launch + the CLI host outside this workflow and a terminal remains `IDLE` with an + `EACCES` spawn error, repair that helper's execute permission first. +- Unix socket paths are automatically shortened under the system temporary + directory when `.dev-stack` would exceed the platform socket-path limit. + ## Make and verify a small change Work on a branch, not on `main`, so the change can become a pull request: diff --git a/apps/api/src/agent-bootstrap.ts b/apps/api/src/agent-bootstrap.ts index 20cb69a9c..9ca7e2bc4 100644 --- a/apps/api/src/agent-bootstrap.ts +++ b/apps/api/src/agent-bootstrap.ts @@ -1,4 +1,4 @@ -import { spaceAgentCorePath, spaceMemoryIndexPath, spaceMemoryMonthlyPath } from "@space/runtime"; +import { SPACE_MEMORY_DIRECTORY, spaceAgentCorePath, spaceMemoryDirectory, spaceMemoryIndexPath, spaceMemoryMonthlyPath } from "@space/runtime"; export interface SpaceAgentBootstrapInput { channel: "WEB_CHAT" | "CLI"; @@ -74,6 +74,8 @@ function cliAgentFilesLine(channel: SpaceAgentBootstrapInput["channel"]): string export function buildSpaceAgentBootstrapPrompt(input: SpaceAgentBootstrapInput): string { const memoryMonth = input.currentMemoryMonth ?? currentMemoryMonth(); + const memoryDirectory = spaceMemoryDirectory(); + const installedMemory = memoryDirectory === SPACE_MEMORY_DIRECTORY; const browserState = input.browserToolsEnabled ? `${input.activeBrowserSessionCount ?? 0} active managed browser session(s) are visible in this room.` : "Managed browser tools are disabled until Space browser runtime gates pass."; @@ -88,17 +90,17 @@ export function buildSpaceAgentBootstrapPrompt(input: SpaceAgentBootstrapInput): "Space Agent Bootstrap:", `- Channel: ${input.channel}. Match the main Codex /etc VS Code session rules where the runtime exposes them.`, ...identityLines(input), - `- Read the compact ${spaceAgentCorePath()} core once. Load only task-relevant domain rules; read /opt/spaceapp/docs/agent_local_workspace_hygiene.md before local project/dev-server/cleanup work.`, + `- Read the compact ${spaceAgentCorePath()} core once. Load only task-relevant domain rules; ${installedMemory ? "read /opt/spaceapp/docs/agent_local_workspace_hygiene.md" : "follow the current source checkout's CONTRIBUTING.md"} before local project/dev-server/cleanup work.`, `- Canonical Space Memory: ${spaceMemoryIndexPath()} plus ${spaceMemoryMonthlyPath(memoryMonth)}.`, - "- Memory parity: read/search/save through the Space Memory bridge (/opt/spaceapp/bin/space-memory-recall, /opt/spaceapp/bin/space-memory-save) or Space API memory endpoints. Treat Space Memory in /opt/spaceapp/docs as canonical operational memory; Codex ad-hoc notes are fallback only.", + `- Memory parity: ${installedMemory ? "read/search/save through the Space Memory bridge (/opt/spaceapp/bin/space-memory-recall, /opt/spaceapp/bin/space-memory-save) or Space API memory endpoints" : "read/search/save directly under the configured source-stack memory root with native filesystem tools, or through authenticated Space API memory endpoints when available; do not use an installed-instance memory bridge"}. Treat Space Memory in ${memoryDirectory} as canonical operational memory; Codex ad-hoc notes are fallback only.`, "- Tools: choose verified native tools first. MCP is on-demand fallback or Space-specific access, never a mandatory substitute for a capable native tool. Keep evidence and authorization requirements regardless of transport.", "- Images: use native vision whenever available. Vision MCP is only for verified text-only input pipelines; unknown model support requires checking capability metadata first.", "- Skills: load only relevant guidance when needed. Workflow skills require operator opt-in; a skill must not force an inferior tool. Do not list every skill or tool at startup.", input.channel === "CLI" ? "- Space-only operations: use space-capability info [tool] or space-capability call with JSON on stdin when needed. No MCP catalog or server startup is required for native tasks." : null, agentFilesLine, - input.channel === "CLI" ? "- Timed or recurring work: `node /opt/spaceapp/scripts/space-task.mjs add --title T --prompt P --every 2h|--daily HH:MM|--at