diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 3ff54d83d0c..af6ed7559b4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -22,36 +22,52 @@ jobs: changes: runs-on: ${{ vars.RUNNER_DEFAULT_LABEL || 'ubuntu-latest' }} outputs: + kilocode_backend: ${{ steps.filter.outputs.kilocode_backend }} cloud_agent: ${{ steps.filter.outputs.cloud_agent }} cloud_agent_next: ${{ steps.filter.outputs.cloud_agent_next }} - webhook_agent: ${{ steps.filter.outputs.webhook_agent }} - security_auto_analysis: ${{ steps.filter.outputs.security_auto_analysis }} - kiloclaw: ${{ steps.filter.outputs.kiloclaw }} - app_builder: ${{ steps.filter.outputs.app_builder }} + workspace_matrix: ${{ steps.workspaces.outputs.matrix }} steps: - uses: useblacksmith/checkout@v1 with: fetch-depth: 0 + - name: Setup Node + uses: actions/setup-node@v4 + with: + node-version: 22 + - name: Detect changes id: filter uses: dorny/paths-filter@v3 with: filters: | + kilocode_backend: + - 'src/**' + - 'packages/db/**' + - 'packages/encryption/**' + - 'packages/trpc/**' + - 'packages/worker-utils/**' + - 'package.json' + - 'pnpm-lock.yaml' + - 'tsconfig.json' + - 'tsconfig.*.json' + - 'next.config.mjs' + - 'jest.config.ts' + - 'postcss.config.mjs' + - 'sentry.*' cloud_agent: - 'cloud-agent/**' cloud_agent_next: - 'cloud-agent-next/**' - webhook_agent: - - 'cloudflare-webhook-agent-ingest/**' - security_auto_analysis: - - 'cloudflare-security-auto-analysis/**' - kiloclaw: - - 'kiloclaw/**' - app_builder: - - 'cloudflare-app-builder/**' + + - name: Detect changed workspaces with tests + id: workspaces + run: | + matrix=$(scripts/changed-workspaces.sh --exclude cloud-agent --exclude cloud-agent-next) + echo "matrix=$matrix" >> "$GITHUB_OUTPUT" typecheck: + needs: changes runs-on: ${{ vars.RUNNER_DEFAULT_LABEL || 'ubuntu-latest' }} steps: - uses: useblacksmith/checkout@v1 @@ -74,9 +90,10 @@ jobs: run: pnpm install --frozen-lockfile - name: Typecheck - run: pnpm run typecheck + run: scripts/typecheck-all.sh lint: + needs: changes runs-on: ${{ vars.RUNNER_DEFAULT_LABEL || 'ubuntu-latest' }} steps: - uses: useblacksmith/checkout@v1 @@ -99,9 +116,10 @@ jobs: run: pnpm install --frozen-lockfile - name: Lint - run: pnpm run lint + run: scripts/lint-all.sh format-check: + needs: changes runs-on: ${{ vars.RUNNER_DEFAULT_LABEL || 'ubuntu-latest' }} steps: - uses: useblacksmith/checkout@v1 @@ -131,6 +149,7 @@ jobs: } drizzle-check: + needs: changes runs-on: ${{ vars.RUNNER_DEFAULT_LABEL || 'ubuntu-latest' }} steps: - uses: useblacksmith/checkout@v1 @@ -157,32 +176,9 @@ jobs: env: POSTGRES_URL: postgresql://unused:unused@localhost:5432/unused - dependency-cycle-check: - runs-on: ${{ vars.RUNNER_DEFAULT_LABEL || 'ubuntu-latest' }} - steps: - - uses: useblacksmith/checkout@v1 - with: - lfs: true - - - name: Setup pnpm - uses: pnpm/action-setup@v2 - with: - version: latest - run_install: false - - - name: Setup Node - uses: actions/setup-node@v4 - with: - node-version: 22 - cache: 'pnpm' - - - name: Install dependencies - run: pnpm install --frozen-lockfile - - - name: Dependency cycle check - run: pnpm run dependency-cycle-check - test: + needs: [changes, typecheck, lint, format-check, drizzle-check] + if: needs.changes.outputs.kilocode_backend == 'true' runs-on: ${{ vars.RUNNER_LARGE_LABEL || 'ubuntu-24.04-8core' }} services: @@ -225,10 +221,15 @@ jobs: - name: Install dependencies run: pnpm install --frozen-lockfile + - name: Dependency cycle check + run: pnpm run dependency-cycle-check + - name: Run tests run: pnpm run test build: + needs: [changes, typecheck, lint, format-check, drizzle-check] + if: needs.changes.outputs.kilocode_backend == 'true' runs-on: ${{ vars.RUNNER_LARGE_LABEL || 'ubuntu-24.04-8core' }} steps: - uses: useblacksmith/checkout@v1 @@ -274,7 +275,7 @@ jobs: run: pnpm run build cloud-agent: - needs: changes + needs: [changes, typecheck, lint, format-check, drizzle-check] if: needs.changes.outputs.cloud_agent == 'true' runs-on: ${{ vars.RUNNER_LARGE_LABEL || 'ubuntu-24.04-8core' }} steps: @@ -306,14 +307,11 @@ jobs: working-directory: cloud-agent/wrapper run: bun run build.ts - - name: Typecheck (cloud-agent) - run: pnpm --filter cloud-agent typecheck - - name: Run cloud-agent tests run: pnpm --filter cloud-agent test:all cloud-agent-next: - needs: changes + needs: [changes, typecheck, lint, format-check, drizzle-check] if: needs.changes.outputs.cloud_agent_next == 'true' runs-on: ${{ vars.RUNNER_LARGE_LABEL || 'ubuntu-24.04-8core' }} steps: @@ -346,150 +344,17 @@ jobs: working-directory: cloud-agent-next/wrapper run: bun run build.ts - - name: Typecheck (cloud-agent-next) - run: pnpm --filter cloud-agent-next typecheck - - name: Run cloud-agent-next tests run: pnpm --filter cloud-agent-next test:all - webhook-agent: - needs: changes - if: needs.changes.outputs.webhook_agent == 'true' - runs-on: ${{ vars.RUNNER_DEFAULT_LABEL || 'ubuntu-latest' }} - steps: - - uses: useblacksmith/checkout@v1 - with: - lfs: true - - - name: Setup pnpm - uses: pnpm/action-setup@v2 - with: - version: latest - run_install: false - - - name: Setup Node - uses: actions/setup-node@v4 - with: - node-version: 22 - cache: 'pnpm' - - - name: Install dependencies - run: pnpm install --frozen-lockfile - - - name: Typecheck (webhook-agent-ingest) - run: pnpm --filter cloudflare-webhook-agent-ingest typecheck - - - name: Run webhook-agent-ingest tests - run: pnpm --filter cloudflare-webhook-agent-ingest test - - security-auto-analysis: - needs: changes - if: needs.changes.outputs.security_auto_analysis == 'true' - runs-on: ${{ vars.RUNNER_DEFAULT_LABEL || 'ubuntu-latest' }} - steps: - - uses: useblacksmith/checkout@v1 - with: - lfs: true - - - name: Setup pnpm - uses: pnpm/action-setup@v2 - with: - version: latest - run_install: false - - - name: Setup Node - uses: actions/setup-node@v4 - with: - node-version: 22 - cache: 'pnpm' - - - name: Install dependencies - run: pnpm install --frozen-lockfile - - - name: Typecheck (security-auto-analysis) - run: pnpm --filter cloudflare-security-auto-analysis typecheck - - - name: Run security-auto-analysis tests - run: pnpm --filter cloudflare-security-auto-analysis test - - kiloclaw: - needs: changes - if: needs.changes.outputs.kiloclaw == 'true' - runs-on: ${{ vars.RUNNER_DEFAULT_LABEL || 'ubuntu-latest' }} - steps: - - uses: useblacksmith/checkout@v1 - with: - lfs: true - - - name: Setup pnpm - uses: pnpm/action-setup@v2 - with: - version: latest - run_install: false - - - name: Setup Node - uses: actions/setup-node@v4 - with: - node-version: 22 - cache: 'pnpm' - - - name: Install dependencies - run: pnpm install --frozen-lockfile - - - name: Typecheck (kiloclaw) - run: pnpm --filter kiloclaw typecheck - - - name: Run kiloclaw tests - run: pnpm --filter kiloclaw test - - - name: Typecheck (secret-catalog) - run: pnpm --filter @kilocode/kiloclaw-secret-catalog typecheck - - - name: Run secret-catalog tests - run: pnpm --filter @kilocode/kiloclaw-secret-catalog test - - app-builder: - needs: changes - if: needs.changes.outputs.app_builder == 'true' - runs-on: ${{ vars.RUNNER_DEFAULT_LABEL || 'ubuntu-latest' }} - steps: - - uses: useblacksmith/checkout@v1 - with: - lfs: true - - - name: Setup pnpm - uses: pnpm/action-setup@v2 - with: - version: latest - run_install: false - - - name: Setup Node - uses: actions/setup-node@v4 - with: - node-version: 22 - cache: 'pnpm' - - - name: Install dependencies - run: pnpm install --frozen-lockfile - - - name: Typecheck (app-builder) - run: pnpm --filter app-builder typecheck - - - name: Run app-builder tests - run: pnpm --filter app-builder test - - cloudflare: + workspace-tests: + needs: [changes, typecheck, lint, format-check, drizzle-check] + if: ${{ needs.changes.outputs.workspace_matrix != '[]' }} strategy: fail-fast: false matrix: - include: - - name: deploy-builder - filter: kilo-deploy-builder - - name: deploy-dispatcher - filter: deploy-dispatcher - - name: code-review - filter: kilo-code-review-worker - name: cloudflare-${{ matrix.name }} + workspace: ${{ fromJson(needs.changes.outputs.workspace_matrix) }} + name: test (${{ matrix.workspace.name }}) runs-on: ${{ vars.RUNNER_DEFAULT_LABEL || 'ubuntu-latest' }} steps: - uses: useblacksmith/checkout@v1 @@ -511,5 +376,5 @@ jobs: - name: Install dependencies run: pnpm install --frozen-lockfile - - name: Typecheck - run: pnpm --filter ${{ matrix.filter }} typecheck + - name: Run tests + run: pnpm --filter ${{ matrix.workspace.name }} test diff --git a/package.json b/package.json index af58838199d..45ca61abb5f 100644 --- a/package.json +++ b/package.json @@ -16,7 +16,7 @@ "format:check": "oxfmt --list-different .", "format:changed": "git diff --name-only $(git merge-base origin/main HEAD) --diff-filter=ACMR -- '**/*.js' '**/*.jsx' '**/*.ts' '**/*.tsx' '**/*.json' '**/*.css' '**/*.md' | xargs -r oxfmt --no-error-on-unmatched-pattern", "test": "NODE_ENV=test jest", - "typecheck": "pnpm --filter @kilocode/trpc run build && tsgo --noEmit && pnpm -r --filter '!kilocode-backend' --filter '!@kilocode/trpc' run typecheck", + "typecheck": "pnpm --filter @kilocode/trpc run build && tsgo --noEmit", "dependency-cycle-check": "madge --circular --warning --extensions ts,tsx --ts-config tsconfig.json --basedir . src", "validate": "pnpm run typecheck && pnpm run lint && pnpm run format:changed && pnpm run test && pnpm run dependency-cycle-check", "prepare": "husky", diff --git a/scripts/changed-workspaces.sh b/scripts/changed-workspaces.sh new file mode 100755 index 00000000000..9a16a1015f9 --- /dev/null +++ b/scripts/changed-workspaces.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash +set -euo pipefail + +# Output a JSON matrix of workspace packages that have file changes and a test script. +# Excludes the root package and packages listed in --exclude arguments. +# +# Usage: +# scripts/changed-workspaces.sh # all changed workspaces with tests +# scripts/changed-workspaces.sh --exclude cloud-agent --exclude cloud-agent-next # skip specific dirs + +excludes=() +while [[ $# -gt 0 ]]; do + case "$1" in + --exclude) + if [[ $# -lt 2 ]]; then + echo "Error: --exclude requires a value" >&2; exit 1 + fi + excludes+=("$2"); shift 2 ;; + *) echo "Unknown option: $1" >&2; exit 1 ;; + esac +done + +base=$(git merge-base origin/main HEAD 2>/dev/null || true) + +# If shared inputs changed, treat all workspaces as changed — shared package or +# lockfile updates can break any downstream workspace. +shared_changed=false +if [ -n "$base" ]; then + if git diff --name-only "$base" -- pnpm-lock.yaml pnpm-workspace.yaml 'packages/**' | grep -q .; then + shared_changed=true + fi +fi + +# Read workspace dirs from pnpm-workspace.yaml +workspace_dirs=$(grep '^ *- ' pnpm-workspace.yaml | sed 's/^ *- //') + +# Collect entries as newline-delimited "name\tdir" pairs, then serialize to JSON once +entries="" +for dir in $workspace_dirs; do + # Skip excluded dirs + skip=false + for ex in "${excludes[@]+"${excludes[@]}"}"; do + if [[ "$dir" == "$ex" || "$dir" == "$ex/"* ]]; then + skip=true + break + fi + done + $skip && continue + + # Must have a package.json with a test script + [ -f "$dir/package.json" ] || continue + has_test=$(node -e "const p=require('./$dir/package.json'); console.log(p.scripts?.test ? '1' : '')" 2>/dev/null) + [ -n "$has_test" ] || continue + + # Check for file changes (if we have a merge base) + if [ -n "$base" ] && ! $shared_changed; then + changed_file=$(git diff --name-only "$base" -- "$dir/" | head -1 || true) + [ -n "$changed_file" ] || continue + fi + + name=$(node -e "console.log(require('./$dir/package.json').name)" 2>/dev/null) + entries+="${name}"$'\t'"${dir}"$'\n' +done + +# Serialize all entries to JSON in a single node invocation (avoids shell interpolation issues) +if [ -z "$entries" ]; then + echo "[]" +else + echo -n "$entries" | node -e " + const lines = require('fs').readFileSync('/dev/stdin','utf8').trim().split('\n'); + const matrix = lines.map(line => { + const [name, dir] = line.split('\t'); + return { name, dir }; + }); + console.log(JSON.stringify(matrix)); + " +fi