python-tests.yml

name: Python Tests

on:
  pull_request:
    types: [opened, synchronize, reopened, ready_for_review, labeled, unlabeled]
  workflow_dispatch:
  push:
    branches:
      - main
    paths:
      - ".github/workflows/python-tests.yml"
      - ".github/workflows/sonarcloud.yml"
      - "sonar-project.properties"
      - "apps/**"
      - "loopx/**"
      - "scripts/**"
      - "tests/**"
      - "examples/**"
      - "package.json"
      - "package-lock.json"
      - "pyproject.toml"
      - "tsconfig.control-plane.json"

permissions:
  contents: read

concurrency:
  group: python-tests-${{ github.ref }}
  cancel-in-progress: true

jobs:
  changes:
    runs-on: ubuntu-latest
    timeout-minutes: 3
    outputs:
      core_tests: ${{ steps.classify.outputs.core_tests }}
      change_kind: ${{ steps.classify.outputs.change_kind }}
      python_tests: ${{ steps.classify.outputs.python_tests }}
      stage2c_tests: ${{ steps.classify.outputs.stage2c_tests }}
      presentation_tests: ${{ steps.classify.outputs.presentation_tests }}
    steps:
      - uses: actions/checkout@v7
        with:
          fetch-depth: 0
      - uses: actions/setup-python@v6
        with:
          python-version: "3.11"
      - name: Validate merge gate semantics
        run: python -m unittest discover -s scripts/ci -p 'test_*.py'
      - name: Classify the exact pull-request change
        id: classify
        env:
          EVENT_NAME: ${{ github.event_name }}
          BASE_SHA: ${{ github.event.pull_request.base.sha }}
          HEAD_SHA: ${{ github.event.pull_request.head.sha }}
          CHECKOUT_SHA: ${{ github.sha }}
          FORCE_FULL: ${{ contains(github.event.pull_request.labels.*.name, 'ci:full') }}
        shell: bash
        run: |
          set -euo pipefail
          extra=()
          if [[ "$FORCE_FULL" == true ]]; then extra+=(--force-full); fi
          if [[ "$EVENT_NAME" == pull_request ]]; then
            python scripts/ci/review_gate.py classify --base "$BASE_SHA" --head "$HEAD_SHA" --plan impact-plan.json "${extra[@]}" >> "$GITHUB_OUTPUT"
          else
            python scripts/ci/review_gate.py classify --base "$CHECKOUT_SHA" --head "$CHECKOUT_SHA" --non-pr --plan impact-plan.json >> "$GITHUB_OUTPUT"
          fi
          python - <<'PY'
          import json, os
          from pathlib import Path
          plan = json.loads(Path("impact-plan.json").read_text())
          with open(os.environ["GITHUB_STEP_SUMMARY"], "a") as summary:
              summary.write("## CI job exemptions\n\n")
              summary.write(f"Change kind: **{plan['change_kind']}**; Python shards: **{plan['python_shards']}**.\n\n")
              summary.write(plan['reason'] + ".\n\n")
              summary.write(f"Python: {plan['python_tests']}; Stage2c: {plan['stage2c_tests']}; frontend: {plan['presentation_tests']}.\n\n")
              summary.write(plan["coverage_scope"] + ". Unknown/mixed changes and main run full. Label ci:full forces full qualification.\n")
          PY
      - uses: actions/upload-artifact@v7
        with:
          name: ci-impact-plan
          path: impact-plan.json
          if-no-files-found: error
          retention-days: 7

  checks:
    needs: changes
    if: needs.changes.outputs.core_tests == 'true'
    runs-on: ubuntu-latest
    timeout-minutes: 15
    steps:
      - name: Check out repository
        uses: actions/checkout@v7
        with:
          fetch-depth: 0

      - name: Set up Python
        uses: actions/setup-python@v6
        with:
          python-version: "3.11"
          cache: pip

      - name: Set up the TypeScript Effect runtime
        uses: actions/setup-node@v6
        with:
          node-version: "24"
          cache: npm
          cache-dependency-path: package-lock.json

      - name: Install test dependencies
        run: python -m pip install --disable-pip-version-check -e ".[test]"

      - name: Verify file authority on the core runtime
        run: >-
          node --no-warnings --experimental-strip-types --test
          tests/control_plane_ts/authority_store.test.ts
          tests/control_plane_ts/local_authority_runtime.test.ts
          tests/control_plane_ts/sqlite_runtime_admission.test.ts

      - name: Set up the qualified SQLite runtime
        uses: actions/setup-node@v6
        with:
          node-version: "22.14"

      - name: Qualify the TypeScript Effect core
        run: |
          npm ci --ignore-scripts
          npm run typecheck:control-plane
          npm run test:control-plane:coverage
          npm run test:dashboard:coverage

      - name: Measure dashboard browser interactions
        env:
          LOOPX_DASHBOARD_COVERAGE: "1"
          LOOPX_PLAYWRIGHT_PACKAGE: ${{ github.workspace }}/apps/presentation/dashboard/node_modules/playwright
        run: |
          npm ci --ignore-scripts --prefix apps/presentation/dashboard
          cd apps/presentation/dashboard
          npx playwright install --with-deps chromium --only-shell
          node ../../../examples/personal-workspace-browser-smoke.mjs

      - name: Upload TypeScript control-plane coverage
        uses: actions/upload-artifact@v7
        with:
          name: typescript-control-plane-coverage
          path: coverage/control-plane/lcov.info
          if-no-files-found: error
          retention-days: 3

      - name: Upload dashboard coverage
        uses: actions/upload-artifact@v7
        with:
          name: dashboard-coverage
          path: coverage/dashboard/*.info
          if-no-files-found: error
          retention-days: 3

      - name: Lint test suite
        run: >-
          python -m ruff check
          tests
          loopx/canary
          loopx/control_plane
          loopx/domain_packs
          loopx/presentation

      - name: Type-check kernel contracts
        run: python -m mypy

      - name: Qualify agent-facing CLI output
        env:
          LOOPX_CLI_OUTPUT_BASE_REF: origin/${{ github.event.pull_request.base.ref || 'main' }}
        run: python examples/control_plane/cli-output-budget-regression-smoke.py

  node-minimum-compatibility:
    needs: changes
    if: needs.changes.outputs.core_tests == 'true'
    runs-on: ubuntu-latest
    timeout-minutes: 10
    steps:
      - uses: actions/checkout@v7
      - uses: actions/setup-node@v6
        with:
          node-version: "22.6"
          cache: npm
          cache-dependency-path: package-lock.json
      - name: Qualify the public minimum Node.js runtime
        run: |
          npm ci --ignore-scripts
          npm run typecheck:control-plane
          # SQLite conformance requires qualified statement finalization (22.14+).
          # Retain admission rejection and all default file/runtime tests here.
          tests=()
          for test in tests/control_plane_ts/*.test.ts; do
            case "$test" in
              */sqlite_authority_store.test.ts|*/local_authority_provider.test.ts) continue ;;
            esac
            tests+=("$test")
          done
          node --no-warnings --experimental-sqlite --experimental-strip-types --test "${tests[@]}"

  node-forward-compatibility:
    needs: changes
    if: needs.changes.outputs.core_tests == 'true'
    continue-on-error: true
    runs-on: ubuntu-latest
    timeout-minutes: 10
    steps:
      - uses: actions/checkout@v7
      - uses: actions/setup-node@v6
        with:
          node-version: "26"
          cache: npm
          cache-dependency-path: package-lock.json
      - name: Probe the next Node.js runtime
        run: |
          npm ci --ignore-scripts
          npm run typecheck:control-plane
          npm run test:control-plane

  test-shard:
    needs: changes
    if: needs.changes.outputs.python_tests == 'true'
    runs-on: ubuntu-latest
    timeout-minutes: 30
    strategy:
      fail-fast: false
      matrix:
        shard: [1, 2, 3, 4]
    steps:
      - uses: actions/checkout@v7
        with:
          fetch-depth: 0
      - uses: actions/setup-python@v6
        with:
          python-version: "3.11"
          cache: pip
      - uses: actions/setup-node@v6
        with:
          node-version: "24"
          cache: npm
          cache-dependency-path: package-lock.json
      - name: Install test dependencies
        run: |
          python -m pip install --disable-pip-version-check -e ".[test]"
          npm ci --ignore-scripts
      - name: Run test shard
        # Split the whole collection, not a hand-maintained list of directories.
        # Without timing history least_duration alternates equal-weight tests.
        # Each runner retains the measured two-worker pool.
        run: >-
          python -m pytest -q -n 2 -m "not stage2c_e2e"
          --splits 4 --group ${{ matrix.shard }}
          --splitting-algorithm least_duration
          --durations=25 --durations-min=1
          --junitxml=junit.xml
          --cov=loopx
          --cov-report=term
      - name: Upload full-shard outcomes for selection audit
        if: always()
        uses: actions/upload-artifact@v7
        with:
          name: python-junit-${{ matrix.shard }}
          path: junit.xml
          if-no-files-found: error
          retention-days: 7
      - name: Upload shard coverage
        uses: actions/upload-artifact@v7
        with:
          name: python-coverage-${{ matrix.shard }}
          path: .coverage
          include-hidden-files: true
          if-no-files-found: error
          retention-days: 3

  pytest:
    # Keep the required check name; a skipped/failed shard must not turn it green.
    if: always() && needs.changes.outputs.python_tests == 'true'
    needs: [changes, checks, test-shard]
    runs-on: ubuntu-latest
    timeout-minutes: 10
    steps:
      - name: Require every upstream check
        env:
          CHECKS_RESULT: ${{ needs.checks.result }}
          SHARDS_RESULT: ${{ needs.test-shard.result }}
        run: |
          test "$CHECKS_RESULT" = success
          test "$SHARDS_RESULT" = success
      - uses: actions/checkout@v7
      - uses: actions/setup-python@v6
        with:
          python-version: "3.11"
          cache: pip
      - run: python -m pip install --disable-pip-version-check -e ".[test]"
      - uses: actions/download-artifact@v7
        with:
          pattern: python-coverage-*
          path: coverage-shards
      - name: Combine complete coverage and enforce the existing floor
        run: |
          shards=()
          for shard in 1 2 3 4; do
            path="coverage-shards/python-coverage-${shard}/.coverage"
            test -s "$path"
            shards+=("$path")
          done
          python -m coverage combine "${shards[@]}"
          python -m coverage report --fail-under=19.6
          python -m coverage xml -o coverage.xml
      - uses: actions/upload-artifact@v7
        with:
          name: python-coverage-xml
          path: coverage.xml
          if-no-files-found: error
          retention-days: 3

  sonar:
    needs: pytest
    uses: ./.github/workflows/sonarcloud.yml
    secrets:
      SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}

  stage2c-suite:
    needs: changes
    if: needs.changes.outputs.stage2c_tests == 'true'
    name: stage2c (${{ matrix.suite }} ${{ matrix.shard }})
    runs-on: ubuntu-latest
    timeout-minutes: 30
    strategy:
      fail-fast: false
      matrix:
        include:
          - {suite: e2e, shard: 1}
          - {suite: e2e, shard: 2}
          - {suite: mutants, shard: 0}
          - {suite: installed, shard: 0}
    steps:
      - uses: actions/checkout@v7
      - uses: actions/setup-python@v6
        with:
          python-version: "3.11"
          cache: pip
          cache-dependency-path: tests/requirements-stage2c-linux-py311.txt
      - uses: actions/setup-node@v6
        with:
          node-version: "24"
          cache: npm
      - name: Install locked test and package build tools
        run: |
          python -m pip install --disable-pip-version-check --require-hashes --only-binary=:all: -r tests/requirements-stage2c-linux-py311.txt
          npm ci --ignore-scripts
      - name: Build and verify the checked-out source package
        run: |
          python -m build --no-isolation --wheel --outdir .local/stage2c-source-install
          python - <<'PYTHON'
          import hashlib
          from pathlib import Path

          directory = Path(".local/stage2c-source-install").resolve()
          wheels = list(directory.glob("*.whl"))
          if len(wheels) != 1:
              raise SystemExit("Expected exactly one wheel from the checked-out source")
          wheel = wheels[0]
          digest = hashlib.sha256(wheel.read_bytes()).hexdigest()
          (directory / "requirements.txt").write_text(
              f"loopx @ {wheel.as_uri()} --hash=sha256:{digest}\n", encoding="utf-8"
          )
          PYTHON
          python -m pip install --disable-pip-version-check --require-hashes --no-deps --no-index -r .local/stage2c-source-install/requirements.txt
          python -m pip check
          # Remove the generated source copy before pytest's normal discovery.
          python -c "import shutil; shutil.rmtree('build')"
      - name: Qualify real CLI, mixed writers, process death, and recovery
        if: matrix.suite == 'e2e'
        env:
          LOOPX_SHADOW_COMPARISON_OUTPUT: .local/stage2c-observables
        # Keep each module's shared workspace and ordered parity rows on one worker.
        run: python -m pytest -q -n 2 --dist loadfile -m stage2c_e2e -p scripts.ci.module_shard --ci-module-shards 2 --ci-module-shard ${{ matrix.shard }} --durations=20 --junitxml=stage2c-e2e.xml
      - name: Reject deliberate correctness regressions
        if: matrix.suite == 'mutants'
        run: python examples/shared-goal-authority-e2e/mutants.py --output .local/stage2c-mutants
      - name: Build independently installed distributions
        if: matrix.suite == 'installed'
        run: python -m build --no-isolation
      - name: Qualify wheel outside the repository
        if: matrix.suite == 'installed'
        run: python examples/shared-goal-authority-e2e/installed.py --artifact dist/*.whl --report-json installed-wheel.json
      - name: Qualify sdist outside the repository
        if: matrix.suite == 'installed'
        run: python examples/shared-goal-authority-e2e/installed.py --artifact dist/*.tar.gz --report-json installed-sdist.json
      - name: Retain bounded acceptance evidence
        if: always()
        uses: actions/upload-artifact@v7
        with:
          name: stage2c-correctness-evidence-${{ matrix.suite }}-${{ matrix.shard }}
          include-hidden-files: true
          if-no-files-found: error
          path: |
            stage2c-e2e.xml
            .local/stage2c-observables/
            installed-wheel.json
            installed-sdist.json
            .local/stage2c-mutants/

  stage2c-correctness-e2e:
    # Preserve the public check name and reject failed, cancelled or skipped lanes.
    if: always() && needs.changes.outputs.stage2c_tests == 'true'
    needs: [changes, stage2c-suite]
    runs-on: ubuntu-latest
    timeout-minutes: 2
    steps:
      - name: Require every Stage 2C lane
        env:
          STAGE2C_RESULT: ${{ needs.stage2c-suite.result }}
        run: test "$STAGE2C_RESULT" = success

  windows-powershell:
    needs: changes
    if: needs.changes.outputs.python_tests == 'true'
    runs-on: windows-latest
    timeout-minutes: 20
    steps:
      - name: Check out repository
        uses: actions/checkout@v7
        with:
          fetch-depth: 0

      - name: Set up Python
        uses: actions/setup-python@v6
        with:
          python-version: "3.11"
          cache: pip

      - name: Set up the TypeScript Effect runtime
        uses: actions/setup-node@v6
        with:
          node-version: "24"

      - name: Install test dependencies
        run: python -m pip install --disable-pip-version-check -e ".[test]"

      - name: Run native Windows lifecycle tests
        run: >-
          python -m pytest -q
          tests/test_command_invocation.py
          tests/test_doctor_install_freshness.py
          tests/test_file_lock.py
          tests/test_file_lock_cross_process.py
          tests/control_plane/test_coordination_file_provider.py
          tests/control_plane/test_effect_runtime_integration.py
          tests/control_plane/test_local_authority_shadow_outbox.py
          tests/test_self_update_runtime_activation.py
          tests/test_windows_install.py

      - name: Set up the qualified SQLite runtime
        uses: actions/setup-node@v6
        with:
          node-version: "22.14"

      - name: Run SQLite CLI integration
        run: python -m pytest -q tests/control_plane/test_sqlite_authority_cli.py

      - name: Run native scheduler and SQLite authority tests
        run: >-
          node --no-warnings --experimental-sqlite --experimental-strip-types --test
          tests/control_plane_ts/scheduler_heartbeat_commit.test.ts
          tests/control_plane_ts/scheduler_heartbeat_commit_cli.test.ts
          tests/control_plane_ts/sqlite_authority_store.test.ts
          tests/control_plane_ts/local_authority_provider.test.ts
          tests/control_plane_ts/sqlite_runtime_admission.test.ts

  presentation:
    needs: changes
    if: needs.changes.outputs.presentation_tests == 'true'
    runs-on: ubuntu-latest
    timeout-minutes: 15
    steps:
      - uses: actions/checkout@v7
      - uses: actions/setup-python@v6
        with:
          python-version: "3.11"
          cache: pip
      - uses: actions/setup-node@v6
        with:
          node-version: "24"
          cache: npm
          cache-dependency-path: apps/presentation/dashboard/package-lock.json
      - run: python -m pip install --disable-pip-version-check -e ".[test]"
      - name: Build and test the actual packaged Dashboard
        working-directory: apps/presentation/dashboard
        run: |
          npm ci --ignore-scripts
          npm run build:chat
          if [[ -n "$(git -C ../../.. status --short --untracked-files=all -- loopx/web/chat)" ]]; then
            echo "Packaged Dashboard differs from its source build." >&2
            exit 1
          fi
          ./node_modules/.bin/playwright install --with-deps chromium
          npm run smoke:personal-workspace-packaged

  merge-gate:
    # Always publish one stable outcome, including documentation-only PRs.
    if: always()
    needs: [changes, checks, pytest, node-minimum-compatibility, stage2c-correctness-e2e, windows-powershell, presentation]
    runs-on: ubuntu-latest
    timeout-minutes: 3
    steps:
      - uses: actions/checkout@v7
      - uses: actions/setup-python@v6
        with:
          python-version: "3.11"
      - name: Reject incomplete or unsuccessful qualification
        env:
          NEEDS_JSON: ${{ toJSON(needs) }}
        run: python scripts/ci/review_gate.py verify