diff --git a/.github/e2e-deployment-inputs.json b/.github/e2e-deployment-inputs.json new file mode 100644 index 000000000..aeef9db1e --- /dev/null +++ b/.github/e2e-deployment-inputs.json @@ -0,0 +1,13 @@ +{ + "prefixes": [ + "apps/" + ], + "files": [ + ".npmrc", + ".vercelignore", + "package-lock.json", + "package.json", + "turbo.json", + "vercel.json" + ] +} diff --git a/.github/workflows/e2e-tests.yml b/.github/workflows/e2e-tests.yml index 954903aa8..b40939161 100644 --- a/.github/workflows/e2e-tests.yml +++ b/.github/workflows/e2e-tests.yml @@ -23,9 +23,11 @@ env: # "Protection Bypass for Automation" value, under Vercel → Project → # Settings → Deployment Protection. Without it the suite fails fast with # an explicit missing-secret message rather than silently asserting - # against Vercel's login page. If no PR preview can be resolved, the - # workflow fails closed rather than silently skipping the gate or probing - # production. Every suite request also carries `X-EventRelay-Probe: e2e` + # against Vercel's login page. Deployment-impacting PRs fail closed if + # their own preview cannot be resolved. PRs that change no input from + # `.github/e2e-deployment-inputs.json` keep the required job present but + # report the live suite as not applicable instead of probing production. + # Every suite request also carries `X-EventRelay-Probe: e2e` # and `User-Agent: EventRelay-E2E/`; proxy.ts emits the # bounded `e2e_probe_request` event so Vercel logs can be filtered by run. # These headers are observability only and never authorize or bypass limits. @@ -64,7 +66,56 @@ jobs: - name: Checkout uses: actions/checkout@v7 + - name: Classify Vercel deployment impact + id: impact + if: github.event_name == 'pull_request' + uses: actions/github-script@v9 + with: + script: | + const fs = require('fs'); + const manifest = JSON.parse( + fs.readFileSync('.github/e2e-deployment-inputs.json', 'utf8'), + ); + const [files, { data: pullRequest }] = await Promise.all([ + github.paginate(github.rest.pulls.listFiles, { + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: context.issue.number, + per_page: 100, + }), + github.rest.pulls.get({ + owner: context.repo.owner, + repo: context.repo.repo, + pull_number: context.issue.number, + }), + ]); + const fileListIncomplete = files.length < pullRequest.changed_files; + const changedPaths = files.flatMap((file) => + [file.filename, file.previous_filename].filter(Boolean), + ); + const deploymentInputs = changedPaths.filter((path) => + manifest.files.includes(path) || + manifest.prefixes.some((prefix) => path.startsWith(prefix)), + ); + // GitHub caps pull-file listings at 3,000 entries. If that cap + // hides any path, fail closed and require a preview rather than + // allowing an unseen deployment input to bypass live E2E. + const needsPreview = fileListIncomplete || deploymentInputs.length > 0; + + core.setOutput('needs_preview', String(needsPreview)); + await core.summary + .addHeading('E2E deployment impact', 3) + .addRaw( + fileListIncomplete + ? `GitHub returned ${files.length} of ${pullRequest.changed_files} changed files; the live E2E suite will fail closed and require this PR's preview.` + : needsPreview + ? `A Vercel deployment input changed (${deploymentInputs.length} matching path(s)); the live E2E suite will run against this PR's preview.` + : 'No Vercel deployment input changed; the live E2E suite is not applicable to this PR.', + ) + .write(); + - name: Setup Node.js + if: github.event_name != 'pull_request' || steps.impact.outputs.needs_preview == 'true' uses: actions/setup-node@v7 with: node-version: '22' @@ -72,7 +123,7 @@ jobs: - name: Extract Vercel preview URL for this PR id: preview - if: github.event_name == 'pull_request' + if: github.event_name == 'pull_request' && steps.impact.outputs.needs_preview == 'true' env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | @@ -137,10 +188,12 @@ jobs: exit 1 - name: Install dependencies + if: github.event_name != 'pull_request' || steps.impact.outputs.needs_preview == 'true' run: npm install --legacy-peer-deps - name: Run E2E tests id: e2e_tests + if: github.event_name != 'pull_request' || steps.impact.outputs.needs_preview == 'true' run: | # The verdict travels out via PIPESTATUS, not via this step's exit code. # Piping to `tee` makes the pipeline return tee's status, and the @@ -154,7 +207,9 @@ jobs: - name: Parse test results id: results - if: always() + if: >- + always() && + (github.event_name != 'pull_request' || steps.impact.outputs.needs_preview == 'true') run: | if [ "${{ steps.e2e_tests.outputs.exit_code }}" = "0" ]; then echo "status=green" >> $GITHUB_OUTPUT @@ -175,7 +230,10 @@ jobs: echo "failed=$FAILED" >> $GITHUB_OUTPUT - name: Post PR comment with results - if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository + if: >- + github.event_name == 'pull_request' && + steps.impact.outputs.needs_preview == 'true' && + github.event.pull_request.head.repo.full_name == github.repository uses: actions/github-script@v9 with: script: | @@ -262,3 +320,13 @@ jobs: run: | echo "✅ ALL TESTS PASSED" echo "Deployment at $BASE_URL is healthy." + + - name: Record E2E as not applicable + if: github.event_name == 'pull_request' && steps.impact.outputs.needs_preview == 'false' + run: | + { + echo "### ⏭️ E2E not applicable" + echo "" + echo "This PR does not change a Vercel deployment input, so it has no PR-specific preview to probe." + echo "The required E2E job still ran and classified the change; pushes to main and deployment-impacting PRs continue to run the full live suite." + } >> "$GITHUB_STEP_SUMMARY" diff --git a/tests/unit/test_workflow_pr_gate_coverage.py b/tests/unit/test_workflow_pr_gate_coverage.py index 98165cd80..159e31e6d 100644 --- a/tests/unit/test_workflow_pr_gate_coverage.py +++ b/tests/unit/test_workflow_pr_gate_coverage.py @@ -11,12 +11,14 @@ from __future__ import annotations +import json from pathlib import Path import pytest import yaml WORKFLOWS = Path(__file__).resolve().parents[2] / ".github/workflows" +DEPLOYMENT_INPUTS = WORKFLOWS.parent / "e2e-deployment-inputs.json" # PR-triggered quality gates. Each must fire regardless of the PR's base branch. GATED_WORKFLOWS = [ @@ -86,3 +88,91 @@ def test_e2e_job_does_not_skip_pull_requests_without_repo_var() -> None: "e2e-tests.yml gates the PR job on E2E_BASE_URL, so PRs without that " "repository variable skip the E2E quality gate entirely" ) + + +def _needs_vercel_preview(*paths: str, changed_files: int | None = None) -> bool: + manifest = json.loads(DEPLOYMENT_INPUTS.read_text()) + if changed_files is not None and len(paths) < changed_files: + return True + return any( + path in manifest["files"] + or any(path.startswith(prefix) for prefix in manifest["prefixes"]) + for path in paths + ) + + +@pytest.mark.parametrize( + "path", + [ + "apps/web/src/app/page.tsx", + "apps/web/next.config.ts", + "package.json", + "package-lock.json", + "turbo.json", + "vercel.json", + ".npmrc", + ".vercelignore", + ], +) +def test_e2e_deployment_inputs_require_a_preview(path: str) -> None: + assert _needs_vercel_preview(path), f"{path} should require a Vercel preview" + + +@pytest.mark.parametrize( + "path", + [ + ".github/workflows/ci.yml", + "artifacts/security/audit.md", + "docs/runbook.md", + "scripts/security/audit_local_mcp_configs.py", + "tests/security/test_mcp_catalog_approval.py", + "tests/unit/test_workflow_pr_gate_coverage.py", + ], +) +def test_non_deployment_inputs_do_not_require_a_preview(path: str) -> None: + assert not _needs_vercel_preview(path), ( + f"{path} should not wait for a preview Vercel intentionally does not build" + ) + + +def test_e2e_workflow_classifies_current_and_previous_file_names() -> None: + """Renaming a web input out of apps/ must still trigger a preview.""" + workflow = yaml.safe_load((WORKFLOWS / "e2e-tests.yml").read_text()) + steps = workflow["jobs"]["e2e"]["steps"] + impact = next(step for step in steps if step.get("id") == "impact") + script = impact["with"]["script"] + + assert "github.paginate(github.rest.pulls.listFiles" in script + assert "github.rest.pulls.get" in script + assert "files.length < pullRequest.changed_files" in script + assert "file.filename" in script + assert "file.previous_filename" in script + assert "e2e-deployment-inputs.json" in script + assert "fileListIncomplete || deploymentInputs.length > 0" in script + assert "core.setOutput('needs_preview', String(needsPreview))" in script + + +def test_incomplete_pr_file_list_requires_a_preview() -> None: + """GitHub caps PR file listings; an unseen app path must not bypass E2E.""" + assert _needs_vercel_preview("docs/first-3000-files.md", changed_files=3001) + + +def test_e2e_execution_steps_require_deployment_impact_on_pull_requests() -> None: + workflow = yaml.safe_load((WORKFLOWS / "e2e-tests.yml").read_text()) + steps = workflow["jobs"]["e2e"]["steps"] + by_name = {step.get("name"): step for step in steps} + + for name in [ + "Setup Node.js", + "Extract Vercel preview URL for this PR", + "Install dependencies", + "Run E2E tests", + "Parse test results", + "Post PR comment with results", + ]: + assert "steps.impact.outputs.needs_preview" in by_name[name].get("if", ""), ( + f"{name} must not run when Vercel has no PR preview to test" + ) + + not_applicable = by_name["Record E2E as not applicable"] + assert "steps.impact.outputs.needs_preview == 'false'" in not_applicable["if"]