diff --git a/.changeset/cli-coverage-comment.md b/.changeset/cli-coverage-comment.md new file mode 100644 index 00000000..95a09725 --- /dev/null +++ b/.changeset/cli-coverage-comment.md @@ -0,0 +1,5 @@ +--- +"@perfect-abstractions/compose-cli": patch +--- + +Add automated CLI test coverage reporting to pull requests. diff --git a/.github/scripts/cli-coverage-report.js b/.github/scripts/cli-coverage-report.js new file mode 100644 index 00000000..f187f811 --- /dev/null +++ b/.github/scripts/cli-coverage-report.js @@ -0,0 +1,36 @@ +const fs = require('node:fs'); +const path = require('node:path'); +const { parseLcovContent, calculateCoverage } = require('./coverage-comment'); + +const MARKER = ''; + +function generateReport(content, commit) { + const metrics = parseLcovContent(content); + const rows = [ + ['Lines', metrics.coveredLines, metrics.totalLines], + ['Functions', metrics.coveredFunctions, metrics.totalFunctions], + ['Branches', metrics.coveredBranches, metrics.totalBranches], + ]; + const table = rows.map(([name, covered, total]) => + `| ${name} | ${calculateCoverage(covered, total)}% | ${covered}/${total} |`).join('\n'); + return `${MARKER}\n## CLI Coverage\n\n` + + `Commit: \`${commit.slice(0, 7)}\`\n\n` + + `| Metric | Coverage | Covered/Total |\n| --- | ---: | ---: |\n${table}\n`; +} + +function main() { + const lcovPath = path.join(process.cwd(), 'cli', 'coverage', 'lcov.info'); + const content = fs.readFileSync(lcovPath, 'utf8'); + const prNumber = Number(process.env.PR_NUMBER); + const headSha = process.env.HEAD_SHA; + const workflowSha = process.env.WORKFLOW_SHA; + if (!Number.isSafeInteger(prNumber) || prNumber <= 0 || !headSha || !workflowSha) { + throw new Error('CLI coverage requires PR_NUMBER, HEAD_SHA, and WORKFLOW_SHA'); + } + fs.writeFileSync('cli-coverage-report.md', generateReport(content, headSha)); + fs.writeFileSync('cli-coverage-data.json', JSON.stringify({ prNumber, headSha, workflowSha })); +} + +if (require.main === module) main(); + +module.exports = { generateReport, MARKER }; diff --git a/.github/scripts/cli-coverage-report.test.js b/.github/scripts/cli-coverage-report.test.js new file mode 100644 index 00000000..886b3b39 --- /dev/null +++ b/.github/scripts/cli-coverage-report.test.js @@ -0,0 +1,12 @@ +const assert = require('node:assert/strict'); +const test = require('node:test'); +const { generateReport, MARKER } = require('./cli-coverage-report'); + +test('renders CLI coverage from LCOV totals', () => { + const report = generateReport('LF:10\nLH:8\nFNF:5\nFNH:4\nBRF:2\nBRH:1\n', 'abcdef123456'); + assert.ok(report.startsWith(`${MARKER}\n## CLI Coverage`)); + assert.match(report, /\| Lines \| 80% \| 8\/10 \|/); + assert.match(report, /\| Functions \| 80% \| 4\/5 \|/); + assert.match(report, /\| Branches \| 50% \| 1\/2 \|/); + assert.match(report, /Commit: `abcdef1`/); +}); diff --git a/.github/scripts/coverage-comment.js b/.github/scripts/coverage-comment.js index 65f3ebff..c84d68d9 100644 --- a/.github/scripts/coverage-comment.js +++ b/.github/scripts/coverage-comment.js @@ -145,3 +145,5 @@ function generateCoverageFile() { if (require.main === module) { generateCoverageFile(); } + +module.exports = { parseLcovContent, calculateCoverage }; diff --git a/.github/scripts/post-cli-coverage-comment.js b/.github/scripts/post-cli-coverage-comment.js new file mode 100644 index 00000000..bbdc811b --- /dev/null +++ b/.github/scripts/post-cli-coverage-comment.js @@ -0,0 +1,35 @@ +const { downloadArtifact, readReport } = require('./workflow-utils'); +const { MARKER } = require('./cli-coverage-report'); + +module.exports = async ({ github, context }, helpers = {}) => { + const download = helpers.downloadArtifact ?? downloadArtifact; + const read = helpers.readReport ?? readReport; + if (!await download(github, context, 'cli-coverage-data')) return; + + const metadataText = read('cli-coverage-data.json'); + if (!metadataText) throw new Error('CLI coverage metadata is missing'); + const metadata = JSON.parse(metadataText); + const { prNumber, headSha, workflowSha } = metadata; + if (!Number.isSafeInteger(prNumber) || prNumber <= 0 || workflowSha !== context.payload.workflow_run.head_sha) { + throw new Error('CLI coverage artifact does not match its workflow run'); + } + + const { owner, repo } = context.repo; + const { data: pr } = await github.rest.pulls.get({ owner, repo, pull_number: prNumber }); + if (pr.head.sha !== headSha || pr.base.repo.full_name !== `${owner}/${repo}`) { + console.log('Skipping coverage from an outdated or unrelated PR run'); + return; + } + + const body = read('cli-coverage-report.md'); + if (!body?.startsWith(MARKER)) throw new Error('CLI coverage report is missing its marker'); + const comments = await github.paginate(github.rest.issues.listComments, { + owner, repo, issue_number: prNumber, per_page: 100, + }); + const existing = comments.find((comment) => comment.user?.type === 'Bot' && comment.body?.includes(MARKER)); + if (existing) { + await github.rest.issues.updateComment({ owner, repo, comment_id: existing.id, body }); + } else { + await github.rest.issues.createComment({ owner, repo, issue_number: prNumber, body }); + } +}; diff --git a/.github/scripts/post-cli-coverage-comment.test.js b/.github/scripts/post-cli-coverage-comment.test.js new file mode 100644 index 00000000..01db7b7c --- /dev/null +++ b/.github/scripts/post-cli-coverage-comment.test.js @@ -0,0 +1,53 @@ +const assert = require('node:assert/strict'); +const test = require('node:test'); +const post = require('./post-cli-coverage-comment'); + +function setup(prSha = 'head-sha') { + const calls = { created: [], updated: [] }; + const github = { + rest: { + pulls: { get: async () => ({ data: { + head: { sha: prSha }, + base: { repo: { full_name: 'owner/repo' } }, + } }) }, + issues: { + listComments: async () => {}, + createComment: async (args) => { calls.created.push(args); }, + updateComment: async (args) => { calls.updated.push(args); }, + }, + }, + paginate: async () => [{ + id: 42, + user: { type: 'Bot' }, + body: '\nOld report', + }], + }; + const context = { + repo: { owner: 'owner', repo: 'repo' }, + payload: { workflow_run: { head_sha: 'head-sha' } }, + }; + const helpers = { + downloadArtifact: async () => true, + readReport: (name) => name.endsWith('.json') + ? JSON.stringify({ prNumber: 17, headSha: 'head-sha', workflowSha: 'head-sha' }) + : '\nNew report', + }; + return { calls, github, context, helpers }; +} + +test('updates the existing CLI coverage comment', async () => { + const { calls, github, context, helpers } = setup(); + await post({ github, context }, helpers); + assert.equal(calls.created.length, 0); + assert.deepEqual(calls.updated, [{ + owner: 'owner', repo: 'repo', comment_id: 42, + body: '\nNew report', + }]); +}); + +test('ignores coverage from an older PR commit', async () => { + const { calls, github, context, helpers } = setup('newer-head-sha'); + await post({ github, context }, helpers); + assert.equal(calls.created.length, 0); + assert.equal(calls.updated.length, 0); +}); diff --git a/.github/workflows/cli-coverage-comment.yml b/.github/workflows/cli-coverage-comment.yml new file mode 100644 index 00000000..6fa0c5a3 --- /dev/null +++ b/.github/workflows/cli-coverage-comment.yml @@ -0,0 +1,27 @@ +name: Post CLI Coverage Comment + +on: + workflow_run: + workflows: ['CLI Coverage'] + types: [completed] + +permissions: + actions: read + contents: read + issues: write + pull-requests: write + +jobs: + comment: + if: github.event.workflow_run.event == 'pull_request' && github.event.workflow_run.conclusion == 'success' + runs-on: ubuntu-latest + steps: + - name: Checkout trusted default branch + uses: actions/checkout@v4 + + - name: Post or update comment + uses: actions/github-script@v7 + with: + script: | + const post = require('./.github/scripts/post-cli-coverage-comment.js'); + await post({ github, context }); diff --git a/.github/workflows/cli-coverage.yml b/.github/workflows/cli-coverage.yml new file mode 100644 index 00000000..d422357f --- /dev/null +++ b/.github/workflows/cli-coverage.yml @@ -0,0 +1,58 @@ +name: CLI Coverage + +on: + pull_request: + paths: + - 'cli/**' + - '.github/workflows/cli-coverage.yml' + - '.github/workflows/cli-coverage-comment.yml' + - '.github/scripts/cli-coverage-report.js' + - '.github/scripts/post-cli-coverage-comment.js' + +permissions: + contents: read + +jobs: + coverage: + runs-on: ubuntu-latest + env: + FOUNDRY_PROFILE: ci + steps: + - name: Checkout PR branch + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Install Foundry + uses: foundry-rs/foundry-toolchain@v1 + + - name: Setup Node.js + uses: actions/setup-node@v4 + with: + node-version: 24.x + cache: npm + + - name: Install dependencies + run: npm ci + + - name: Build CLI + run: npm run cli@build + + - name: Run CLI coverage + run: npm run cli@coverage + + - name: Generate report + env: + PR_NUMBER: ${{ github.event.pull_request.number }} + HEAD_SHA: ${{ github.event.pull_request.head.sha }} + WORKFLOW_SHA: ${{ github.event.pull_request.head.sha }} + run: node .github/scripts/cli-coverage-report.js + + - name: Upload report + uses: actions/upload-artifact@v4 + with: + name: cli-coverage-data + path: | + cli-coverage-report.md + cli-coverage-data.json + retention-days: 1 diff --git a/cli/vitest.config.ts b/cli/vitest.config.ts index 2f4dac18..cd7f564e 100644 --- a/cli/vitest.config.ts +++ b/cli/vitest.config.ts @@ -2,5 +2,10 @@ module.exports = { test: { environment: "node", include: ["test/**/*.test.ts"], + coverage: { + include: ["src/**/*.ts"], + exclude: ["src/templates/**"], + reporter: ["text", "lcov"], + }, }, };