Add standard CodeceptJS conformance tests

.github/workflows/main.yaml

@@ -51,6 +51,7 @@ jobs:
     permissions:
       contents: read
       actions: read
+      pull-requests: write
 
     steps:
       - name: Checkout
@@ -62,7 +63,13 @@ jobs:
           node-version: ${{ matrix.node }}
           cache: npm
 
+      - name: Prepare conformance diagnostics
+        run: |
+          echo "CTRF_STANDARD_ARTIFACTS=$RUNNER_TEMP/conformance" >> "$GITHUB_ENV"
+          mkdir -p "$RUNNER_TEMP/conformance"
+
       - name: Install
+        id: install
         run: npm ci
 
       - name: Type check (TypeScript 7)
@@ -72,16 +79,80 @@ jobs:
         run: npm run test:coverage
 
       - name: Framework integration
+        if: ${{ !cancelled() && steps.install.outcome == 'success' }}
         run: npm run test:integration
 
       - name: Packed consumer checks
+        if: ${{ !cancelled() && steps.install.outcome == 'success' }}
         run: npm run test:package
 
+      - name: Run shared checker controls
+        if: ${{ !cancelled() && steps.install.outcome == 'success' }}
+        run: npm run test:standard:checker
+
+      - name: Run unit conformance
+        if: ${{ !cancelled() && steps.install.outcome == 'success' }}
+        run: npm run test:standard:conformance:unit
+
+      - name: Run installed-package conformance with real CodeceptJS
+        if: ${{ !cancelled() && steps.install.outcome == 'success' }}
+        run: npm run test:standard:conformance:integration
+
+      - name: Publish conformance test reports
+        uses: ctrf-io/github-test-reporter@v1
+        if: ${{ !cancelled() && steps.install.outcome == 'success' }}
+        with:
+          report-path: '${{ runner.temp }}/conformance/reports/*.json'
+          title: 'Reporter conformance — contract families (Node ${{ matrix.node }})'
+          summary-report: true
+          suite-folded-report: true
+          group-by: suite
+          pull-request: ${{ matrix.node == '24' && github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository }}
+          overwrite-comment: true
+          comment-tag: reporter-conformance
+          collapse-large-reports: true
+          upload-artifact: true
+          artifact-name: ctrf-conformance-node-${{ matrix.node }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Publish shared checker controls
+        uses: ctrf-io/github-test-reporter@v1
+        if: ${{ !cancelled() && steps.install.outcome == 'success' }}
+        with:
+          report-path: '${{ runner.temp }}/conformance/checker-reports/*.json'
+          title: 'Shared checker controls (Node ${{ matrix.node }})'
+          summary-report: true
+          suite-folded-report: true
+          group-by: suite
+          pull-request: ${{ matrix.node == '24' && github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository }}
+          overwrite-comment: true
+          comment-tag: reporter-checker
+          upload-artifact: true
+          artifact-name: ctrf-checker-node-${{ matrix.node }}
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Upload conformance results and diagnostics
+        uses: actions/upload-artifact@v4
+        if: always()
+        with:
+          name: conformance-node-${{ matrix.node }}
+          path: ${{ runner.temp }}/conformance/
+          include-hidden-files: true
+          if-no-files-found: warn
+
       - name: Publish Test Report
         uses: ctrf-io/github-test-reporter@v1
         with:
-          report-path: './ctrf/*.json'
-          summary-delta-report: true
+          report-path: './ctrf/reporter-tests.json'
+          title: 'Reporter tests (Node ${{ matrix.node }})'
+          summary-report: true
+          test-report: true
+          collapse-large-reports: true
+          pull-request: ${{ matrix.node == '24' && github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository }}
+          overwrite-comment: true
+          comment-tag: reporter-tests
           github-report: true
           insights-report: true
           flaky-rate-report: true
@@ -90,7 +161,7 @@ jobs:
           artifact-name: ctrf-test-report-node-${{ matrix.node }}
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-        if: always() && hashFiles('ctrf/*.json') != ''
+        if: always() && hashFiles('ctrf/reporter-tests.json') != ''
 
       - name: Build
         run: npm run build

CONTRIBUTING.md

@@ -61,7 +61,7 @@ Run `npm run build:check` to check source types with TypeScript 7, and
 
 ## Local verification
 
-Use Node 22.12.0 or newer. CI tests Node 22, 24 and 26; quality and security
+Use Node 22.18.0 or newer for development. CI tests Node 22, 24 and 26; quality and security
 stages run on Node 26. Keep direct dependencies, peer dependencies and explicit
 security overrides pinned to exact versions. Update the lockfile with changes.
 
@@ -85,3 +85,59 @@ The tsup declaration build uses the TypeScript 6 compiler API. Keep its
 
 This version removes the old repository-local CTRF type declarations. Use
 canonical `ctrf` types and strict 0.2.0 validation in new report tests.
+
+## Standard conformance tests
+
+Run the relevant checks before submitting a pull request and report their results.
+CI publishes the existing reporter tests through GitHub Test Reporter. The
+reference Node job maintains a `reporter-tests` PR comment, separately from the
+conformance comment; both comments are overwritten on subsequent builds.
+
+### Conformance tests
+
+For reporter output changes, update the affected conformance fixtures and run:
+
+```bash
+npm run test:standard:conformance:unit
+npm run test:standard:conformance:integration
+```
+
+`npm run test:standard:conformance` runs both levels even when unit fails.
+The [suite guide](tests/standard/conformance/README.md) explains setup and file roles.
+Unit tests use controlled native inputs; integration tests install the packed
+reporter and run real CodeceptJS through its public plugin interface with independent native observers.
+They need npm registry access to install the pinned dependencies and remain
+separate from `npm test`.
+
+Keep common family IDs stable, update `codeceptjs.json` when capabilities change,
+and provide positive evidence for supported properties. Unsupported optional
+features are skipped with reasons; unrecorded broken behavior must fail.
+Expected-failure entries in `tests/standard/conformance/expected-failures.mjs`
+must identify a fix follow-up and match specific levels/samples. Use a local
+draft identifier until issue publication, then link the GitHub issue. Keep checks running;
+unexpected passes and changed failures must fail. Remove markers with the fix.
+Check schema and written-specification implications, including valid omission and
+conditional rules. Recommendations remain advisory. Do not change expectations
+to match incorrect reporter output.
+
+CI runs both conformance levels as separate steps alongside the existing integration checks in the test/build
+job on Node 22, 24 and 26. A unit failure does not prevent integration from running.
+The job publishes CTRF results for both levels through GitHub Test Reporter, with
+the built-in summary and folded suite table, grouped by level and check category. Node 24 also maintains one PR
+comment tagged `reporter-conformance`, overwriting it on later runs. It describes
+the reference runtime; check the full matrix for compatibility results. Fork PRs
+keep build summaries/artifacts without posting a comment. Expected failures retain
+explicit test-name labels and use Other with defect metadata rather than passed.
+Unit/integration names and identities stay distinct; the conformance comment
+contains contract families only. Shared checker controls use the same independent
+fixtures and test IDs across reporters, run with `npm run test:standard:checker`,
+and publish separately under `reporter-checker`. Reporter-specific fixture,
+matcher and reporting controls belong in the reporter's own tests and
+`reporter-tests` results. `npm run test:standard` runs all standard suites.
+CTRF artifacts, conformance summaries and integration reports/logs are retained,
+including on failure. The commands use the console and CTRF reporters selected by
+native test configuration and `tests/standard/conformance/run.mjs`; no additional
+runner JSON output is required. Ordinary CTRF is `ctrf/reporter-tests.json`; exclude
+fixture-generated producer reports from publication.
+In your PR, include commands and results and explain skips, expected failures and
+warnings. Track defect details and fixes in GitHub issues rather than these docs.

README.md

@@ -126,7 +126,7 @@ If you find this project useful, consider giving it a GitHub star ⭐ It means a
 
 ## Node and framework support
 
-Version 0.1.0 requires Node.js **22.12.0 or newer** and CodeceptJS **4.2.0**.
+Version 0.1.0 requires Node.js **22.12.0 or newer** and CodeceptJS **4.2.2**.
 The package uses ESM internally and publishes ESM and CommonJS entry points,
 with declarations included for TypeScript consumers. The callable plugin factory
 is preserved for CommonJS configuration. Earlier CodeceptJS 3 and Node 18/20
@@ -199,24 +199,22 @@ npm does not propagate those overrides into a consumer's installation.
 
 ### Consumer dependency audit
 
-A fresh consumer installing the CodeceptJS 4.2.0 peer currently reports ten
-framework-related audit findings (three high, six moderate and one low).
-The reporter checkout applies the exact development overrides below and retains
-three moderate findings through CodeceptJS's `sprintf-js` dependency; its
-production-only audit is clean. The overrides do not resolve every framework
-finding. npm ignores a dependency package's overrides, so consumers who need
-these patches must add them to their own root `package.json`:
+Audit the application root after installing the reporter and its CodeceptJS peer.
+The reporter checkout uses the tested development overrides below; npm does not
+propagate a dependency package's overrides into consumer installations. A clean
+checkout audit therefore does not establish a clean application audit. Consumers
+that need these transitive patches must add them to their own root `package.json`:
 
 ```json
 {
   "overrides": {
     "codeceptjs": {
       "@xmldom/xmldom": "0.9.12",
       "axios": "1.20.0",
-      "uuid": "11.1.1",
+      "uuid": "14.0.3",
       "mocha": {
-        "diff": "8.0.3",
-        "serialize-javascript": "7.1.2"
+        "diff": "9.0.0",
+        "serialize-javascript": "7.1.4"
       }
     }
   }

biome.json

@@ -1,5 +1,5 @@
 {
-	"$schema": "https://biomejs.dev/schemas/2.5.15/schema.json",
+	"$schema": "https://biomejs.dev/schemas/2.6.2/schema.json",
 	"vcs": {
 		"enabled": true,
 		"clientKind": "git",
@@ -40,5 +40,10 @@
 				"organizeImports": "off"
 			}
 		}
+	},
+	"markdown": {
+		"formatter": {
+			"enabled": false
+		}
 	}
 }