diff --git a/.changeset/integration-check.md b/.changeset/integration-check.md new file mode 100644 index 0000000000000..0b4378d51baac --- /dev/null +++ b/.changeset/integration-check.md @@ -0,0 +1,7 @@ +--- +'@astryxdesign/cli': patch +--- + +[feat] New `astryx integration check`, the CI check for an integration package's own build. It takes no package argument and checks the integration package in the current folder. It runs the four `doctor integration` checks (`validate`, `components`, `templates`, `docs`), prints each result, then a summary, and exits 1 when any of them would. `--strict` also exits 1 on any warning and on any check that did not run, such as a package with no `astryx.integration.*`: add `"typecheck": "tsc --noEmit && astryx integration check --strict"` to package.json so every build keeps the package's docs in sync. Its exit codes and JSON are a contract for the builds that run it: a new check, or a new failure code, joins only in a minor release named in the release notes. JSON: a new `integration.check` envelope carries each check's own envelope unchanged. Its docs say to run it in CI, so `astryx search ci` finds it. The `doctor integration` subcommands, their exit codes, and their JSON do not change. + +@josephfarina diff --git a/packages/cli/README.md b/packages/cli/README.md index 301bcb360c41a..81b0e5be3123f 100644 --- a/packages/cli/README.md +++ b/packages/cli/README.md @@ -480,6 +480,7 @@ Every response has a `type` discriminant. The full set is below (generated from | `integration.template-conflicts` | validated (false when no integration manifest was found, so nothing was inspected), the integration identity, structural issues, and non-blocking Core template-id conflicts as {id, severity: warning, integrationPackage, integrationType, integrationName, coreMatches, message, command}. | | `integration.component-conflicts` | validated (false when no integration manifest was found, so nothing was inspected), the integration identity, structural issues, and non-blocking conflicts where an integration component name is also owned by Core; each conflict includes the exact package-qualified command. When the manifest loads and declares a components root, propDrift compares documented props with the TypeScript props: status (checked \| skipped), reason, typescript, components, typesCompared, typesSkipped, findings (all warnings), unchecked, typesNotCompared, and propsNotChecked. | | `integration.doc-conflicts` | validated (false when no integration manifest was found, so nothing was inspected), the integration identity, structural issues, and Core doc overlaps. Each finding includes `severity` (`info` \| `error`) and `relationship` (`replaces` \| `extends` \| `accidental`). | +| `integration.check` | Every source check of an integration package in one result: validated (false when no integration manifest was found, so nothing was checked), the package name and version, strict, and checks, one entry per check (validate, components, templates, docs), each with check, errors, warnings, skipped, failed, and response, the envelope that check returns on its own. The totals are errors, warnings, and skipped; unchecked counts what the checks could not compare. With --strict, failures lists each warning and each piece of skipped work again as an error, in the issue format the checks use; it is empty without --strict. failed decides the exit code. | diff --git a/packages/cli/api/index.mjs b/packages/cli/api/index.mjs index 92c48be87c1bc..7f5b08e34ec54 100644 --- a/packages/cli/api/index.mjs +++ b/packages/cli/api/index.mjs @@ -67,6 +67,7 @@ export { integrationComponentConflicts, integrationDocConflicts, } from './integration/authoring-checks.mjs'; +export {integrationCheck} from './integration/check/check.mjs'; export {AstryxError} from './error.mjs'; // The one shared logger: the `logger` instance side-effecting commands write // through, plus its generated `Logger` type. Part of the public surface so an @@ -100,3 +101,4 @@ export * from './integration/integration-authoring.type.mjs'; export * from './integration/pack-check.type.mjs'; export * from './integration/validate-integration.type.mjs'; export * from './integration/authoring-checks.type.mjs'; +export * from './integration/integration-check.type.mjs'; diff --git a/packages/cli/api/integration/check/check.mjs b/packages/cli/api/integration/check/check.mjs new file mode 100644 index 0000000000000..10aac71685abc --- /dev/null +++ b/packages/cli/api/integration/check/check.mjs @@ -0,0 +1,357 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +/** + * @file `astryx integration check`: every `doctor integration` check in one + * call, for a package's own build. + * + * Runs `validate`, `components`, `templates` and `docs` against the + * integration package in `cwd` and counts each result the way that check's + * own subcommand exits, so the + * run fails exactly when one of them would. With `strict` it also fails on + * any warning and on any check, or comparison a check carries, that did not + * run, so a package can gate its build on its docs staying in sync. `--strict` + * adds no check: it reports the same warnings and skipped work again, as + * errors in the checks' own issue format, and fails on them. + * + * What fails it is a contract for the builds that run it: a new check, or a + * new failure code, joins only in a minor release that names it. + * `test/integration-check-failures.test.mjs` lists each one, so adding one + * shows up in review. + * + * @input A local or installed integration package. + * @output Each check's own envelope with its errors, warnings and skipped + * work, the totals, and whether the run fails. + * @position Backs `astryx integration check`. + */ + +import { + integrationComponentConflicts, + integrationDocConflicts, + integrationTemplateConflicts, +} from '../authoring-checks.mjs'; +import { + summarizeIssues, + validateIntegration, +} from '../validate-integration.mjs'; + +/** @typedef {import('../integration-check.type.mjs').IntegrationCheckName} IntegrationCheckName */ +/** @typedef {import('../integration-check.type.mjs').IntegrationCheckReport} IntegrationCheckReport */ +/** @typedef {import('../integration-check.type.mjs').IntegrationCheckResponse} IntegrationCheckResponse */ +/** @typedef {import('../integration-check.type.mjs').IntegrationCheckResult} IntegrationCheckResult */ +/** @typedef {import('../integration-check.type.mjs').IntegrationCheckOptions} IntegrationCheckOptions */ +/** @typedef {import('../integration-check.type.mjs').IntegrationCheckCommandResponse} IntegrationCheckCommandResponse */ +/** @typedef {import('../integration-check.type.mjs').IntegrationCheckFailure} IntegrationCheckFailure */ + +/** The checks `integration check` runs, in the order it runs them. */ +export const INTEGRATION_CHECKS = /** @type {const} */ ([ + 'validate', + 'components', + 'templates', + 'docs', +]); + +/** The skipped-work line for a package with no integration manifest. */ +export const NO_MANIFEST_SKIP = + 'No astryx.integration.* found next to package.json, so nothing was checked.'; + +/** + * The code a conflict carries when `--strict` reports it: conflicts have no + * code of their own. + */ +export const CONFLICT_CODES = /** @type {const} */ ({ + components: 'component_name_conflict', + templates: 'template_id_conflict', +}); + +/** The code a check with no integration manifest carries under `--strict`. */ +export const NO_MANIFEST_CODE = 'missing_manifest'; + +/** + * The code a skipped comparison carries under `--strict`: its key in the + * check's result, in snake case, then `_skipped` (`propDrift` is + * `prop_drift_skipped`). + * @param {string} key + */ +export function skippedCode(key) { + return `${key.replace(/([a-z0-9])([A-Z])/g, '$1_$2').toLowerCase()}_skipped`; +} + +/** Each check's envelope type, by check. */ +const CHECK_OF_TYPE = /** @type {const} */ ({ + 'integration.validate': 'validate', + 'integration.component-conflicts': 'components', + 'integration.template-conflicts': 'templates', + 'integration.doc-conflicts': 'docs', +}); + +/** + * Whether a value is a report a check carries beside its issues: a + * comparison with a `status` of `checked` or `skipped` and its `findings`. + * @param {unknown} value + * @returns {value is IntegrationCheckReport} + */ +export function isCheckReport(value) { + if (value == null || typeof value !== 'object' || Array.isArray(value)) { + return false; + } + const report = /** @type {{status?: unknown, findings?: unknown}} */ (value); + return ( + (report.status === 'checked' || report.status === 'skipped') && + Array.isArray(report.findings) + ); +} + +/** + * One finding as an issue: its own code, message, fix, file, and line where + * it has them. + * @param {any} finding + * @param {string} code + * @returns {Omit} + */ +function asIssue(finding, code) { + const file = finding.file ?? finding.doc ?? finding.source; + return { + code, + message: String(finding.message ?? ''), + ...(typeof finding.component === 'string' + ? {component: finding.component} + : {}), + ...(typeof finding.prop === 'string' ? {prop: finding.prop} : {}), + ...(typeof finding.fix === 'string' ? {fix: finding.fix} : {}), + ...(typeof file === 'string' ? {file} : {}), + ...(typeof finding.line === 'number' ? {line: finding.line} : {}), + }; +} + +/** + * Every warning in one check's result, and every piece of work it skipped, + * as issues: what `--strict` reports as errors. The counts come from the + * same list, so the default run and the strict run read the same data. + * @param {IntegrationCheckName} check + * @param {IntegrationCheckResponse} response + * @param {{noPackage?: boolean}} [context] `noPackage`: validate found no + * package.json, which only it reports; then no other check names a + * missing manifest as skipped work + * @returns {{warnings: Array>, skipped: Array>}} + */ +export function strictIssues(check, response, context = {}) { + const data = + /** @type {Record & IntegrationCheckResponse['data']} */ ( + response.data + ); + const warnings = []; + for (const issue of data.issues) { + if (issue.severity === 'warning') { + warnings.push(asIssue(issue, issue.code)); + } + } + const conflictCode = /** @type {Record} */ (CONFLICT_CODES)[ + check + ]; + for (const conflict of /** @type {any[]} */ ( + Array.isArray(data.conflicts) ? data.conflicts : [] + )) { + if (conflict.severity === 'warning' && conflictCode) { + warnings.push(asIssue(conflict, conflictCode)); + } + } + for (const finding of /** @type {any[]} */ ( + Array.isArray(data.findings) ? data.findings : [] + )) { + if (finding.severity === 'warning') { + warnings.push(asIssue(finding, finding.code ?? finding.relationship)); + } + } + const skipped = []; + const noPackage = + context.noPackage === true || + data.issues.some(issue => issue.code === 'no_package'); + if (!data.validated && !noPackage) { + skipped.push({code: NO_MANIFEST_CODE, message: NO_MANIFEST_SKIP}); + } + for (const [key, value] of Object.entries(data)) { + if (!isCheckReport(value)) continue; + if (value.status === 'skipped') { + skipped.push({ + code: skippedCode(key), + message: value.reason ?? `${key} did not run`, + }); + continue; + } + for (const finding of /** @type {any[]} */ (value.findings)) { + if ((finding.severity ?? 'warning') !== 'warning') continue; + warnings.push( + asIssue(finding, finding.code ?? finding.problem ?? finding.kind), + ); + } + // A comparison that ran but judged no prop type (a TypeScript older than + // the type comparison reads) skipped that work. + const typesSkipped = /** @type {{typesSkipped?: unknown}} */ (value) + .typesSkipped; + if (typeof typesSkipped === 'string') { + skipped.push({ + code: 'prop_types_skipped', + message: `prop types not judged: ${typesSkipped}`, + }); + } + } + return {warnings, skipped}; +} + +/** + * Count one check's result the way its own subcommand exits. Errors are what + * make it exit 1: error-severity issues, and for `docs` error findings. + * Warnings are warning issues, conflicts and findings, and the findings of + * any report the check carries. Skipped names work that did not run: no + * integration manifest, or a report that was skipped. + * @param {IntegrationCheckResponse} response + * @param {{noPackage?: boolean}} [context] as {@link strictIssues} takes it + * @returns {Omit} + */ +export function summarizeCheck(response, context = {}) { + const check = /** @type {IntegrationCheckName} */ ( + CHECK_OF_TYPE[response.type] + ); + let errors = summarizeIssues(response.data.issues).errors; + // The docs check is the one whose findings fail it. + if (response.type === 'integration.doc-conflicts') { + errors += response.data.findings.filter( + finding => finding.severity === 'error', + ).length; + } + const {warnings, skipped} = strictIssues(check, response, context); + return { + errors, + warnings: warnings.length, + skipped: skipped.map(each => each.message), + failed: errors > 0, + }; +} + +/** + * Run every `doctor integration` check against the integration package in + * `cwd`, as its own build does, and return the `integration.check` envelope. + * + * @param {IntegrationCheckOptions} [options] + * @returns {Promise} + */ +export async function integrationCheck(options = {}) { + const {cwd = process.cwd(), strict = false} = options; + // One after another: each loads the package through the same discovery. + /** @type {IntegrationCheckResponse[]} */ + const responses = []; + responses.push(await validateIntegration(undefined, {cwd})); + responses.push(await integrationComponentConflicts(undefined, {cwd})); + responses.push(await integrationTemplateConflicts(undefined, {cwd})); + responses.push(await integrationDocConflicts(undefined, {cwd})); + + const [validation] = responses; + return { + type: 'integration.check', + data: { + validated: validation.data.validated, + name: validation.data.name, + version: validation.data.version, + strict, + ...combineChecks(responses, strict), + }, + }; +} + +/** + * Count the four checks' results into one summary. Each check's own row + * counts as its subcommand exits; the totals and `failures` count validate's + * issues once, though the other checks repeat them, and every other finding + * on its own, so two findings with one message stay two. + * @param {IntegrationCheckResponse[]} responses validate first + * @param {boolean} strict + */ +export function combineChecks(responses, strict) { + const [validation] = responses; + // Only validate reports a missing package.json; the other checks read it + // from there instead of each naming a missing manifest. + const noPackage = validation.data.issues.some( + issue => issue.code === 'no_package', + ); + /** @type {IntegrationCheckResult[]} */ + const checks = responses.map(response => ({ + check: /** @type {IntegrationCheckName} */ (CHECK_OF_TYPE[response.type]), + ...summarizeCheck(response, {noPackage}), + response, + })); + // Later checks repeat validate's issues and its missing manifest: each + // match is consumed once per check, as from a multiset. Every other issue + // counts, even a repeat within one check. + const keyOf = (/** @type {string} */ kind, /** @type {object} */ issue) => + `${kind}\0${JSON.stringify(issue)}`; + const validateFound = strictIssues('validate', validation, {noPackage}); + /** @type {Map} */ + const fromValidate = new Map(); + for (const key of [ + ...validation.data.issues + .filter(issue => issue.severity === 'error') + .map(issue => keyOf('error', asIssue(issue, issue.code))), + ...validateFound.warnings.map(issue => keyOf('warning', issue)), + ...validateFound.skipped.map(issue => keyOf('skipped', issue)), + ]) { + fromValidate.set(key, (fromValidate.get(key) ?? 0) + 1); + } + + let errors = 0; + let warnings = 0; + let unchecked = 0; + /** @type {string[]} */ + const skipped = []; + // `--strict` adds no check: it reports the same warnings and skipped work + // as errors, and they fail the run. + /** @type {IntegrationCheckFailure[]} */ + const failures = []; + checks.forEach(({check, response}, index) => { + /** @type {Map} */ + const repeats = index === 0 ? new Map() : new Map(fromValidate); + const isRepeat = (/** @type {string} */ key) => { + const left = repeats.get(key) ?? 0; + if (left === 0) return false; + repeats.set(key, left - 1); + return true; + }; + for (const issue of response.data.issues) { + if ( + issue.severity === 'error' && + !isRepeat(keyOf('error', asIssue(issue, issue.code))) + ) { + errors++; + } + } + if (response.type === 'integration.doc-conflicts') { + errors += response.data.findings.filter( + finding => finding.severity === 'error', + ).length; + } + for (const value of Object.values(response.data)) { + const list = /** @type {{unchecked?: unknown}} */ (value)?.unchecked; + if (isCheckReport(value) && Array.isArray(list)) unchecked += list.length; + } + const found = + index === 0 ? validateFound : strictIssues(check, response, {noPackage}); + for (const issue of found.warnings) { + if (isRepeat(keyOf('warning', issue))) continue; + warnings++; + if (strict) failures.push({check, severity: 'error', ...issue}); + } + for (const issue of found.skipped) { + if (isRepeat(keyOf('skipped', issue))) continue; + skipped.push(`${check}: ${issue.message}`); + if (strict) failures.push({check, severity: 'error', ...issue}); + } + }); + return { + checks, + errors, + warnings, + skipped, + unchecked, + failures, + failed: checks.some(check => check.failed) || failures.length > 0, + }; +} diff --git a/packages/cli/api/integration/check/check.test.mjs b/packages/cli/api/integration/check/check.test.mjs new file mode 100644 index 0000000000000..fad6521e5d920 --- /dev/null +++ b/packages/cli/api/integration/check/check.test.mjs @@ -0,0 +1,489 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +/** + * @file integrationCheck(): each check counted the way its own subcommand + * exits, the reports a check carries, and what strict adds: the same data, + * reported again as errors. + */ + +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import {afterEach, beforeEach, describe, expect, it} from 'vitest'; +import { + integrationComponentConflicts, + integrationDocConflicts, + integrationTemplateConflicts, +} from '../authoring-checks.mjs'; +import { + CONFLICT_CODES, + NO_MANIFEST_CODE, + NO_MANIFEST_SKIP, + combineChecks, + integrationCheck, + isCheckReport, + skippedCode, + summarizeCheck, +} from './check.mjs'; +import {validateIntegration} from '../validate-integration.mjs'; +import {discoverCoreTemplates} from '../../../foundation/discovery/template-adapter.mjs'; + +const IDENTITY = {validated: true, name: '@acme/widgets', version: '1.2.3'}; + +describe('summarizeCheck', () => { + it('counts error and warning issues, and fails only on errors', () => { + expect( + summarizeCheck({ + type: 'integration.validate', + data: { + ...IDENTITY, + issues: [ + {code: 'missing_root', severity: 'error', message: 'm'}, + {code: 'unknown_manifest_key', severity: 'warning', message: 'w'}, + ], + }, + }), + ).toEqual({errors: 1, warnings: 1, skipped: [], failed: true}); + }); + + it('counts warning conflicts as warnings and ignores info', () => { + const conflict = (/** @type {'warning' | 'info'} */ severity) => + /** @type {any} */ ({severity, id: 'x', message: 'm'}); + expect( + summarizeCheck({ + type: 'integration.template-conflicts', + data: { + ...IDENTITY, + issues: [], + conflicts: [conflict('warning'), conflict('info')], + }, + }), + ).toEqual({errors: 0, warnings: 1, skipped: [], failed: false}); + }); + + it('fails the docs check on an error finding, as its subcommand does', () => { + const finding = (/** @type {'error' | 'info'} */ severity) => + /** @type {any} */ ({severity, topic: 't', message: 'm'}); + expect( + summarizeCheck({ + type: 'integration.doc-conflicts', + data: { + ...IDENTITY, + issues: [], + findings: [finding('error'), finding('info')], + }, + }), + ).toEqual({errors: 1, warnings: 0, skipped: [], failed: true}); + }); + + it('names a package with no manifest as skipped work, but not a missing package.json, which is an error', () => { + expect( + summarizeCheck({ + type: 'integration.component-conflicts', + data: { + validated: false, + name: null, + version: null, + issues: [], + conflicts: [], + }, + }).skipped, + ).toEqual([NO_MANIFEST_SKIP]); + expect( + summarizeCheck({ + type: 'integration.validate', + data: { + validated: false, + name: null, + version: null, + issues: [{code: 'no_package', severity: 'error', message: 'm'}], + }, + }), + ).toEqual({errors: 1, warnings: 0, skipped: [], failed: true}); + }); + + it('counts the findings of a report a check carries as warnings, and a skipped report as skipped work', () => { + const data = /** @type {any} */ ({ + ...IDENTITY, + issues: [], + conflicts: [], + compared: {status: 'checked', findings: [{severity: 'warning'}, {}]}, + typed: { + status: 'skipped', + reason: 'TypeScript is not installed', + findings: [], + }, + notAReport: {status: 'checked'}, + }); + expect( + summarizeCheck({type: 'integration.component-conflicts', data}), + ).toEqual({ + errors: 0, + warnings: 2, + skipped: ['TypeScript is not installed'], + failed: false, + }); + expect(isCheckReport(data.compared)).toBe(true); + expect(isCheckReport(data.notAReport)).toBe(false); + }); + + it('names prop types the comparison could not judge as skipped work, which strict fails and the default run shows', () => { + const reason = + 'TypeScript 4.7.4 is older than 4.8, the oldest the type comparison reads'; + /** @type {any[]} */ + const responses = [ + { + type: 'integration.validate', + data: {...IDENTITY, issues: []}, + }, + { + type: 'integration.component-conflicts', + data: { + ...IDENTITY, + issues: [], + conflicts: [], + propDrift: {status: 'checked', findings: [], typesSkipped: reason}, + }, + }, + { + type: 'integration.template-conflicts', + data: {...IDENTITY, issues: [], conflicts: []}, + }, + { + type: 'integration.doc-conflicts', + data: {...IDENTITY, issues: [], findings: []}, + }, + ]; + const message = `prop types not judged: ${reason}`; + const loose = combineChecks(responses, false); + expect(loose.skipped).toEqual([`components: ${message}`]); + expect(loose.failures).toEqual([]); + expect(loose.failed).toBe(false); + const strict = combineChecks(responses, true); + expect(strict.failures).toEqual([ + { + check: 'components', + severity: 'error', + code: 'prop_types_skipped', + message, + }, + ]); + expect(strict.failed).toBe(true); + }); + + it('names a skipped comparison by its key in snake case', () => { + expect(skippedCode('propDrift')).toBe('prop_drift_skipped'); + expect(skippedCode('docTypes')).toBe('doc_types_skipped'); + }); +}); + +/** + * What the default run reports as warnings and skipped work, read from each + * check's own envelope: what `--strict` must report again as errors. A later + * check's repeat of one of validate's items is validate's, counted once. + * @param {any} result an `integration.check` envelope + */ +function warningsAndSkips(result) { + /** @type {Map} */ + const fromValidate = new Map(); + /** @type {Array<{check: string, code: string, message: string}>} */ + const found = []; + result.data.checks.forEach((/** @type {any} */ {check, response}, i) => { + const {data} = response; + /** @type {any[]} */ + const reports = Object.values(data).filter(isCheckReport); + const items = [ + ...data.issues + .filter((/** @type {any} */ x) => x.severity === 'warning') + .map((/** @type {any} */ x) => [x.code, x.message]), + ...(data.conflicts ?? []) + .filter((/** @type {any} */ x) => x.severity === 'warning') + .map((/** @type {any} */ x) => [CONFLICT_CODES[check], x.message]), + ...reports.flatMap(report => + report.findings.map((/** @type {any} */ f) => [ + f.code ?? f.problem ?? f.kind, + f.message, + ]), + ), + ...(data.validated ? [] : [[NO_MANIFEST_CODE, NO_MANIFEST_SKIP]]), + ...Object.entries(data) + .filter(([, v]) => isCheckReport(v) && v.status === 'skipped') + .map(([key, v]) => [skippedCode(key), v.reason]), + ...reports + .filter(report => typeof report.typesSkipped === 'string') + .map(report => [ + 'prop_types_skipped', + `prop types not judged: ${report.typesSkipped}`, + ]), + ]; + const repeats = new Map(fromValidate); + for (const [code, message] of items) { + const key = `${code}\0${message}`; + const left = repeats.get(key) ?? 0; + if (i === 0) fromValidate.set(key, left + 1); + else if (left > 0) { + repeats.set(key, left - 1); + continue; + } + found.push({check, code, message}); + } + }); + return found; +} + +describe('combineChecks', () => { + const respond = (/** @type {string} */ type, /** @type {object} */ data) => + /** @type {any} */ ({ + type, + data: {...IDENTITY, issues: [], conflicts: [], findings: [], ...data}, + }); + const sameMessage = (/** @type {string} */ component) => ({ + component, + prop: 'label', + severity: 'warning', + kind: 'required_mismatch', + message: + '"label" is required in the code, but the doc does not mark it required.', + doc: 'components/List.doc.mjs', + }); + const responses = [ + respond('integration.validate', { + issues: [ + { + code: 'unknown_manifest_key', + severity: 'warning', + message: 'Unknown key.', + }, + ], + }), + respond('integration.component-conflicts', { + issues: [ + { + code: 'unknown_manifest_key', + severity: 'warning', + message: 'Unknown key.', + }, + ], + report: { + status: 'checked', + findings: [sameMessage('List'), sameMessage('ListItem')], + unchecked: [ + {component: 'Ghost', doc: 'components/Ghost.doc.mjs', reason: 'x'}, + ], + }, + }), + respond('integration.template-conflicts', {}), + respond('integration.doc-conflicts', {}), + ]; + + it('counts a package-level issue once, and two findings with one message twice, naming each component', () => { + const loose = combineChecks(responses, false); + const strict = combineChecks(responses, true); + + expect(loose).toMatchObject({ + errors: 0, + warnings: 3, + unchecked: 1, + failed: false, + }); + expect(strict.warnings).toBe(3); + expect( + strict.failures.map(f => [f.check, f.code, f.component ?? null]), + ).toEqual([ + ['validate', 'unknown_manifest_key', null], + ['components', 'required_mismatch', 'List'], + ['components', 'required_mismatch', 'ListItem'], + ]); + expect(strict.failures[1]).toMatchObject({ + prop: 'label', + file: 'components/List.doc.mjs', + }); + }); + + it('counts each finding a check repeats within itself, and consumes only validate’s own issues as repeats', () => { + const issue = (/** @type {string} */ code, severity = 'warning') => ({ + code, + severity, + message: code, + }); + const root = issue('invalid_root', 'error'); + const link = issue('invalid_doc_graph'); + const broken = issue('bad_link', 'error'); + const repeated = ['validate', 'component', 'template', 'doc'].map(kind => + respond( + `integration.${kind === 'validate' ? kind : `${kind}-conflicts`}`, + { + issues: kind === 'doc' ? [root, link, link, broken, broken] : [root], + }, + ), + ); + + const loose = combineChecks(repeated, false); + const strict = combineChecks(repeated, true); + + expect(loose.checks.map(c => `${c.errors}/${c.warnings}`)).toEqual([ + '1/0', + '1/0', + '1/0', + '3/2', + ]); + expect(loose).toMatchObject({errors: 3, warnings: 2}); + expect(strict.failures.map(f => [f.check, f.code])).toEqual([ + ['docs', 'invalid_doc_graph'], + ['docs', 'invalid_doc_graph'], + ]); + }); + + it('states each option’s default in its reference doc', async () => { + const {doc} = await import('../integrationCheck.doc.mjs'); + const params = new Map(doc.params?.map(param => [param.name, param])); + expect(params.get('options.strict')?.default).toBe('false'); + expect(params.get('options.cwd')?.default).toBe('process.cwd()'); + }); +}); + +describe('integrationCheck', () => { + /** @type {string} */ + let pkg; + + beforeEach(() => { + // Under the repository, so the components check resolves Core. + pkg = fs.mkdtempSync( + path.join(process.cwd(), '.astryx-integration-check-'), + ); + fs.writeFileSync( + path.join(pkg, 'package.json'), + JSON.stringify({name: '@acme/widgets', version: '1.2.3'}), + ); + fs.writeFileSync( + path.join(pkg, 'astryx.integration.mjs'), + "export default {templates: './templates'};\n", + ); + fs.mkdirSync(path.join(pkg, 'templates')); + fs.writeFileSync( + path.join(pkg, 'templates', 'acme-page.doc.mjs'), + "export default {type: 'page', name: 'Acme page', displayName: 'Acme page', description: 'fixture', replaces: 'missing-core-page'};\n", + ); + fs.writeFileSync( + path.join(pkg, 'templates', 'acme-page.tsx'), + 'export default function Fixture() { return null; }\n', + ); + }); + + afterEach(() => { + fs.rmSync(pkg, {recursive: true, force: true}); + }); + + it('returns each check’s own envelope, unchanged, in the order the checks run', async () => { + const result = await integrationCheck({cwd: pkg}); + + expect(result.type).toBe('integration.check'); + expect(result.data.checks.map(check => check.response)).toEqual([ + await validateIntegration(undefined, {cwd: pkg}), + await integrationComponentConflicts(undefined, {cwd: pkg}), + await integrationTemplateConflicts(undefined, {cwd: pkg}), + await integrationDocConflicts(undefined, {cwd: pkg}), + ]); + }); + + it('fails when a check fails on its own, with or without strict', async () => { + const result = await integrationCheck({cwd: pkg}); + + const templates = result.data.checks.find( + check => check.check === 'templates', + ); + expect(templates?.failed).toBe(true); + expect(result.data).toMatchObject({ + validated: true, + strict: false, + failures: [], + failed: true, + }); + expect(result.data.errors).toBeGreaterThan(0); + }); + + it('fails on warnings only with strict, which reports exactly the default run’s warnings and skipped work again, as errors', async () => { + const core = (await discoverCoreTemplates()).find( + template => template.type === 'page', + ); + fs.writeFileSync( + path.join(pkg, 'astryx.integration.mjs'), + "export default {templates: './templates', unknownKey: true};\n", + ); + fs.rmSync(path.join(pkg, 'templates', 'acme-page.doc.mjs')); + fs.renameSync( + path.join(pkg, 'templates', 'acme-page.tsx'), + path.join(pkg, 'templates', `${core?.dirName}.tsx`), + ); + fs.writeFileSync( + path.join(pkg, 'templates', `${core?.dirName}.doc.mjs`), + "export default {type: 'page', name: 'Acme page', displayName: 'Acme page', description: 'fixture'};\n", + ); + + const loose = await integrationCheck({cwd: pkg}); + const strict = await integrationCheck({cwd: pkg, strict: true}); + + expect(loose.data).toMatchObject({errors: 0, failures: [], failed: false}); + // The unknown key is validate's, and every check starts from validate's + // issues: it counts once. + expect(loose.data.warnings).toBe(2); + expect(strict.data).toMatchObject({strict: true, errors: 0, failed: true}); + expect(strict.data.warnings).toBe(loose.data.warnings); + // Same data: the same checks, read once each, in both modes. + expect(strict.data.checks.map(check => check.response)).toEqual( + loose.data.checks.map(check => check.response), + ); + // Only the severity changes. + expect(new Set(strict.data.failures.map(f => f.severity))).toEqual( + new Set(['error']), + ); + expect( + strict.data.failures.map(({check, code, message}) => ({ + check, + code, + message, + })), + ).toEqual(warningsAndSkips(loose)); + expect(strict.data.failures.map(f => [f.check, f.code])).toEqual([ + ['validate', 'unknown_manifest_key'], + ['templates', 'template_id_conflict'], + ]); + }); + + it('counts a missing root once, though every check reports it', async () => { + fs.rmSync(path.join(pkg, 'templates'), {recursive: true, force: true}); + + const result = await integrationCheck({cwd: pkg}); + + expect(result.data.checks.map(check => check.failed)).toEqual([ + true, + true, + true, + true, + ]); + expect(result.data.errors).toBe(1); + expect(result.data.failed).toBe(true); + }); + + it('reads a missing package.json from validate alone', async () => { + const empty = fs.mkdtempSync(path.join(os.tmpdir(), 'astryx-no-pkg-')); + try { + const result = await integrationCheck({cwd: empty, strict: true}); + + expect(result.data).toMatchObject({ + errors: 1, + skipped: [], + failures: [], + failed: true, + }); + expect(result.data.checks.map(check => check.skipped)).toEqual([ + [], + [], + [], + [], + ]); + } finally { + fs.rmSync(empty, {recursive: true, force: true}); + } + }); +}); diff --git a/packages/cli/api/integration/integration-check.type.mjs b/packages/cli/api/integration/integration-check.type.mjs new file mode 100644 index 0000000000000..82d3816a87fa4 --- /dev/null +++ b/packages/cli/api/integration/integration-check.type.mjs @@ -0,0 +1,84 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +/** + * @file Typedefs for `integrationCheck()` / `astryx integration check`. + */ + +/** + * Options for `integrationCheck()`. + * @typedef {object} IntegrationCheckOptions + * @property {string} [cwd] The integration package folder to check; the + * current directory by default. + * @property {boolean} [strict] Also fail on any warning and on any check that + * did not run, not only on errors. + */ + +/** + * The checks `integrationCheck()` runs, in the order it runs them. + * @typedef {'validate' | 'components' | 'templates' | 'docs'} IntegrationCheckName + */ + +/** + * The envelope one check returns on its own, unchanged. + * @typedef {import('./validate-integration.type.mjs').ValidateIntegrationResponse + * | import('./authoring-checks.type.mjs').IntegrationComponentConflictResponse + * | import('./authoring-checks.type.mjs').IntegrationTemplateConflictResponse + * | import('./authoring-checks.type.mjs').IntegrationDocConflictResponse} IntegrationCheckResponse + */ + +/** + * A comparison a check carries beside its issues, such as one that needs the + * project's own TypeScript. `skipped` means it did not run, and `reason` says + * why; each of its findings is a warning. + * @typedef {object} IntegrationCheckReport + * @property {'checked' | 'skipped'} status + * @property {string} [reason] + * @property {Array<{severity?: string}>} findings + */ + +/** + * One check's result, counted the way its own subcommand exits. + * @typedef {object} IntegrationCheckResult + * @property {IntegrationCheckName} check + * @property {number} errors What makes the check's own subcommand exit 1. + * @property {number} warnings Warning issues, conflicts and findings, and the + * findings of every report the check carries. + * @property {string[]} skipped Why work did not run: no integration manifest, + * or a report the check carries that was skipped. + * @property {boolean} failed Whether the check's own subcommand exits 1. + * @property {IntegrationCheckResponse} response The check's own envelope. + */ + +/** + * A warning, or work a check skipped, as `--strict` reports it: an error in + * the checks' own issue format, with the finding's own code (a conflict's is + * `component_name_conflict` or `template_id_conflict`, a package with no + * manifest's is `missing_manifest`, and a skipped comparison's is its key in + * snake case with `_skipped`), message, and its fix, file, and line where it + * has them. + * @typedef {object} IntegrationCheckFailure + * @property {IntegrationCheckName} check + * @property {'error'} severity + * @property {string} code + * @property {string} message + * @property {string} [component] the component a finding is about + * @property {string} [prop] the prop a finding is about + * @property {string} [fix] + * @property {string} [file] + * @property {number} [line] + */ + +/** + * `astryx --json integration check [--strict]`. + * + * `failed` decides the exit code: any check that fails on its own, and with + * `strict` also any warning or skipped work, which `failures` then lists as + * errors (it is empty without `strict`). + * @typedef {object} IntegrationCheckCommandResponse + * @property {'integration.check'} type + * `unchecked` counts the items the checks list as not compared, which never + * fail it. + * @property {{validated: boolean, name: string | null, version: string | null, strict: boolean, checks: IntegrationCheckResult[], errors: number, warnings: number, skipped: string[], unchecked: number, failures: IntegrationCheckFailure[], failed: boolean}} data + */ + +export {}; diff --git a/packages/cli/api/integration/integrationCheck.doc.mjs b/packages/cli/api/integration/integrationCheck.doc.mjs new file mode 100644 index 0000000000000..3bd450c052e13 --- /dev/null +++ b/packages/cli/api/integration/integrationCheck.doc.mjs @@ -0,0 +1,73 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +/** + * @file FunctionDoc for `integrationCheck()` / `astryx integration check`. + * Colocated with the API function it documents; the shape source of truth stays + * in `integration-check.type.mjs`. + */ + +/** @type {import('@astryxdesign/cli/authoring').FunctionDoc} */ +export const doc = { + type: 'function', + kind: 'api', + name: 'integrationCheck', + namespace: 'cli/api', + displayName: 'integrationCheck()', + summary: + 'Run every source check against the integration package in the current folder, as its build does: validate, components, templates, and docs.', + description: + 'Runs `validateIntegration`, `integrationComponentConflicts`, ' + + '`integrationTemplateConflicts`, and `integrationDocConflicts` against the ' + + 'integration package in `options.cwd`, and counts each result the way that ' + + "check's own subcommand exits. The run fails when any of the four would fail on " + + 'its own. With `strict` it also fails on any warning and on any check that did ' + + 'not run: a package with no integration manifest, or a comparison a check ' + + "carries that was skipped, such as one that needs the project's TypeScript. " + + '`strict` adds no check: it reports those warnings and skipped work again, as ' + + "errors in the checks' own issue format. " + + 'A new check, or a new failure code, joins only in a minor release, named in the release notes.', + importPath: '@astryxdesign/cli/api', + signature: + 'integrationCheck(options?: IntegrationCheckOptions): Promise', + keywords: ['integration', 'check', 'build', 'strict', 'validate', 'doctor'], + params: [ + { + name: 'options.cwd', + type: 'string', + default: 'process.cwd()', + description: 'The integration package folder to check.', + }, + { + name: 'options.strict', + type: 'boolean', + default: 'false', + description: + 'Also fail on any warning and on any check that did not run, not only on errors.', + }, + ], + returns: [ + { + type: 'integration.check', + description: + "`data.validated` (false when no integration manifest was found, so nothing was checked), `data.name` and `data.version` of the package, `data.strict`, and `data.checks`, one entry per check in the order they run (`validate`, `components`, `templates`, `docs`), each with `check`, `errors` (what makes that check's own subcommand exit 1), `warnings`, `skipped` (why work did not run), `failed` (whether that subcommand exits 1), and `response`, the envelope the check returns on its own. The totals are `data.errors`, `data.warnings`, and `data.skipped` (each reason prefixed with its check); an issue every check starts from, such as one about the manifest, counts once, and every other finding counts on its own. `data.unchecked` counts the items the checks list as not compared, which never fail it. With `strict`, `data.failures` lists each warning and each piece of skipped work once again as an error in the checks' own issue format ({check, severity: `error`, code, message, component, prop, fix, file, line}, each where it applies); it is empty without `strict`. `data.failed` decides the exit code.", + }, + ], + examples: [ + { + label: 'Check the local package', + code: 'const r = await integrationCheck();', + }, + { + label: 'Fail on warnings too', + code: 'const {data} = await integrationCheck({strict: true});\nif (data.failed) process.exitCode = 1;', + }, + ], + command: 'integration check', + related: [ + 'validateIntegration', + 'integrationComponentConflicts', + 'integrationTemplateConflicts', + 'integrationDocConflicts', + 'integrationPackCheck', + ], +}; diff --git a/packages/cli/api/search/search.integration-check.test.mjs b/packages/cli/api/search/search.integration-check.test.mjs new file mode 100644 index 0000000000000..386fa4d399e93 --- /dev/null +++ b/packages/cli/api/search/search.integration-check.test.mjs @@ -0,0 +1,24 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +/** + * @file A reader looking for the check a build runs finds + * `astryx integration check` first. + */ + +import {describe, expect, it} from 'vitest'; +import {search} from './search.mjs'; + +describe('search finds integration check for CI', () => { + it.each([ + 'ci', + 'continuous integration', + 'typecheck integration', + 'integration check', + ])('`%s` ranks it first', async query => { + const {results} = (await search(query)).data; + expect(results[0], JSON.stringify(results.slice(0, 3))).toMatchObject({ + domain: 'doc', + name: 'cli/commands/integration-check', + }); + }); +}); diff --git a/packages/cli/assets/docs/tree/checks.doc.mjs b/packages/cli/assets/docs/tree/checks.doc.mjs index 0d6beba5866d2..1c083195e0543 100644 --- a/packages/cli/assets/docs/tree/checks.doc.mjs +++ b/packages/cli/assets/docs/tree/checks.doc.mjs @@ -12,7 +12,7 @@ export const docs = { placement: {parent: 'namespace:ship', slot: 'guides', order: 20}, title: 'Check an integration', category: 'guide', - keywords: ['check before publishing', 'validate an integration', 'ci'], + keywords: ['check before publishing', 'validate an integration'], description: 'Pick the check for each problem, learn what fails and what only warns, and run every check in CI.', sections: [ @@ -22,7 +22,7 @@ export const docs = { content: [ { type: 'prose', - text: 'Run these in the package folder. The four `doctor integration` checks read your source; `integration verify` checks the package npm would publish.', + text: 'Run these in the package folder. The four `doctor integration` checks read your source, and `integration check` runs all four; `integration verify` checks the package npm would publish.', }, { type: 'table', @@ -52,6 +52,12 @@ export const docs = { 'A topic takes a Core topic name without `replaces` or `extends`, a doc is invalid (`invalid_doc`), or a namespace or placement fails, which hides the doc (`invalid_doc_graph`)', 'A link names no doc (`invalid_doc_graph`)', ], + [ + '`npx astryx integration check`', + 'Everything the four `doctor integration` checks prove, in one command, for your build', + 'Any of the four would exit 1 on its own. With `--strict`, also any warning, and any check that did not run, such as a package with no `astryx.integration.*`', + 'Anything the four warn about, unless you pass `--strict`', + ], [ '`npx astryx integration verify`', 'The packed package holds every file, shows the same contributions, resolves every public import, and declares the CLI it needs', @@ -72,19 +78,21 @@ export const docs = { content: [ { type: 'prose', - text: 'Chain the five checks so the first failure stops the run. Install devDependencies first, because the components check needs Core.', + text: "Run `integration check --strict` in your package's own build, so every build keeps your docs, types, and links in sync, and run `integration verify` before you publish. Install devDependencies first, because the components check needs Core, and `tsc --noEmit` needs `typescript` and a `tsconfig.json`. In package.json:", }, { type: 'code', - lang: 'bash', - code: 'npx astryx doctor integration validate && npx astryx doctor integration components && npx astryx doctor integration templates && npx astryx doctor integration docs && npx astryx integration verify', + lang: 'json', + code: '"typecheck": "tsc --noEmit && astryx integration check --strict"', }, { type: 'list', style: 'unordered', items: [ - '`integration verify` runs `validate` but none of the other three: a Core name clash, a `replaces` that names no Core template, a topic that takes a Core name, or a hidden guide still passes it.', - 'Warnings keep exit code 0, so read them before you publish.', + '`integration check` runs `validate`, `components`, `templates`, and `docs` in that order, prints each result, then a summary, and exits 1 when any of them would.', + '`--strict` also exits 1 on any warning and on any check that did not run. Without it, warnings keep exit code 0: use that while you fix the warnings a package already has.', + 'Its exit codes and JSON are a contract for your build: a new check, or a new failure code, joins only in a minor release that names it in the release notes. A patch can still ship a Core component, template, or CLI topic whose name conflicts with one of yours.', + '`integration verify` runs `validate` but none of the other three: a Core name clash, a `replaces` that names no Core template, a topic that takes a Core name, or a hidden guide still passes it. Run both before you publish: `npx astryx integration check --strict && npx astryx integration verify`.', 'Bare `npx astryx doctor` in the package also warns about a doc section over 32 KB, which no check above measures.', ], }, diff --git a/packages/cli/assets/docs/tree/quick-start.doc.mjs b/packages/cli/assets/docs/tree/quick-start.doc.mjs index 3275f811476cb..445803d3cc0ea 100644 --- a/packages/cli/assets/docs/tree/quick-start.doc.mjs +++ b/packages/cli/assets/docs/tree/quick-start.doc.mjs @@ -184,7 +184,7 @@ export const docs = { }, { type: 'prose', - text: 'Peer ranges and releases are covered in {@link generic:versioning}; `files` and publishing in {@link generic:publishing}.', + text: 'Peer ranges and releases are covered in {@link generic:versioning}; `files` and publishing in {@link generic:publishing}. Add `"typecheck": "tsc --noEmit && astryx integration check --strict"` to `scripts` so each run checks your types and that your docs, props, and links still match the code. It needs `typescript` and a `tsconfig.json`; see {@link generic:checks}.', }, ], }, diff --git a/packages/cli/clients/cli/commands/doctor.mjs b/packages/cli/clients/cli/commands/doctor.mjs index 5e1b11a83e59f..9c5ea67ab11d8 100644 --- a/packages/cli/clients/cli/commands/doctor.mjs +++ b/packages/cli/clients/cli/commands/doctor.mjs @@ -415,6 +415,23 @@ function printDocConflicts(data) { emit(...output); } +/** + * Print one `doctor integration` check's result as its own subcommand prints + * it. `astryx integration check` prints each check it runs through this. + * @param {import('../../../api/integration/integration-check.type.mjs').IntegrationCheckResponse} response + */ +export function printIntegrationCheckResponse(response) { + if (response.type === 'integration.validate') { + printIntegrationValidation(response.data); + } else if (response.type === 'integration.component-conflicts') { + printComponentConflicts(response.data); + } else if (response.type === 'integration.template-conflicts') { + printTemplateConflicts(response.data); + } else { + printDocConflicts(response.data); + } +} + /** @param {import('commander').Command} program */ async function runProjectDoctor(program) { const report = await runChecks(); diff --git a/packages/cli/clients/cli/commands/integration-check.doc.mjs b/packages/cli/clients/cli/commands/integration-check.doc.mjs new file mode 100644 index 0000000000000..1fc36f2491c46 --- /dev/null +++ b/packages/cli/clients/cli/commands/integration-check.doc.mjs @@ -0,0 +1,85 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +/** + * @file CommandDoc for `astryx integration check`. + */ + +/** @type {import('@astryxdesign/cli/authoring').CommandDoc} */ +export const doc = { + type: 'command', + name: 'integration check', + displayName: 'astryx integration check', + namespace: 'cli/commands', + summary: + "The CI check for an integration package's own build: every source check, run in continuous integration or a typecheck script, against the package in the current folder", + description: + 'Takes no package argument: it runs the four `doctor integration` checks, ' + + '`validate`, `components`, `templates`, and `docs`, against the integration ' + + 'package in the current folder, which is where its own build runs it. It prints ' + + 'each result as that check ' + + 'prints it, then a summary with the errors, warnings, and skipped work of each, ' + + 'and exits 1 when any of the four would exit 1 on its own. With `--strict` it ' + + 'also exits 1 on any warning and on any check, or comparison within a check, ' + + 'that did not run, such as a package with no `astryx.integration.*` or a ' + + 'comparison skipped because the project has no TypeScript, or prop types not ' + + 'judged because its TypeScript is older than 4.8: it adds no check, ' + + 'and lists each of those again as an error, after the checks, in their own ' + + 'issue format. That is the line ' + + "for the package's own build, so its docs, types, and links stay in sync: run " + + 'it as a `ci` step, or add ' + + '`"typecheck": "tsc --noEmit && astryx integration check --strict"` to the ' + + 'scripts in package.json. Its exit codes and JSON are a contract for the builds ' + + 'that run it: a new check, or a new failure code, joins only in a minor release ' + + 'that names it in the release notes. A patch can still ship a Core component, ' + + 'template, or CLI topic whose name conflicts with one of yours. Run ' + + '`integration check` on every build; run ' + + '`integration verify` before you publish, to check that the packed package ' + + 'works in an app. To check an installed integration by name, run the ' + + '`doctor integration` subcommands with its package name.', + fn: 'integrationCheck', + options: [ + { + flag: '--strict', + param: 'options.strict', + description: + 'Also exit 1 on any warning and on any check that did not run. A new check or failure code joins only in a minor release, named in the release notes.', + }, + ], + examples: [ + { + label: + 'In CI, as the typecheck script: add "typecheck": "tsc --noEmit && astryx integration check --strict" to package.json', + cli: 'integration check --strict', + }, + { + label: 'Errors only, while you fix the warnings a package already has', + cli: 'integration check', + }, + { + label: 'Read the result as JSON', + cli: 'integration check --strict --json', + }, + { + label: + 'Before you publish: check that the packed package works in an app', + cli: 'integration verify', + }, + ], + exitCodes: [ + { + code: 0, + when: 'no check fails on its own; with --strict, also no warning, and every check ran', + }, + { + code: 1, + when: 'a check fails on its own (an error); with --strict, also any warning, or a check that did not run', + }, + ], + related: [ + 'integration verify', + 'doctor integration validate', + 'doctor integration components', + 'doctor integration templates', + 'doctor integration docs', + ], +}; diff --git a/packages/cli/clients/cli/commands/integration-check.test.mjs b/packages/cli/clients/cli/commands/integration-check.test.mjs new file mode 100644 index 0000000000000..a6068022f82a6 --- /dev/null +++ b/packages/cli/clients/cli/commands/integration-check.test.mjs @@ -0,0 +1,267 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +/** + * @file `astryx integration check`: every source check in one command, the + * summary, exit codes with and without `--strict`, and the `--strict` + * failures, in text and JSON alike. + */ + +import {afterEach, beforeEach, describe, expect, it} from 'vitest'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import {runCli} from '../../../test-utils/run-cli.mjs'; +import {discoverCoreTemplates} from '../../../foundation/discovery/template-adapter.mjs'; + +/** @type {string} */ +let tmpDir; + +beforeEach(() => { + // Under the repository, so the components check resolves Core. + tmpDir = fs.mkdtempSync( + path.join(process.cwd(), '.astryx-integration-check-cli-'), + ); +}); + +afterEach(() => { + fs.rmSync(tmpDir, {recursive: true, force: true}); +}); + +/** + * A package whose one page template takes a Core template's id: one warning + * (`templates` conflict). The doc matches TemplateDoc, so no other check + * warns about it. + * @param {{id: string, replaces?: string, extraManifest?: string, dir?: string}} options + */ +function writePackage({id, replaces, extraManifest = '', dir = tmpDir}) { + fs.writeFileSync( + path.join(dir, 'package.json'), + JSON.stringify({name: '@acme/widgets', version: '1.2.3'}), + ); + fs.writeFileSync( + path.join(dir, 'astryx.integration.mjs'), + `export default {templates: './templates'${extraManifest}};\n`, + ); + fs.mkdirSync(path.join(dir, 'templates'), {recursive: true}); + fs.writeFileSync( + path.join(dir, 'templates', `${id}.doc.mjs`), + `export default {type: 'page', name: 'Integration template', displayName: 'Integration template', description: 'fixture'${replaces == null ? '' : `, replaces: ${JSON.stringify(replaces)}`}};\n`, + ); + fs.writeFileSync( + path.join(dir, 'templates', `${id}.tsx`), + 'export default function Fixture() { return null; }\n', + ); +} + +async function coreTemplateId() { + const core = (await discoverCoreTemplates()).find( + template => template.type === 'page', + ); + expect(core).toBeDefined(); + return /** @type {{dirName: string}} */ (core).dirName; +} + +describe('astryx integration check', () => { + it('runs validate, components, templates, and docs in order, then a summary, and exits 0 on a warning', async () => { + writePackage({id: await coreTemplateId()}); + + const run = await runCli(['integration', 'check'], tmpDir); + + const order = [ + 'Validating integration: @acme/widgets@1.2.3', + 'Checking integration components: @acme/widgets@1.2.3', + 'Checking integration templates: @acme/widgets@1.2.3', + 'Checking integration docs: @acme/widgets@1.2.3', + 'Integration check: @acme/widgets@1.2.3', + ].map(line => run.stdout.indexOf(line)); + expect(order.every(index => index >= 0)).toBe(true); + expect([...order].sort((a, b) => a - b)).toEqual(order); + expect(run.stdout).toContain('Consider renaming'); + expect(run.stdout).not.toContain('Failing under --strict'); + expect(run.stdout).toContain( + '[ok] 0 errors, 1 warning, 0 skipped. No check failed;', + ); + // Each check's output stands apart from the next. + const lines = run.stdout.split('\n'); + for (const header of [ + 'Checking integration components:', + 'Checking integration templates:', + 'Checking integration docs:', + ]) { + expect(lines[lines.findIndex(line => line.includes(header)) - 1]).toBe( + '', + ); + } + expect(run.status).toBe(0); + }); + + it('--strict lists the warning again as an error, after the checks, and exits 1', async () => { + const id = await coreTemplateId(); + writePackage({id}); + + const run = await runCli(['integration', 'check', '--strict'], tmpDir); + + expect(run.stdout).toContain( + 'Integration check: @acme/widgets@1.2.3 (strict)', + ); + const block = run.stdout.indexOf('Failing under --strict'); + expect(block).toBeGreaterThan( + run.stdout.indexOf('Checking integration docs:'), + ); + expect(run.stdout.slice(block)).toMatch( + /severity: \[fail\]\ncheck: +templates\ncode: +template_id_conflict\n/, + ); + expect(run.stdout).toContain( + '[fail] 0 errors, 1 warning, 0 skipped. --strict fails on every warning and on skipped work;', + ); + expect(run.status).toBe(1); + }); + + it('--json carries each check with its own envelope, the --strict failures, and exits as the text does', async () => { + const id = await coreTemplateId(); + writePackage({id}); + + const run = await runCli( + ['integration', 'check', '--strict', '--json'], + tmpDir, + ); + + const parsed = JSON.parse(run.stdout.trim()); + expect(parsed.type).toBe('integration.check'); + expect(parsed.data).toMatchObject({ + validated: true, + name: '@acme/widgets', + version: '1.2.3', + strict: true, + errors: 0, + warnings: 1, + skipped: [], + failed: true, + }); + expect(parsed.data.checks.map(check => check.check)).toEqual([ + 'validate', + 'components', + 'templates', + 'docs', + ]); + const templates = parsed.data.checks[2]; + expect(templates).toMatchObject({ + check: 'templates', + errors: 0, + warnings: 1, + skipped: [], + failed: false, + response: { + type: 'integration.template-conflicts', + data: {conflicts: [expect.objectContaining({id, severity: 'warning'})]}, + }, + }); + expect(parsed.data.failures).toEqual([ + { + check: 'templates', + severity: 'error', + code: 'template_id_conflict', + message: templates.response.data.conflicts[0].message, + }, + ]); + expect(run.status).toBe(1); + }); + + it('exits 1 without --strict when one check fails on its own, and lists no failures', async () => { + writePackage({id: 'acme-app-shell', replaces: 'missing-core-shell'}); + + const text = await runCli(['integration', 'check'], tmpDir); + const json = await runCli(['integration', 'check', '--json'], tmpDir); + + expect(text.stdout).toContain('missing_template_replacement_target'); + expect(text.stdout).toMatch( + /\[fail\] [1-9]\d* errors?, .*Address the items marked \[fail\] above\./, + ); + expect(text.stdout).not.toContain('Failing under --strict'); + expect(JSON.parse(json.stdout.trim()).data).toMatchObject({ + strict: false, + failures: [], + failed: true, + }); + expect(text.status).toBe(1); + expect(json.status).toBe(1); + }); + + it('counts a package with no manifest as skipped work: exit 0, and 1 with --strict', async () => { + fs.writeFileSync( + path.join(tmpDir, 'package.json'), + JSON.stringify({name: '@acme/plain', version: '1.0.0'}), + ); + + const loose = await runCli(['integration', 'check'], tmpDir); + expect(loose.stdout).toContain( + "Nothing was checked: no astryx.integration.* found next to package.json. Run this command in an integration package's folder.", + ); + // One cause, however many checks it stops: counted once. + expect(loose.stdout).toContain('[ok] 0 errors, 0 warnings, 1 skipped.'); + expect(loose.status).toBe(0); + + const strict = await runCli( + ['integration', 'check', '--strict', '--json'], + tmpDir, + ); + const {data} = JSON.parse(strict.stdout.trim()); + expect(data).toMatchObject({validated: false, errors: 0, failed: true}); + // One cause, however many checks it stops: listed once. + expect(data.failures).toEqual([ + { + check: 'validate', + severity: 'error', + code: 'missing_manifest', + message: + 'No astryx.integration.* found next to package.json, so nothing was checked.', + }, + ]); + expect(strict.status).toBe(1); + }); + + it('exits 1 for no package.json, as validate does', async () => { + const empty = fs.mkdtempSync(path.join(os.tmpdir(), 'astryx-no-pkg-')); + try { + const run = await runCli(['integration', 'check'], empty); + expect(run.stdout).toContain( + 'Nothing was checked: no package.json found.', + ); + expect(run.stdout).toContain('Integration check: (no package.json)'); + expect(run.stdout).toContain( + '[fail] 1 error, 0 warnings, 0 skipped. No package.json was found here, so nothing was checked.', + ); + expect(run.stdout).not.toContain('marked [fail] above'); + expect(run.status).toBe(1); + } finally { + fs.rmSync(empty, {recursive: true, force: true}); + } + }); + + it('states in its help that it is the CI check, the line to add, and what --strict promises', async () => { + const run = await runCli(['integration', 'check', '--help'], tmpDir); + + expect(run.stdout).toContain( + "The CI check for an integration package's own build", + ); + expect(run.stdout).toContain( + '"typecheck": "tsc --noEmit && astryx integration check --strict"', + ); + expect(run.stdout).toContain( + 'A new check or failure code joins only in a minor release, named in the release notes.', + ); + expect(run.stdout).toContain('integration verify'); + expect(run.stdout).toContain('against the package in the current folder'); + expect(run.stdout).not.toContain('[package]'); + }); + + it('takes no package name: one passed is an error, not a check of the current folder', async () => { + writePackage({id: 'acme-page'}); + + const run = await runCli(['integration', 'check', '@acme/widgets'], tmpDir); + + expect(run.stderr).toContain('too many arguments'); + expect(run.stdout).not.toContain('Integration check:'); + expect(run.status).toBe(1); + }); +}); diff --git a/packages/cli/clients/cli/commands/integration.doc.mjs b/packages/cli/clients/cli/commands/integration.doc.mjs index 2d3a546c6478a..1259c684db44b 100644 --- a/packages/cli/clients/cli/commands/integration.doc.mjs +++ b/packages/cli/clients/cli/commands/integration.doc.mjs @@ -8,13 +8,17 @@ export const doc = { namespace: 'cli/commands', summary: 'Author and verify an Astryx integration package', description: - 'Add contributions to your package, then check the packed package the way an app receives it. The guides start at {@link namespace:integrations}.', - subcommands: ['add', 'verify', 'pack'], + 'Add contributions to your package, check them on every build with `integration check --strict`, then check the packed package the way an app receives it. The guides start at {@link namespace:integrations}.', + subcommands: ['add', 'check', 'verify', 'pack'], examples: [ { label: 'Add a component', cli: 'astryx integration add component AcmeWidget', }, + { + label: 'Check the package in CI, as part of its build', + cli: 'astryx integration check --strict', + }, { label: 'Check the package before publishing', cli: 'astryx integration verify', diff --git a/packages/cli/clients/cli/commands/integration.mjs b/packages/cli/clients/cli/commands/integration.mjs index d77413596164d..754284160ec61 100644 --- a/packages/cli/clients/cli/commands/integration.mjs +++ b/packages/cli/clients/cli/commands/integration.mjs @@ -3,18 +3,22 @@ /** @file CLI bindings for integration authoring and package verification. */ import {integrationAdd} from '../../../api/integration/add-contribution.mjs'; +import {integrationCheck} from '../../../api/integration/check/check.mjs'; import {integrationPackCheck} from '../../../api/integration/pack-check.mjs'; import {jsonOut} from '../../../foundation/response/json.mjs'; import {ERROR_CODES} from '../../../foundation/response/error-codes.mjs'; import {NO_RESULT_SET} from '../../../foundation/debug/index.mjs'; -import {emit, list, section, text} from '../formatters/index.mjs'; +import {emit, list, records, section, text} from '../formatters/index.mjs'; import {cliError} from '../lib/cli-error.mjs'; import {defineCommand} from '../lib/define-command.mjs'; import {doc as integrationGroup} from './integration.doc.mjs'; import {doc as integrationAddCommand} from './integration-add.doc.mjs'; +import {doc as integrationCheckCommand} from './integration-check.doc.mjs'; import {doc as integrationVerifyCommand} from './integration-verify.doc.mjs'; import {doc as integrationPackCommand} from './integration-pack.doc.mjs'; import {doc as integrationAddFn} from '../../../api/integration/integrationAdd.doc.mjs'; +import {doc as integrationCheckFn} from '../../../api/integration/integrationCheck.doc.mjs'; +import {printIntegrationCheckResponse} from './doctor.mjs'; import {doc as integrationPackCheckFn} from '../../../api/integration/integrationPackCheck.doc.mjs'; /** @@ -88,6 +92,102 @@ async function verifyPackage(program) { return NO_RESULT_SET; } +/** + * Run `integration check` and print each check as its own subcommand prints + * it, then one summary: each check's errors, warnings and skipped work, and + * what decided the exit code. + * @param {import('commander').Command} program + * @param {{strict?: boolean}} options + */ +async function checkPackage(program, options) { + const result = await integrationCheck({strict: options.strict === true}); + if (program.opts().json) { + jsonOut(result); + } else { + const {data} = result; + const noPackage = data.checks[0]?.response.data.issues.some( + issue => issue.code === 'no_package', + ); + if (noPackage) { + emit( + text( + 'Nothing was checked: no package.json found. Run this command inside a package directory.', + ), + ); + } else if (!data.validated) { + emit( + text( + 'Nothing was checked: no astryx.integration.* found next to package.json. ' + + "Run this command in an integration package's folder.", + ), + ); + } else { + data.checks.forEach((check, index) => { + // A blank line keeps each check's output apart from the next. + if (index > 0) emit(text('')); + printIntegrationCheckResponse(check.response); + }); + } + const plural = (/** @type {number} */ n, /** @type {string} */ word) => + `${n} ${word}${n === 1 ? '' : 's'}`; + const counts = + `${plural(data.errors, 'error')}, ${plural(data.warnings, 'warning')}, ` + + `${data.skipped.length} skipped`; + const notCompared = + data.unchecked > 0 + ? ` ${plural(data.unchecked, 'item')} could not be compared; the checks above list them.` + : ''; + const closing = noPackage + ? `[fail] ${counts}. No package.json was found here, so nothing was checked.` + : data.checks.some(check => check.failed) + ? `[fail] ${counts}. Address the items marked [fail] above.` + : data.failed + ? `[fail] ${counts}. --strict fails on every warning and on skipped work; fix them, or drop --strict to fail only on errors.` + : data.warnings > 0 || data.skipped.length > 0 + ? `[ok] ${counts}. No check failed; review the warnings and skipped work above (--strict fails on them).${notCompared}` + : notCompared + ? `[ok] ${counts}. No check failed.${notCompared}` + : `[ok] ${counts}. Every check passed.`; + const identity = noPackage + ? '(no package.json)' + : data.version != null + ? `${data.name}@${data.version}` + : data.name; + emit( + // --strict lists each warning and each skipped check again, as the + // error it now is, in the issue format the checks use. + ...(data.failures.length > 0 + ? [ + section('Failing under --strict'), + records(data.failures, { + fields: [ + 'severity', + 'check', + 'code', + 'component', + 'prop', + 'message', + 'fix', + 'file', + 'line', + ], + format: {severity: () => '[fail]'}, + }), + ] + : []), + section( + `Integration check: ${identity ?? '(local package)'}${data.strict ? ' (strict)' : ''}`, + ), + records(data.checks, { + fields: ['check', 'failed', 'errors', 'warnings', 'skipped'], + }), + text(closing), + ); + } + if (result.data.failed) process.exitCode = 1; + return NO_RESULT_SET; +} + /** @param {import('commander').Command} program */ export function registerIntegration(program) { /** @type {import('commander').Command} */ @@ -159,6 +259,12 @@ export function registerIntegration(program) { }); addCmd.allowExcessArguments(false); + // It takes no package name, so a name passed by mistake is an error, not a + // silent check of the current folder. + defineCommand(integration, integrationCheckCommand, { + fn: integrationCheckFn, + action: async options => checkPackage(program, options), + }).allowExcessArguments(false); defineCommand(integration, integrationVerifyCommand, { fn: integrationPackCheckFn, action: async () => verifyPackage(program), diff --git a/packages/cli/clients/cli/commands/text-json-parity.test.mjs b/packages/cli/clients/cli/commands/text-json-parity.test.mjs index ac73647826bb5..453d01b396277 100644 --- a/packages/cli/clients/cli/commands/text-json-parity.test.mjs +++ b/packages/cli/clients/cli/commands/text-json-parity.test.mjs @@ -341,6 +341,11 @@ const CASES = [ args: ['doctor', 'integration', 'validate'], skipFieldChecks: true, }, + { + name: 'integration check', + args: ['integration', 'check'], + skipFieldChecks: true, + }, { name: 'doctor integration components', args: ['doctor', 'integration', 'components'], diff --git a/packages/cli/clients/cli/index.mjs b/packages/cli/clients/cli/index.mjs index d23160456a2d8..5058391a34d68 100644 --- a/packages/cli/clients/cli/index.mjs +++ b/packages/cli/clients/cli/index.mjs @@ -98,6 +98,7 @@ export const JSON_SUPPORTED = new Set([ 'theme targets', 'theme palette generate', 'integration add', + 'integration check', 'integration verify', 'integration pack', 'upgrade', diff --git a/packages/cli/foundation/doc-compiler/doc-loads.test.mjs b/packages/cli/foundation/doc-compiler/doc-loads.test.mjs index 92ccca7370336..630f093b030e5 100644 --- a/packages/cli/foundation/doc-compiler/doc-loads.test.mjs +++ b/packages/cli/foundation/doc-compiler/doc-loads.test.mjs @@ -195,8 +195,10 @@ const RUNNERS = { runs: SELF_DOCS, sites: [ 'import ../../../api/integration/integrationAdd.doc.mjs', + 'import ../../../api/integration/integrationCheck.doc.mjs', 'import ../../../api/integration/integrationPackCheck.doc.mjs', 'import ./integration-add.doc.mjs', + 'import ./integration-check.doc.mjs', 'import ./integration-pack.doc.mjs', 'import ./integration-verify.doc.mjs', 'import ./integration.doc.mjs', diff --git a/packages/cli/foundation/response/response-types.doc.mjs b/packages/cli/foundation/response/response-types.doc.mjs index 41ec57b2280d3..786d25d6ac6ac 100644 --- a/packages/cli/foundation/response/response-types.doc.mjs +++ b/packages/cli/foundation/response/response-types.doc.mjs @@ -349,5 +349,10 @@ export const doc = { description: 'validated (false when no integration manifest was found, so nothing was inspected), the integration identity, structural issues, and Core doc overlaps. Each finding includes `severity` (`info` | `error`) and `relationship` (`replaces` | `extends` | `accidental`).', }, + { + value: 'integration.check', + description: + 'Every source check of an integration package in one result: validated (false when no integration manifest was found, so nothing was checked), the package name and version, strict, and checks, one entry per check (validate, components, templates, docs), each with check, errors, warnings, skipped, failed, and response, the envelope that check returns on its own. The totals are errors, warnings, and skipped; unchecked counts what the checks could not compare. With --strict, failures lists each warning and each piece of skipped work again as an error, in the issue format the checks use; it is empty without --strict. failed decides the exit code.', + }, ], }; diff --git a/packages/cli/test/cli-strict-guard.test.mjs b/packages/cli/test/cli-strict-guard.test.mjs new file mode 100644 index 0000000000000..4b2529e87cf5e --- /dev/null +++ b/packages/cli/test/cli-strict-guard.test.mjs @@ -0,0 +1,130 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +/** + * @file Only `astryx integration check` may fail on a warning + * (architecture:cli-surface INV32, DEC-4). + * + * A command's exit code does not depend on its warnings: a warning is for the + * reader, and a build that runs the command must not break when a release + * adds one. `integration check --strict` is the one command a package runs to + * fail its build on warnings, and the CLI surface spec admits it by name. + * Two lists, read from the live CLI, keep it the only one: + * - every command that offers `--strict`, from the options each command + * parses (so an option missing from its CommandDoc still counts); + * - every command whose documented exit codes make a warning fail it: a + * non-zero exit code whose condition names a warning, unless it says the + * warning does not change the exit code. + * Both must be exactly `integration check`. A new entry fails here until the + * spec admits it. + */ + +import {describe, expect, it} from 'vitest'; +import {JSON_SUPPORTED, program} from '../clients/cli/index.mjs'; +import {buildManifest} from '../clients/cli/lib/manifest.mjs'; + +/** The commands the CLI surface spec allows to fail on a warning. */ +const ALLOWED = ['integration check']; + +const manifest = buildManifest(program, { + jsonSupported: JSON_SUPPORTED, + version: '0.0.0-test', +}); + +/** @type {any[]} */ +const commands = []; +const walk = (/** @type {any} */ command) => { + commands.push(command); + (command.subcommands ?? []).forEach(walk); +}; +manifest.commands.forEach(walk); + +/** + * Whether an exit-code condition makes a warning fail the command. + * @param {string} when + */ +function failsOnWarning(when) { + return ( + /\bwarn/i.test(when) && + !/\b(?:does not|doesn't|never)\s+(?:change|affect)\b[^.]*\bexit code\b/i.test( + when, + ) && + !/\bwarnings?\s+(?:are|is)\s+allowed\b/i.test(when) + ); +} + +describe('only integration check fails on warnings', () => { + it('reads every command', () => { + expect(commands.length).toBeGreaterThan(30); + }); + + it('is the only command that offers --strict', () => { + const strict = commands + .filter(command => + (command.options ?? []).some( + (/** @type {any} */ option) => + /(?:^|[\s,])--strict\b/.test( + String(option.flags ?? option.flag ?? ''), + ) || option.long === '--strict', + ), + ) + .map(command => command.name) + .sort(); + expect( + strict, + 'A new --strict needs the CLI surface spec to admit it first.', + ).toEqual(ALLOWED); + }); + + it('is the only command whose exit code depends on a warning', () => { + const onWarning = commands + .filter(command => + (command.exitCodes ?? []).some( + (/** @type {{code: number, when: string}} */ exit) => + exit.code !== 0 && failsOnWarning(exit.when), + ), + ) + .map(command => command.name) + .sort(); + expect( + onWarning, + 'A command that fails on a warning needs the CLI surface spec to admit it first.', + ).toEqual(ALLOWED); + }); + + it('does not count a mode that exists to detect a condition, which reports it as an error', () => { + // `theme build --check` fails on stale outputs, and the deprecated + // `integration pack --check` on an incomplete tarball: errors their own + // mode exists to detect, not warnings. + const names = commands.map(command => command.name); + expect(names).toEqual( + expect.arrayContaining(['theme build', 'integration pack']), + ); + for (const name of ['theme build', 'integration pack']) { + const command = commands.find(each => each.name === name); + expect( + (command.exitCodes ?? []).some( + (/** @type {{code: number, when: string}} */ exit) => + exit.code !== 0 && failsOnWarning(exit.when), + ), + name, + ).toBe(false); + } + }); + + it('tells a condition that fails on a warning from one that does not', () => { + expect( + failsOnWarning( + 'with --strict, also any warning, or a check that did not run', + ), + ).toBe(true); + expect( + failsOnWarning( + 'a doc is invalid. A link that names no doc is a warning, and does not change the exit code', + ), + ).toBe(false); + expect( + failsOnWarning('no error-severity issues (warnings are allowed)'), + ).toBe(false); + expect(failsOnWarning('the tarball is incomplete or invalid')).toBe(false); + }); +}); diff --git a/packages/cli/test/integration-check-failures.json b/packages/cli/test/integration-check-failures.json new file mode 100644 index 0000000000000..a5d9fd521dd52 --- /dev/null +++ b/packages/cli/test/integration-check-failures.json @@ -0,0 +1,66 @@ +{ + "checks": ["validate", "components", "templates", "docs"], + "results": { + "validate": ["docTypes", "issues"], + "components": ["conflicts", "issues", "propDrift"], + "templates": ["conflicts", "issues"], + "docs": ["findings", "issues"] + }, + "reports": { + "docTypes": "doc_types_skipped", + "propDrift": "prop_drift_skipped" + }, + "codes": [ + "ambiguous_component_replacement", + "ambiguous_template_replacement", + "codemod_outside_version", + "component_name_conflict", + "core_not_found", + "doc_types_skipped", + "duplicate_doc", + "duplicate_provider", + "inactive_component_replacement", + "integration_error", + "invalid_agent_docs", + "invalid_codemod", + "invalid_codemod_version", + "invalid_component", + "invalid_component_replacement", + "invalid_doc", + "invalid_doc_graph", + "invalid_doc_reference", + "invalid_manifest", + "invalid_package_json", + "invalid_package_spec", + "invalid_root", + "invalid_template", + "invalid_template_replacement", + "invalid_theme", + "missing_component_replacement_target", + "missing_field", + "missing_manifest", + "missing_root", + "missing_template_replacement_target", + "multiple_manifests", + "no_package", + "not_in_code", + "not_in_doc", + "package_not_found", + "prop_drift_skipped", + "prop_types_skipped", + "required_mismatch", + "root_outside_package", + "shadowed_component_name", + "source_without_component_doc", + "template_id_conflict", + "type_mismatch", + "unknown_field", + "unknown_manifest_key", + "unreachable_contribution", + "unreachable_scan_truncated", + "unread_theme_folder", + "unreadable_folder", + "unreadable_root", + "wrong_type" + ] +} diff --git a/packages/cli/test/integration-check-failures.test.mjs b/packages/cli/test/integration-check-failures.test.mjs new file mode 100644 index 0000000000000..127a0657483d9 --- /dev/null +++ b/packages/cli/test/integration-check-failures.test.mjs @@ -0,0 +1,304 @@ +// Copyright (c) Meta Platforms, Inc. and affiliates. + +/** + * @file Everything that can fail `astryx integration check --strict`. + * + * Builds run `integration check --strict`, so what fails it is a contract: a + * new check, or a new failure code, joins only in a minor release that names + * it. The list + * beside this file, integration-check-failures.json, names every check, every + * field of a check's result that is counted, every code a failure can carry, + * and every comparison a check carries, read here from the code: + * - `checks`: the checks `integration check` runs, in order; + * - `results`: per check, the fields of its result besides its identity, read + * from a run against a package with every kind of root; + * - `codes`: every issue code the checks' modules can raise (scanned in each + * module the command imports), the codes a conflict, a package with no + * manifest, or a skipped comparison carries, and every finding code a + * comparison declares; + * - `reports`: each comparison a check carries, with the code its skip has. + * Adding to any of them fails this test until the list changes, so a new way + * to fail a build always shows up in review. + */ + +import * as fs from 'node:fs'; +import * as path from 'node:path'; +import {fileURLToPath} from 'node:url'; +import {afterAll, beforeAll, describe, expect, it} from 'vitest'; +import { + CONFLICT_CODES, + INTEGRATION_CHECKS, + NO_MANIFEST_CODE, + integrationCheck, + isCheckReport, + skippedCode, +} from '../api/integration/check/check.mjs'; + +const HERE = path.dirname(fileURLToPath(import.meta.url)); +const CLI = path.resolve(HERE, '..'); +const LIST = path.join(HERE, 'integration-check-failures.json'); +const ENTRY = path.join(CLI, 'api', 'integration', 'check', 'check.mjs'); + +/** + * Modules the command imports whose codes cannot fail it, with why. + * @type {Record} + */ +const NOT_SCANNED = { + // Only `integration verify` compares a packed tarball with the source. + 'foundation/integrations/contribution-inventory.mjs': + 'its codes come from comparing a packed tarball, which only integration verify does', +}; + +/** The authoring schemas: their codes are the schema library's, not issues. */ +const AUTHORING = `authoring${path.sep}`; + +/** @type {{checks: string[], results: Record, codes: string[], reports: Record}} */ +const list = JSON.parse(fs.readFileSync(LIST, 'utf8')); + +/** + * Every CLI module `entry` reaches through static relative imports, docs and + * tests aside. + * @param {string} entry + */ +function reachableModules(entry) { + const IMPORT = + /(?:import|export)\s[^'"]*?from\s*['"](\.[^'"]+)['"]|import\(\s*['"](\.[^'"]+)['"]\s*\)/g; + const seen = new Set(); + const queue = [entry]; + while (queue.length > 0) { + const file = /** @type {string} */ (queue.shift()); + if (seen.has(file)) continue; + seen.add(file); + const text = fs.readFileSync(file, 'utf8'); + for (const match of text.matchAll(IMPORT)) { + let target = path.resolve(path.dirname(file), match[1] ?? match[2]); + if (!fs.existsSync(target) && fs.existsSync(`${target}.mjs`)) { + target = `${target}.mjs`; + } + if ( + !target.startsWith(CLI + path.sep) || + !/\.(?:mjs|js)$/.test(target) || + /\.(?:doc|test)\.mjs$/.test(target) || + !fs.existsSync(target) + ) { + continue; + } + queue.push(target); + } + } + return [...seen]; +} + +/** + * The issue codes and finding codes one module can raise: a `code:` it + * writes (or its fallback), each code in the first argument of `issueError` + * and `issueWarning` under any name it imports them as (both branches of a + * conditional), the code an error list is added under, the members of a + * `code` property typed as a union of codes, and the members of a + * finding-kind union whose members are all snake-case codes. + * @param {string} text + */ +function codesIn(text) { + const codes = new Set(); + const constants = new Map( + [ + ...text.matchAll( + /\bconst\s+([A-Z][A-Z0-9_]*)\s*=\s*'([a-z][a-z0-9_]*)'/g, + ), + ].map(m => [m[1], m[2]]), + ); + const raisers = ['issueError', 'issueWarning']; + for (const m of text.matchAll(/\bissue(?:Error|Warning)\s+as\s+(\w+)/g)) { + raisers.push(m[1]); + } + const raise = new RegExp(`\\b(?:${raisers.join('|')})\\(([^,]*),`, 'g'); + for (const m of text.matchAll(raise)) { + for (const each of m[1].matchAll( + /'([a-z][a-z0-9_]*)'|\b([A-Z][A-Z0-9_]*)\b/g, + )) { + const code = each[1] ?? constants.get(each[2]); + if (code) codes.add(code); + } + } + for (const m of text.matchAll(/@property\s+\{([^}]*)\}\s+code\b/g)) { + const members = [...m[1].matchAll(/'([^']*)'/g)].map(each => each[1]); + if ( + members.length > 0 && + members.every(member => /^[a-z]+(?:_[a-z]+)+$/.test(member)) + ) { + for (const member of members) codes.add(member); + } + } + for (const m of text.matchAll(/\bcode:\s*'([a-z][a-z0-9_]*)'/g)) + codes.add(m[1]); + for (const m of text.matchAll( + /\bcode:\s*[^,\n}]*?\?\?\s*'([a-z][a-z0-9_]*)'/g, + )) { + codes.add(m[1]); + } + for (const m of text.matchAll( + /\baddErrors\([^;]*?'([a-z][a-z0-9_]*)'\s*\)/g, + )) { + codes.add(m[1]); + } + for (const m of text.matchAll( + /@typedef\s+\{([^}]*)\}\s+\w*(?:Kind|Problem)\b/g, + )) { + const members = [...m[1].matchAll(/'([^']*)'/g)].map(each => each[1]); + if ( + members.length > 0 && + members.every(member => /^[a-z]+(?:_[a-z]+)+$/.test(member)) + ) { + for (const member of members) codes.add(member); + } + } + return codes; +} + +/** @type {string} */ +let pkg; +/** @type {import('../api/integration/integration-check.type.mjs').IntegrationCheckCommandResponse} */ +let result; + +beforeAll(async () => { + // Under the repository, so the components check resolves Core. Every kind + // of root, so every field a check's result can carry is present. + pkg = fs.mkdtempSync(path.join(process.cwd(), '.astryx-check-failures-')); + const write = (/** @type {string} */ rel, /** @type {string} */ text) => { + fs.mkdirSync(path.dirname(path.join(pkg, rel)), {recursive: true}); + fs.writeFileSync(path.join(pkg, rel), text); + }; + write( + 'package.json', + JSON.stringify({name: '@acme/widgets', version: '1.2.3'}), + ); + write( + 'astryx.integration.mjs', + "export default {components: './components', templates: './templates', docs: './docs'};\n", + ); + write( + 'components/AcmeButton.doc.mjs', + "export default {type: 'component', name: 'AcmeButton', displayName: 'Acme Button', description: 'A button.', props: []};\n", + ); + write( + 'components/AcmeButton.tsx', + 'export function AcmeButton() { return null; }\n', + ); + write( + 'templates/acme-page.doc.mjs', + "export default {type: 'page', name: 'Acme page', displayName: 'Acme page', description: 'A page.'};\n", + ); + write( + 'templates/acme-page.tsx', + 'export default function Fixture() { return null; }\n', + ); + write( + 'docs/acme-guide.doc.mjs', + "export default {type: 'generic', name: 'acme-guide', title: 'Acme guide', description: 'A guide.', sections: [{title: 'One', content: [{type: 'prose', text: 'One.'}]}]};\n", + ); + result = await integrationCheck({cwd: pkg, strict: true}); +}, 60_000); + +afterAll(() => { + fs.rmSync(pkg, {recursive: true, force: true}); +}); + +describe('what can fail integration check --strict', () => { + it('runs the listed checks, in order', () => { + expect([...INTEGRATION_CHECKS]).toEqual(list.checks); + expect(result.data.checks.map(check => check.check)).toEqual(list.checks); + }); + + it('counts the listed fields of each check’s result', () => { + /** @type {Record} */ + const fields = {}; + for (const {check, response} of result.data.checks) { + fields[check] = Object.keys(response.data) + .filter(key => !['validated', 'name', 'version'].includes(key)) + .sort(); + } + expect( + fields, + 'A check’s result has a new field. If it can fail integration check, it is a new failure: list it (a minor-release change).', + ).toEqual( + Object.fromEntries( + Object.entries(list.results).map(([check, keys]) => [ + check, + [...keys].sort(), + ]), + ), + ); + }); + + it('lists each comparison a check carries, with the code its skip has', () => { + /** @type {Record} */ + const reports = {}; + for (const {response} of result.data.checks) { + for (const [key, value] of Object.entries(response.data)) { + if (isCheckReport(value)) reports[key] = skippedCode(key); + } + } + expect(reports).toEqual(list.reports); + }); + + it('lists every code a failure can carry', () => { + const codes = new Set([ + ...Object.values(CONFLICT_CODES), + NO_MANIFEST_CODE, + ...Object.values(list.reports), + ]); + for (const file of reachableModules(ENTRY)) { + const rel = path.relative(CLI, file).split(path.sep).join('/'); + if (rel in NOT_SCANNED || file.startsWith(path.join(CLI, AUTHORING))) { + continue; + } + for (const code of codesIn(fs.readFileSync(file, 'utf8'))) { + codes.add(code); + } + } + const found = [...codes].sort(); + const added = found.filter(code => !list.codes.includes(code)); + const removed = list.codes.filter(code => !codes.has(code)); + expect( + added, + 'These codes can now fail integration check --strict. List them: a new failure code joins only in a minor release that names it.', + ).toEqual([]); + expect( + removed, + 'Nothing raises these codes any more. Remove them from integration-check-failures.json.', + ).toEqual([]); + }); + + it('reads the codes a module raises, under every name', () => { + expect( + [ + ...codesIn( + [ + "import {issueError as error} from './x.mjs';", + "const DUPLICATE = 'duplicate_thing';", + "error('first_code', 'm'); issueWarning(DUPLICATE, 'm');", + "error(found ? 'ternary_one' : 'ternary_two', 'm');", + "/** @property {'typed_code_one' | 'typed_code_two'} code */", + "issues.push({code: 'second_code', severity: 'warning'});", + "x = {code: e.code ?? 'fallback_code'};", + "addErrors(issues, errors, 'added_code');", + "/** @typedef {'not_in_code' | 'type_mismatch'} DriftKind */", + "/** @typedef {'page' | 'block'} TemplateKind */", + ].join('\n'), + ), + ].sort(), + ).toEqual([ + 'added_code', + 'duplicate_thing', + 'fallback_code', + 'first_code', + 'not_in_code', + 'second_code', + 'ternary_one', + 'ternary_two', + 'type_mismatch', + 'typed_code_one', + 'typed_code_two', + ]); + }); +});