diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 779b919..54b6889 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,6 +1,7 @@ name: CI on: + workflow_call: push: branches: [main] pull_request: @@ -24,6 +25,11 @@ jobs: cache-dependency-path: packages/typescript/package-lock.json - run: npm ci --ignore-scripts - run: npm run check + - name: Install and test packed npm artifact + run: | + mkdir -p ../../npm-dist + npm pack --pack-destination ../../npm-dist + node ../../tools/smoke_npm.mjs ../../npm-dist/*.tgz test: runs-on: ubuntu-latest diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 7ec1fd3..b304227 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -8,7 +8,11 @@ permissions: contents: read jobs: + validate: + uses: ./.github/workflows/ci.yml + build: + needs: validate if: github.event.release.target_commitish == 'main' runs-on: ubuntu-latest outputs: @@ -53,6 +57,10 @@ jobs: npm run check mkdir -p ../../npm-dist npm pack --pack-destination ../../npm-dist + - name: Install and test final npm tarball + run: | + test "$(find npm-dist -maxdepth 1 -type f -name '*.tgz' | wc -l)" -eq 1 + node tools/smoke_npm.mjs npm-dist/*.tgz - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.0 with: name: python-distributions diff --git a/RELEASING.md b/RELEASING.md index f22b60e..f1c32b4 100644 --- a/RELEASING.md +++ b/RELEASING.md @@ -43,6 +43,14 @@ subsequent release. Never store an npm publish token in GitHub. announcing the release. The workflow fails closed if the tag differs from the contract version. It +requires the reusable CI workflow, including its Python matrix and TypeScript +checks, to succeed before building. Failed, cancelled or skipped validation +blocks both publishers. The final npm tarball is installed into a fresh consumer +directory and exercised through its public imports, bundled valid/invalid +conformance controls and golden event construction before artifact upload. +The smoke check also verifies installed identity, import origin and declarations. + +The release workflow builds each distribution once, sends the same artifacts to the registries, and attaches them to the GitHub release. diff --git a/tests/test_release_safeguards.py b/tests/test_release_safeguards.py new file mode 100644 index 0000000..64ef5c2 --- /dev/null +++ b/tests/test_release_safeguards.py @@ -0,0 +1,78 @@ +"""Exercise the release dependency graph and packed-artifact handoff.""" +import ast +from pathlib import Path +import re +import unittest + +ROOT = Path(__file__).resolve().parents[1] + + +def jobs(source): + return dict(re.findall(r"^ ([\w-]+):\n(.*?)(?=^ [\w-]+:\n|\Z)", source, re.M | re.S)) + + +def field(block, name): + match = re.search(rf"^ {name}: (.+)$", block, re.M) + return match[1] if match else None + + +def scheduled(source, validation, target): + results = {"validate": validation} + for name, block in jobs(source).items(): + if name == "validate": + continue + needs = field(block, "needs") or "" + dependencies = [item.strip() for item in needs.strip("[]").split(",") if item.strip()] + assert all(dependency in results for dependency in dependencies) + eligible = all(results[dependency] == "success" for dependency in dependencies) + if condition := field(block, "if"): + tree = ast.parse(condition, mode="eval").body + # Fail on a new status-function guard; it must not silently inherit + # this model's default success() behavior. + assert isinstance(tree, ast.Compare) + assert ast.unparse(tree.left) == "github.event.release.target_commitish" + assert len(tree.ops) == 1 and isinstance(tree.ops[0], ast.Eq) + assert len(tree.comparators) == 1 + eligible = eligible and target == ast.literal_eval(tree.comparators[0]) + results[name] = "success" if eligible else "skipped" + return results + + +class ReleaseSafeguardsTests(unittest.TestCase): + def test_validation_failure_and_target_controls_reach_both_publishers(self): + source = (ROOT / ".github/workflows/release.yml").read_text() + for validation in ("success", "failure", "cancelled", "skipped"): + for target in ("main", "other"): + with self.subTest(validation=validation, target=target): + result = scheduled(source, validation, target) + expected = "success" if validation == "success" and target == "main" else "skipped" + for job in ("build", "publish-pypi", "publish-npm", "release-assets"): + self.assertEqual(result[job], expected) + + def test_validation_reuses_ci_with_the_python_matrix_and_typescript(self): + release = jobs((ROOT / ".github/workflows/release.yml").read_text()) + self.assertEqual(field(release["validate"], "uses"), "./.github/workflows/ci.yml") + ci = (ROOT / ".github/workflows/ci.yml").read_text() + self.assertRegex(ci, r"(?m)^ workflow_call:") + self.assertIn('python-version: ["3.10", "3.11", "3.12", "3.13"]', ci) + self.assertIn("npm run check", jobs(ci)["typescript"]) + + def test_packed_tarball_is_checked_before_the_immutable_upload(self): + release = jobs((ROOT / ".github/workflows/release.yml").read_text()) + build = release["build"] + pack = build.index("npm pack --pack-destination ../../npm-dist") + smoke = build.index("node tools/smoke_npm.mjs npm-dist/*.tgz") + upload = build.index("name: npm-distribution") + self.assertLess(pack, smoke) + self.assertLess(smoke, upload) + self.assertIn("path: npm-dist/*.tgz", build) + publisher = release["publish-npm"] + self.assertIn("name: npm-distribution", publisher) + self.assertIn("npm publish ./npm-dist/*.tgz", publisher) + self.assertNotIn("npm pack", publisher) + self.assertNotRegex(build, r"(?m)^ environment:") + self.assertNotIn("id-token: write", build) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/smoke_npm.mjs b/tools/smoke_npm.mjs new file mode 100644 index 0000000..0ff0c49 --- /dev/null +++ b/tools/smoke_npm.mjs @@ -0,0 +1,66 @@ +// Exercise the packed consumer surface outside the checkout, without dev tools. +import assert from "node:assert/strict"; +import {execFileSync} from "node:child_process"; +import {mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync} from "node:fs"; +import {tmpdir} from "node:os"; +import {join, resolve} from "node:path"; +import {fileURLToPath} from "node:url"; + +assert.equal(process.argv.length, 3, "usage: node tools/smoke_npm.mjs distribution.tgz"); +const distribution = resolve(process.argv[2]); +const root = fileURLToPath(new URL("../", import.meta.url)); +const metadata = JSON.parse(readFileSync(join(root, "packages/typescript/package.json"), "utf8")); +const fixtures = {}; +for (const kind of ["valid", "invalid"]) { + const directory = join(root, "conformance/fixtures", kind); + fixtures[kind] = readdirSync(directory).filter(name => name.endsWith(".json")).sort() + .map(name => ({name, event: JSON.parse(readFileSync(join(directory, name), "utf8"))})); + assert.ok(fixtures[kind].length > 0, `missing ${kind} controls`); +} +const golden = JSON.parse(readFileSync(join(root, "compatibility/golden/event-factory.json"), "utf8")); +const directory = mkdtempSync(join(tmpdir(), "agentrust-npm-")); +try { + writeFileSync(join(directory, "package.json"), JSON.stringify({private: true, type: "module"})); + writeFileSync(join(directory, "controls.json"), JSON.stringify({metadata, fixtures, golden})); + execFileSync("npm", ["install", "--ignore-scripts", "--no-audit", "--no-fund", distribution], + {cwd: directory, stdio: "inherit"}); + writeFileSync(join(directory, "smoke.test.mjs"), ` +import assert from "node:assert/strict"; +import {existsSync, readFileSync, realpathSync} from "node:fs"; +import {join, sep} from "node:path"; +import test from "node:test"; +import {fileURLToPath} from "node:url"; +import {SchemaValidator, EventValidationError, EventFactory} from "@agentrust-io/telemetry"; +const {metadata, fixtures, golden} = JSON.parse(readFileSync("controls.json", "utf8")); +const installed = realpathSync(join("node_modules", metadata.name)); +test("installed identity, module origin and declarations", () => { + const actual = JSON.parse(readFileSync(join(installed, "package.json"), "utf8")); + assert.equal(actual.name, metadata.name); + assert.equal(actual.version, metadata.version); + const module = realpathSync(fileURLToPath(import.meta.resolve(metadata.name))); + assert.ok(module.startsWith(installed + sep)); + assert.ok(existsSync(join(installed, actual.types)), "missing type declarations"); +}); +const validator = SchemaValidator.bundled(); +test("bundled schemas accept all conformant controls", () => { + for (const {name, event} of fixtures.valid) { + assert.doesNotThrow(() => validator.validate(event), name); + } +}); +test("bundled schemas refuse all invalid controls", () => { + for (const {name, event} of fixtures.invalid) { + assert.throws(() => validator.validate(event), EventValidationError, name); + } +}); +test("installed event construction matches the golden control", () => { + const factory = new EventFactory(validator, {name: "parity-test", version: "1.0.0"}, + () => 1787079000000000000n, () => "018f0f7d-7a13-7cc2-8000-000000000042"); + assert.deepEqual(factory.build("usage.recorded", + {runId: "run-1", agentId: "agent-1", workflowId: "workflow-1"}, + {scope: "model_call", operation: "chat", input_tokens: 7}), golden); +}); +`); + execFileSync(process.execPath, ["--test", "smoke.test.mjs"], {cwd: directory, stdio: "inherit"}); +} finally { + rmSync(directory, {recursive: true, force: true}); +}