From f631eb9f3aa8780ff63cc7a01589b78b69fa01fe Mon Sep 17 00:00:00 2001 From: itvi-1234 Date: Fri, 23 Oct 2026 14:46:50 +0530 Subject: [PATCH 1/2] Generate TypeScript bindings in CI Add TypeScript targets for common-integrations and env-configuration to the package catalog, named @runtimeconditions/, and a generate-typescript.py script that normalizes each model with rc, runs the TypeScript emitter, verifies the package, and adds the model, the root extension, and a file manifest like the Go and Python packages have. The generation workflow now runs it and commits bindings//typescript to main. rc selects only the languages it emits itself when no target is given, so a catalog language with its own script no longer breaks rc bindings check and update. Signed-off-by: itvi-1234 --- .github/actions/binding-tools/action.yml | 21 +- .github/workflows/binding-generate.yml | 14 +- .../ci/generate-typescript.py | 275 ++++++++++++++++++ tooling/extension-bindings/ci/generation.py | 20 +- .../ci/test_generate_typescript.py | 202 +++++++++++++ .../extension-bindings/orchestrator/plan.go | 8 +- .../orchestrator/plan_test.go | 22 ++ tooling/extension-bindings/packages.yaml | 16 + 8 files changed, 567 insertions(+), 11 deletions(-) create mode 100644 tooling/extension-bindings/ci/generate-typescript.py create mode 100644 tooling/extension-bindings/ci/test_generate_typescript.py diff --git a/.github/actions/binding-tools/action.yml b/.github/actions/binding-tools/action.yml index 78d8c11..5f5f243 100644 --- a/.github/actions/binding-tools/action.yml +++ b/.github/actions/binding-tools/action.yml @@ -2,7 +2,7 @@ name: Prepare binding tools description: Select targets, install the exact compilers and prerequisites, and build the shared generator inputs: targets: - description: Comma-separated package-key:go or package-key:python targets, or all + description: Comma-separated package-key:go, package-key:python, or package-key:typescript targets, or all required: true publication: description: Include the committed package dependency closure for publication @@ -17,6 +17,12 @@ outputs: has-python: description: Whether a Python target is selected value: ${{ steps.configure.outputs.has-python }} + typescript-version: + description: Exact catalog Node.js version, empty when no TypeScript target is selected + value: ${{ steps.configure.outputs.typescript-version }} + has-typescript: + description: Whether a TypeScript target is selected + value: ${{ steps.configure.outputs.has-typescript }} source-commit: description: Checked-out source commit value: ${{ steps.configure.outputs.source-commit }} @@ -53,6 +59,7 @@ runs: python tooling/extension-bindings/ci/generation.py configure --targets "$INPUT_TARGETS" python tooling/extension-bindings/ci/generate-go.py configure --targets "$INPUT_TARGETS" python tooling/extension-bindings/ci/generate-python.py configure --targets "$INPUT_TARGETS" + python tooling/extension-bindings/ci/generate-typescript.py configure --targets "$INPUT_TARGETS" fi - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 with: @@ -110,6 +117,18 @@ runs: printf 'RC_PYTHON_PROFILER_BIN=%s\n' "$tools/python-venv/bin/runtimeconditions-python-profiler" printf 'RC_PYTHON_PROFILER_ARTIFACT=%s\n' "$wheel" } >> "$GITHUB_ENV" + - name: Set up the pinned Node.js + if: steps.configure.outputs.has-typescript == 'true' + uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6.5.0 + with: + node-version: ${{ steps.configure.outputs.typescript-version }} + cache: npm + cache-dependency-path: tooling/extension-bindings/emitters/typescript/package-lock.json + - name: Build the TypeScript emitter from its pinned dependencies + if: steps.configure.outputs.has-typescript == 'true' + shell: bash + working-directory: tooling/extension-bindings/emitters/typescript + run: npm ci --ignore-scripts && npm run build - name: Build shared generator shell: bash run: go -C tooling/extension-bindings/orchestrator build -trimpath -buildvcs=false -o "$RUNNER_TEMP/binding-tools/rc" ./cmd/rc diff --git a/.github/workflows/binding-generate.yml b/.github/workflows/binding-generate.yml index 47dd044..4de9d4b 100644 --- a/.github/workflows/binding-generate.yml +++ b/.github/workflows/binding-generate.yml @@ -1,4 +1,4 @@ -name: Generate and commit Go and Python bindings +name: Generate and commit Go, Python and TypeScript bindings on: push: @@ -10,18 +10,20 @@ on: - 'catalog/**' - 'bindings/**/go/**' - 'bindings/**/python/**' + - 'bindings/**/typescript/**' - 'tooling/extension-bindings/packages.yaml' - 'tooling/extension-bindings/toolchain.lock.yaml' - 'tooling/extension-bindings/model/**' - 'tooling/extension-bindings/normalizer/**' - 'tooling/extension-bindings/emitters/go/**' - 'tooling/extension-bindings/emitters/python/**' + - 'tooling/extension-bindings/emitters/typescript/**' - 'tooling/extension-bindings/orchestrator/**' - 'tooling/extension-bindings/ci/**' workflow_dispatch: inputs: targets: - description: 'Comma-separated package-key:go or package-key:python targets, or all; empty uses automatic targets' + description: 'Comma-separated package-key:go, package-key:python, or package-key:typescript targets, or all; empty uses automatic targets' type: string default: '' @@ -40,7 +42,7 @@ jobs: permissions: contents: write env: - INPUT_TARGETS: ${{ inputs.targets || vars.BINDING_GENERATION_TARGETS || 'common-integrations:go,env-configuration:go,common-integrations:python,env-configuration:python' }} + INPUT_TARGETS: ${{ inputs.targets || vars.BINDING_GENERATION_TARGETS || 'common-integrations:go,env-configuration:go,common-integrations:python,env-configuration:python,common-integrations:typescript,env-configuration:typescript' }} GOTOOLCHAIN: local steps: - name: Check out current main @@ -59,6 +61,7 @@ jobs: shell: bash env: HAS_PYTHON: ${{ steps.tools.outputs.has-python }} + HAS_TYPESCRIPT: ${{ steps.tools.outputs.has-typescript }} run: | set -euo pipefail python -m unittest discover -s tooling/extension-bindings/ci -p 'test_*.py' @@ -71,6 +74,9 @@ jobs: tooling/extension-bindings/emitters/python/tests \ --ignore=tooling/extension-bindings/emitters/python/tests/test_packages.py fi + if [[ "$HAS_TYPESCRIPT" == true ]]; then + npm --prefix tooling/extension-bindings/emitters/typescript test + fi - name: Generate and verify bindings env: @@ -81,6 +87,8 @@ jobs: --rc "$RC" --core-schema "$CORE_SCHEMA" python tooling/extension-bindings/ci/generate-python.py update --targets "$INPUT_TARGETS" \ --rc "$RC" --core-schema "$CORE_SCHEMA" + python tooling/extension-bindings/ci/generate-typescript.py update --targets "$INPUT_TARGETS" \ + --rc "$RC" --core-schema "$CORE_SCHEMA" - name: Commit verified generated source to main env: diff --git a/tooling/extension-bindings/ci/generate-typescript.py b/tooling/extension-bindings/ci/generate-typescript.py new file mode 100644 index 0000000..4a687bc --- /dev/null +++ b/tooling/extension-bindings/ci/generate-typescript.py @@ -0,0 +1,275 @@ +"""Configure, update, and check TypeScript bindings for the Actions workflow.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import re +import shutil +import subprocess +import tempfile +from pathlib import Path + +import yaml + +from generation import ( + Target, + exact_version, + of_language, + select_targets, + summarize, + write_outputs, +) + +LANGUAGE = "typescript" +TOOLING = Path("tooling/extension-bindings") +EMITTER = TOOLING / "emitters/typescript" +TARGET_API = "runtimeconditions.io/typescript-package-target/v1alpha1" +TARGET_KIND = "RuntimeConditionsTypeScriptPackageTarget" +FILE_MANIFEST = "runtimeconditions.file-manifest.yaml" +MODEL_FILE = "runtimeconditions.binding-model.yaml" +EXTENSION_FILE = "runtimeconditions.extension.yaml" +RESOURCES = "runtimeconditions" + + +def sha256(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def emitter_digest(emitter: Path) -> str: + """Identify the emitter by its source, so the digest does not depend on a build.""" + names = ["package.json", "package-lock.json", "package-target.schema.yaml"] + for folder, pattern in (("src", "*.ts"), ("templates", "*")): + names += sorted( + path.relative_to(emitter).as_posix() + for path in (emitter / folder).glob(pattern) + if path.is_file() + ) + files = {name: sha256((emitter / name).read_bytes()) for name in names} + return sha256(json.dumps(files, sort_keys=True, separators=(",", ":")).encode()) + + +def file_manifest(directory: Path) -> str: + """List every file except the manifest itself, in the format the Go tooling writes.""" + files = { + path.relative_to(directory).as_posix(): sha256(path.read_bytes()) + for path in directory.rglob("*") + if path.is_file() and path.name != FILE_MANIFEST + } + lines = [ + "apiVersion: runtimeconditions.io/file-manifest/v1alpha1", + "kind: RuntimeConditionsFileManifest", + "files:", + ] + for name in sorted(files, key=lambda item: item.encode("utf-8")): + if not re.fullmatch(r"[A-Za-z0-9_./-]+", name): + raise ValueError(f"unexpected generated file name: {name}") + lines.append(f" {name}: {files[name]}") + return "\n".join(lines) + "\n" + + +def package_target( + catalog: dict, key: str, model: dict, emitter_sha256: str +) -> dict: + """Build the emitter's package target from the catalog and the normalized model.""" + package = catalog["packages"][key] + config = package["languages"][LANGUAGE] + root = package["rootExtension"] + dependencies = [] + for edge in sorted( + item["to"] + for item in model.get("dependencyEdges") or [] + if item["from"] == root + ): + providers = [ + other["languages"][LANGUAGE] + for other in catalog["packages"].values() + if other["rootExtension"] == edge and LANGUAGE in other["languages"] + ] + if len(providers) != 1: + raise ValueError( + f"{key}:{LANGUAGE}: direct dependency {edge} requires exactly one " + f"{LANGUAGE} catalog provider (found {len(providers)})" + ) + dependencies.append( + { + "extension": edge, + "packageName": providers[0]["name"], + "version": str(providers[0]["version"]), + } + ) + target = { + "apiVersion": TARGET_API, + "kind": TARGET_KIND, + "packageKey": key, + "rootExtension": root, + "packageName": config["name"], + "version": str(config["version"]), + "sourceDirectory": config["sourceDirectory"], + "minimumNodeVersion": ".".join(str(config["languageVersion"]).split(".")[:2]), + "publicationMode": config["publicationMode"], + "repositoryUrl": catalog["repositoryUrl"], + "dependencies": dependencies, + "emitterSha256": emitter_sha256, + } + if "registryId" in config: + target["registryId"] = config["registryId"] + return target + + +def root_extension_source(root: Path, lock: dict, extension: str) -> bytes: + """Read the root extension from the catalog, checking it against the dependency lock.""" + entries = [item for item in lock["extensions"] if item["id"] == extension] + if len(entries) != 1: + raise ValueError(f"the dependency lock must list {extension} exactly once") + backend, _, locator = entries[0]["sourceLocator"].partition(":") + if backend != "catalog": + raise ValueError(f"{extension}: only catalog extension sources are supported") + data = (root / "catalog" / locator).read_bytes() + if sha256(data) != entries[0]["sourceSha256"]: + raise ValueError(f"{extension}: catalog source differs from the dependency lock") + return data + + +def tools(root: Path) -> tuple[str, Path]: + node = os.environ.get("RC_BINDINGS_NODE", "node") + main = Path( + os.environ.get("RC_TYPESCRIPT_EMITTER", root / EMITTER / "dist/src/main.js") + ) + if not main.is_file(): + raise ValueError( + f"build the TypeScript emitter first (npm ci && npm run build in {root / EMITTER}): " + f"{main} is missing" + ) + return node, main + + +def generate( + root: Path, catalog: dict, key: str, rc: Path, core: Path, work: Path +) -> Path: + """Generate one verified package tree beneath work and return its directory.""" + node, main = tools(root) + tooling = root / TOOLING + model_path, lock_path = work / f"{key}.model.yaml", work / f"{key}.lock.yaml" + subprocess.run( + [ + str(rc.resolve()), + "bindings", + "normalize", + "--packages", + str(tooling / "packages.yaml"), + "--toolchain-lock", + str(tooling / "toolchain.lock.yaml"), + "--core-schema", + str(core.resolve()), + "--target", + f"{key}:{LANGUAGE}", + "--output", + str(model_path), + "--dependency-lock-output", + str(lock_path), + ], + cwd=root, + check=True, + stdout=subprocess.DEVNULL, + ) + model = yaml.safe_load(model_path.read_text(encoding="utf-8")) + lock = yaml.safe_load(lock_path.read_text(encoding="utf-8")) + target = package_target(catalog, key, model, emitter_digest(root / EMITTER)) + target_path = work / f"{key}.target.yaml" + target_path.write_text(yaml.safe_dump(target, sort_keys=False), encoding="utf-8") + package = work / key + common = ["--model", str(model_path), "--package-config", str(target_path)] + subprocess.run( + [node, str(main), "emit", *common, "--output", str(package)], + cwd=root, + check=True, + stdout=subprocess.DEVNULL, + ) + subprocess.run( + [node, str(main), "verify", *common, "--input", str(package)], + cwd=root, + check=True, + stdout=subprocess.DEVNULL, + ) + resources = package / RESOURCES + shutil.copyfile(model_path, resources / MODEL_FILE) + (resources / EXTENSION_FILE).write_bytes( + root_extension_source(root, lock, target["rootExtension"]) + ) + (package / FILE_MANIFEST).write_text(file_manifest(package), encoding="utf-8") + return package + + +def tree(directory: Path) -> dict[str, bytes]: + return { + path.relative_to(directory).as_posix(): path.read_bytes() + for path in sorted(directory.rglob("*")) + if path.is_file() + } + + +def differences(expected: Path, actual: Path) -> list[str]: + left, right = tree(expected), tree(actual) if actual.is_dir() else {} + return [ + name + for name in sorted(set(left) | set(right)) + if left.get(name) != right.get(name) + ] + + +def run( + root: Path, targets: list[Target], rc: Path, core: Path, synchronize: bool +) -> None: + selected = of_language(targets, LANGUAGE) + if not selected: + summarize("No TypeScript targets selected; nothing to do.") + return + catalog = yaml.safe_load( + (root / TOOLING / "packages.yaml").read_text(encoding="utf-8") + ) + with tempfile.TemporaryDirectory(prefix="typescript-bindings-") as name: + for key, directory, _ in selected: + package = generate(root, catalog, key.split(":")[0], rc, core, Path(name)) + destination = root / directory + if synchronize: + shutil.rmtree(destination, ignore_errors=True) + shutil.copytree(package, destination) + elif changed := differences(package, destination): + raise ValueError( + f"{key}: generated source differs from {directory}: " + + ", ".join(changed) + ) + verb = "Verified and synchronized" if synchronize else "Verified" + summarize(f"{verb} {len(selected)} TypeScript binding targets.") + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("command", choices=["configure", "update", "check"]) + parser.add_argument("--root", type=Path, default=Path.cwd()) + parser.add_argument("--targets", default="") + parser.add_argument("--rc", type=Path) + parser.add_argument("--core-schema", type=Path) + args = parser.parse_args() + root = args.root.resolve() + try: + targets = select_targets(root / TOOLING / "packages.yaml", args.targets) + if args.command == "configure": + version = exact_version(targets, LANGUAGE) + write_outputs( + typescript_version=version, + has_typescript="true" if version else "false", + ) + else: + if args.rc is None or args.core_schema is None: + raise ValueError(f"{args.command} requires --rc and --core-schema") + run(root, targets, args.rc, args.core_schema, args.command == "update") + except (ValueError, KeyError, OSError, subprocess.CalledProcessError) as error: + parser.exit(1, f"binding generation failed: {error}\n") + + +if __name__ == "__main__": + main() diff --git a/tooling/extension-bindings/ci/generation.py b/tooling/extension-bindings/ci/generation.py index 193aca6..b1cba1d 100644 --- a/tooling/extension-bindings/ci/generation.py +++ b/tooling/extension-bindings/ci/generation.py @@ -10,7 +10,7 @@ import yaml -LANGUAGES = {"go": "Go", "python": "Python"} +LANGUAGES = {"go": "Go", "python": "Python", "typescript": "TypeScript"} Target = tuple[str, str, str] @@ -34,11 +34,19 @@ def exact_version(targets: list[Target], language: str) -> str: return versions.pop() if versions else "" +def join_labels(labels: list[str]) -> str: + if len(labels) < 3: + return " and ".join(labels) + return ", ".join(labels[:-1]) + " and " + labels[-1] + + def describe(targets: list[Target]) -> str: - return " and ".join( - label - for language, label in LANGUAGES.items() - if of_language(targets, language) + return join_labels( + [ + label + for language, label in LANGUAGES.items() + if of_language(targets, language) + ] ) @@ -79,7 +87,7 @@ def select_targets( ) if not keys or len(set(keys)) != len(keys): raise ValueError("select at least one binding target without duplicates") - enabled = " and ".join(LANGUAGES[language] for language in languages) + enabled = join_labels([LANGUAGES[language] for language in languages]) for key in keys: if key not in targets: raise ValueError( diff --git a/tooling/extension-bindings/ci/test_generate_typescript.py b/tooling/extension-bindings/ci/test_generate_typescript.py new file mode 100644 index 0000000..b6d63b5 --- /dev/null +++ b/tooling/extension-bindings/ci/test_generate_typescript.py @@ -0,0 +1,202 @@ +"""Exercise the TypeScript generation script.""" + +from __future__ import annotations + +import hashlib +from pathlib import Path +from unittest.mock import patch + +from generation import select_targets +from testsupport import RepoTestCase, load_script, run_script + +script = load_script("generate-typescript.py") + +ROOT_EXTENSION = "https://example.test/owner/v1/extension.yaml" +ADDON_EXTENSION = "https://example.test/addon/v1/extension.yaml" + + +def typescript(name: str, directory: str, version: str = "0.1.0") -> dict: + return { + "coordinate": name, + "name": name, + "version": version, + "sourceDirectory": directory, + "languageVersion": "20.19.6", + "publicationMode": "registry", + "registryId": "npm", + } + + +class TypeScriptGenerationTests(RepoTestCase): + def setUp(self) -> None: + super().setUp() + self.packages["owner"]["languages"]["typescript"] = typescript( + "@example/owner", "bindings/owner/typescript" + ) + self.packages["owner"]["rootExtension"] = ROOT_EXTENSION + self.packages["addon"]["languages"]["typescript"] = typescript( + "@example/addon", "bindings/addon/typescript" + ) + self.packages["addon"]["rootExtension"] = ADDON_EXTENSION + self.write_catalog() + self.catalog_document = { + "repositoryUrl": "https://github.com/example/extensions", + "packages": self.packages, + } + + def test_configure_reports_the_node_version_and_whether_typescript_is_selected(self) -> None: + output = self.root.parent / "github-output" + for targets, version, has_typescript in [ + ("owner:typescript", "20.19.6", "true"), + ("owner:go", "", "false"), + ("owner:go,owner:typescript", "20.19.6", "true"), + ]: + with self.subTest(targets=targets): + values = run_script( + "generate-typescript.py", + self.root, + output, + "configure", + "--targets", + targets, + ) + self.assertEqual( + values, + {"typescript-version": version, "has-typescript": has_typescript}, + ) + + def test_the_emitter_digest_follows_its_sources_and_not_its_build(self) -> None: + emitter = self.root / "emitter" + for name, data in { + "package.json": "{}", + "package-lock.json": "{}", + "package-target.schema.yaml": "type: object\n", + "src/plan.ts": "export {};\n", + "templates/package.json.tmpl": "{}", + }.items(): + path = emitter / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(data, encoding="utf-8") + first = script.emitter_digest(emitter) + (emitter / "dist").mkdir() + (emitter / "dist/plan.js").write_text("built\n", encoding="utf-8") + self.assertEqual(script.emitter_digest(emitter), first) + (emitter / "src/plan.ts").write_text("export const x = 1;\n", encoding="utf-8") + self.assertNotEqual(script.emitter_digest(emitter), first) + + def test_the_file_manifest_lists_every_file_but_itself_in_the_go_format(self) -> None: + package = self.root / "package" + (package / "src").mkdir(parents=True) + (package / "src/b.ts").write_text("b\n", encoding="utf-8") + (package / "a.json").write_text("a\n", encoding="utf-8") + (package / script.FILE_MANIFEST).write_text("stale\n", encoding="utf-8") + digest = lambda text: hashlib.sha256(text.encode()).hexdigest() + self.assertEqual( + script.file_manifest(package), + "apiVersion: runtimeconditions.io/file-manifest/v1alpha1\n" + "kind: RuntimeConditionsFileManifest\n" + "files:\n" + f" a.json: {digest('a' + chr(10))}\n" + f" src/b.ts: {digest('b' + chr(10))}\n", + ) + + def test_the_package_target_comes_from_the_catalog_and_the_model(self) -> None: + model = {"dependencyEdges": [{"from": ROOT_EXTENSION, "to": ADDON_EXTENSION}]} + target = script.package_target(self.catalog_document, "owner", model, "f" * 64) + self.assertEqual(target["packageName"], "@example/owner") + self.assertEqual(target["sourceDirectory"], "bindings/owner/typescript") + self.assertEqual(target["minimumNodeVersion"], "20.19") + self.assertEqual(target["registryId"], "npm") + self.assertEqual(target["repositoryUrl"], "https://github.com/example/extensions") + self.assertEqual(target["emitterSha256"], "f" * 64) + self.assertEqual( + target["dependencies"], + [{"extension": ADDON_EXTENSION, "packageName": "@example/addon", "version": "0.1.0"}], + ) + + def test_a_package_without_dependency_edges_has_no_dependencies(self) -> None: + target = script.package_target(self.catalog_document, "addon", {}, "f" * 64) + self.assertEqual(target["dependencies"], []) + + def test_a_dependency_needs_exactly_one_typescript_provider(self) -> None: + del self.packages["addon"]["languages"]["typescript"] + model = {"dependencyEdges": [{"from": ROOT_EXTENSION, "to": ADDON_EXTENSION}]} + with self.assertRaisesRegex(ValueError, "found 0"): + script.package_target(self.catalog_document, "owner", model, "f" * 64) + + def test_the_root_extension_is_read_from_the_catalog_and_checked_against_the_lock(self) -> None: + source = b"extension: definition\n" + self.write("catalog/owner/owner-v1.yaml", source.decode()) + lock = { + "extensions": [ + { + "id": ROOT_EXTENSION, + "sourceBackend": "catalog", + "sourceLocator": "catalog:owner/owner-v1.yaml", + "sourceSha256": hashlib.sha256(source).hexdigest(), + } + ] + } + self.assertEqual(script.root_extension_source(self.root, lock, ROOT_EXTENSION), source) + lock["extensions"][0]["sourceSha256"] = "0" * 64 + with self.assertRaisesRegex(ValueError, "differs from the dependency lock"): + script.root_extension_source(self.root, lock, ROOT_EXTENSION) + lock["extensions"][0]["sourceLocator"] = "https:example.test/owner.yaml" + with self.assertRaisesRegex(ValueError, "only catalog extension sources"): + script.root_extension_source(self.root, lock, ROOT_EXTENSION) + with self.assertRaisesRegex(ValueError, "exactly once"): + script.root_extension_source(self.root, lock, ADDON_EXTENSION) + + def test_generation_needs_the_built_emitter(self) -> None: + with patch.dict("os.environ", {}, clear=True), self.assertRaisesRegex( + ValueError, "build the TypeScript emitter first" + ): + script.tools(self.root) + built = self.root / "main.js" + built.write_text("", encoding="utf-8") + with patch.dict("os.environ", {"RC_TYPESCRIPT_EMITTER": str(built)}, clear=True): + self.assertEqual(script.tools(self.root), ("node", built)) + + def generated(self, files: dict[str, str]): + def generate(root, catalog, key, rc, core, work): + package = work / key + for name, data in files.items(): + path = package / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(data, encoding="utf-8") + return package + + return generate + + def test_update_replaces_the_committed_tree_and_check_compares_it(self) -> None: + self.write("tooling/extension-bindings/packages.yaml", self.catalog.read_text(encoding="utf-8")) + self.write("bindings/owner/typescript/stale.ts", "old generated source\n") + targets = select_targets(self.catalog, "owner:typescript") + files = {"package.json": "{}\n", "src/bindings.ts": "new\n"} + with patch.object(script, "generate", self.generated(files)): + with self.assertRaisesRegex(ValueError, "stale.ts"): + script.run(self.root, targets, Path("rc"), Path("core.yaml"), False) + script.run(self.root, targets, Path("rc"), Path("core.yaml"), True) + script.run(self.root, targets, Path("rc"), Path("core.yaml"), False) + committed = self.root / "bindings/owner/typescript" + self.assertFalse((committed / "stale.ts").exists()) + self.assertEqual((committed / "src/bindings.ts").read_text(encoding="utf-8"), "new\n") + + def test_update_leaves_other_languages_alone(self) -> None: + targets = select_targets(self.catalog, "owner:go") + with patch.object(script, "generate") as generate: + script.run(self.root, targets, Path("rc"), Path("core.yaml"), True) + generate.assert_not_called() + self.assertTrue((self.root / "bindings/owner/go/stale.go").exists()) + + def test_typescript_targets_carry_their_directory_and_exact_node_version(self) -> None: + self.assertIn( + ("owner:typescript", "bindings/owner/typescript", "20.19.6"), + select_targets(self.catalog, ""), + ) + + +if __name__ == "__main__": + import unittest + + unittest.main() diff --git a/tooling/extension-bindings/orchestrator/plan.go b/tooling/extension-bindings/orchestrator/plan.go index 93b439f..9364686 100644 --- a/tooling/extension-bindings/orchestrator/plan.go +++ b/tooling/extension-bindings/orchestrator/plan.go @@ -20,12 +20,18 @@ type BuildPlan struct { Targets []Target `yaml:"targets" json:"targets"` } +// nativeLanguages are the languages rc itself emits, verifies, and packages. Other catalog +// languages are generated by their own scripts, so only an explicit target selects them. +var nativeLanguages = map[string]bool{"go": true, "python": true} + func (p *Project) SelectTargets(keys []string) (BuildPlan, error) { selected := map[string]bool{} if len(keys) == 0 { for key, set := range p.Catalog.Packages { for lang := range set.Languages { - selected[key+":"+lang] = true + if nativeLanguages[lang] { + selected[key+":"+lang] = true + } } } } else { diff --git a/tooling/extension-bindings/orchestrator/plan_test.go b/tooling/extension-bindings/orchestrator/plan_test.go index 1d70b1e..ebde6cc 100644 --- a/tooling/extension-bindings/orchestrator/plan_test.go +++ b/tooling/extension-bindings/orchestrator/plan_test.go @@ -204,3 +204,25 @@ func TestCatalogRejectsUnsafeModuleCoordinate(t *testing.T) { t.Fatal("unsafe module coordinate accepted") } } + +func TestDefaultSelectionLeavesLanguagesWithoutANativeAdapterToTheirOwnScripts(t *testing.T) { + p, _ := testProject(t, "go") + set := p.Catalog.Packages["new-channel"] + set.Languages["typescript"] = LanguageTarget{Coordinate: "@example/new-channel", Name: "@example/new-channel", Version: "0.7.0", SourceDirectory: "bindings/new-channel/typescript", LanguageVersion: "20.19.6", PublicationMode: "github-tag"} + plan, err := p.SelectTargets(nil) + if err != nil { + t.Fatal(err) + } + for _, target := range plan.Targets { + if target.Language == "typescript" { + t.Fatalf("default selection included %s", target.Key) + } + } + if len(plan.Targets) != 1 { + t.Fatalf("default selection %v", plan.Targets) + } + plan, err = p.SelectTargets([]string{"new-channel:typescript"}) + if err != nil || len(plan.Targets) != 1 || plan.Targets[0].Language != "typescript" { + t.Fatalf("explicit selection %v %v", plan.Targets, err) + } +} diff --git a/tooling/extension-bindings/packages.yaml b/tooling/extension-bindings/packages.yaml index 15e0348..7943e4e 100644 --- a/tooling/extension-bindings/packages.yaml +++ b/tooling/extension-bindings/packages.yaml @@ -41,6 +41,14 @@ packages: languageVersion: 3.12.10 publicationMode: registry registryId: pypi + typescript: + coordinate: '@runtimeconditions/common-integrations' + name: '@runtimeconditions/common-integrations' + version: 0.1.0 + sourceDirectory: bindings/common-integrations/typescript + languageVersion: 20.19.6 + publicationMode: registry + registryId: npm env-configuration: rootExtension: https://runtimeconditions.io/extensions/env-configuration/v1alpha1/runtimeconditions.extension.yaml coreProfileSchemaVersion: 0.2.0 @@ -60,6 +68,14 @@ packages: languageVersion: 3.12.10 publicationMode: registry registryId: pypi + typescript: + coordinate: '@runtimeconditions/env-configuration' + name: '@runtimeconditions/env-configuration' + version: 0.1.0 + sourceDirectory: bindings/env-configuration/typescript + languageVersion: 20.19.6 + publicationMode: registry + registryId: npm google-analytics: rootExtension: https://runtimeconditions.io/extensions/google/analytics/0.1.0/runtimeconditions.extension.yaml coreProfileSchemaVersion: 0.2.0 From 80bf17a178d4a8751eba6559caeffbe8d0bec597 Mon Sep 17 00:00:00 2001 From: itvi-1234 Date: Fri, 23 Oct 2026 14:46:50 +0530 Subject: [PATCH 2/2] Publish TypeScript bindings to npm with trusted publishing Replace the placeholder with a workflow that builds the committed TypeScript packages in dependency order, checks them with npm pack, and publishes the ones that are not on npm yet. The job requests an OIDC token (id-token: write), so npm authenticates the workflow itself and no npm token is stored. It defaults to a dry run. Signed-off-by: itvi-1234 --- .github/workflows/binding-publish-npm.yml | 65 +++++--- tooling/extension-bindings/ci/publish-npm.py | 143 ++++++++++++++++++ .../extension-bindings/ci/test_publish_npm.py | 123 +++++++++++++++ 3 files changed, 310 insertions(+), 21 deletions(-) create mode 100644 tooling/extension-bindings/ci/publish-npm.py create mode 100644 tooling/extension-bindings/ci/test_publish_npm.py diff --git a/.github/workflows/binding-publish-npm.yml b/.github/workflows/binding-publish-npm.yml index cdb1cae..daa24f2 100644 --- a/.github/workflows/binding-publish-npm.yml +++ b/.github/workflows/binding-publish-npm.yml @@ -1,4 +1,4 @@ -name: Publish TypeScript bindings (placeholder, publishes nothing) +name: Publish TypeScript bindings on: workflow_dispatch: @@ -8,7 +8,7 @@ on: type: string default: '' dry_run: - description: 'Must stay true: the workflow is a placeholder and verifies and publishes nothing' + description: 'Build and check the packages without publishing them to npm' type: boolean default: true @@ -23,26 +23,49 @@ jobs: publish: if: github.ref == 'refs/heads/main' runs-on: ubuntu-24.04 - timeout-minutes: 15 + timeout-minutes: 20 + permissions: + contents: read + # Trusted publishing: npm accepts this workflow's OIDC identity, so no npm token is stored. + id-token: write + env: + INPUT_TARGETS: ${{ inputs.targets }} + EXECUTE: ${{ !inputs.dry_run }} steps: - - name: Reject real publication - if: ${{ !inputs.dry_run }} - run: | - echo "::error::npm publication is not implemented. Run with dry_run enabled; nothing was published." - exit 1 - - name: Record that nothing was verified or published - env: - TARGETS: ${{ inputs.targets }} - DRY_RUN: ${{ inputs.dry_run }} + - name: Check out repository + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + with: + persist-credentials: false + + - name: Set up Python for the publication helper + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 + with: + python-version: '3.12.10' + + - name: Install the helper dependency + run: python -m pip install --disable-pip-version-check PyYAML==6.0.3 + + - name: Set up Node.js for npm trusted publishing + uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6.5.0 + with: + node-version: '24.11.0' + registry-url: 'https://registry.npmjs.org' + package-manager-cache: false + + - name: Install the pinned emitter dependencies, which provide the TypeScript compiler + working-directory: tooling/extension-bindings/emitters/typescript + run: npm ci --ignore-scripts + + - name: Test the publication helper + run: python -m unittest discover -s tooling/extension-bindings/ci -p 'test_publish_npm.py' + + - name: Build and check the committed packages, then publish them + shell: bash run: | set -euo pipefail - { - echo "## Placeholder: nothing was verified or published" - echo - echo "npm publication is not implemented. This run did not verify, build, or publish any package." - echo - echo "- targets: \`${TARGETS:-}\`" - echo "- dry_run: \`${DRY_RUN}\`" - } >> "$GITHUB_STEP_SUMMARY" - echo "::notice::Placeholder only. No package was verified or published." + args=() + if [[ "$EXECUTE" == true ]]; then + args+=(--execute) + fi + python tooling/extension-bindings/ci/publish-npm.py --targets "$INPUT_TARGETS" "${args[@]}" diff --git a/tooling/extension-bindings/ci/publish-npm.py b/tooling/extension-bindings/ci/publish-npm.py new file mode 100644 index 0000000..be1dea0 --- /dev/null +++ b/tooling/extension-bindings/ci/publish-npm.py @@ -0,0 +1,143 @@ +"""Build the committed TypeScript bindings and publish them to npm with trusted publishing.""" + +from __future__ import annotations + +import argparse +import json +import os +import shutil +import subprocess +import tempfile +from pathlib import Path + +import yaml + +from generation import Target, of_language, select_targets, summarize + +LANGUAGE = "typescript" +TOOLING = Path("tooling/extension-bindings") +EMITTER = TOOLING / "emitters/typescript" +MINIMUM_NPM = (11, 5, 1) + + +def read_package(root: Path, catalog: dict, target: Target) -> dict: + """Read a committed package.json and check it against the catalog entry.""" + key, directory, _ = target + package = json.loads((root / directory / "package.json").read_text(encoding="utf-8")) + expected = catalog["packages"][key.split(":")[0]]["languages"][LANGUAGE] + if package["name"] != expected["name"] or package["version"] != str(expected["version"]): + raise ValueError( + f"{key}: {directory}/package.json is " + f"{package['name']}@{package['version']}, but the catalog says " + f"{expected['name']}@{expected['version']}" + ) + return package + + +def publication_order(packages: dict[str, dict]) -> list[str]: + """Order package names so every selected dependency comes before its dependents.""" + ordered: list[str] = [] + state: dict[str, int] = {} + + def visit(name: str) -> None: + if state.get(name) == 2: + return + if state.get(name) == 1: + raise ValueError(f"package dependency cycle at {name}") + state[name] = 1 + for dependency in sorted(packages[name].get("dependencies", {})): + if dependency in packages: + visit(dependency) + state[name] = 2 + ordered.append(name) + + for name in sorted(packages): + visit(name) + return ordered + + +def check_npm_version(version: str) -> None: + numbers = tuple(int(part) for part in version.strip().split("-")[0].split(".")[:3]) + if numbers < MINIMUM_NPM: + raise ValueError( + "trusted publishing needs npm " + + ".".join(map(str, MINIMUM_NPM)) + + f" or later, but npm {version.strip()} is installed" + ) + + +def run(command: list[str], cwd: Path, **options) -> subprocess.CompletedProcess: + return subprocess.run(command, cwd=cwd, check=True, text=True, **options) + + +def already_published(name: str, version: str) -> bool: + result = subprocess.run( + ["npm", "view", f"{name}@{version}", "version"], + capture_output=True, + text=True, + ) + return result.returncode == 0 and result.stdout.strip() == version + + +def publish(root: Path, targets: list[Target], execute: bool) -> None: + selected = of_language(targets, LANGUAGE) + if not selected: + summarize("No TypeScript targets selected; nothing to publish.") + return + tsc = root / EMITTER / "node_modules/typescript/bin/tsc" + if not tsc.is_file(): + raise ValueError(f"install the emitter dependencies first (npm ci in {root / EMITTER}): {tsc} is missing") + if execute: + check_npm_version(run(["npm", "--version"], root, capture_output=True).stdout) + catalog = yaml.safe_load((root / TOOLING / "packages.yaml").read_text(encoding="utf-8")) + packages = {} + directories = {} + for target in selected: + package = read_package(root, catalog, target) + packages[package["name"]] = package + directories[package["name"]] = root / target[1] + published = [] + with tempfile.TemporaryDirectory(prefix="npm-bindings-") as name: + # Each package is built where its dependencies resolve, from the same + # committed tree a consumer would receive. + workspace = Path(name) + for package_name in publication_order(packages): + destination = workspace / "node_modules" / package_name + shutil.copytree(directories[package_name], destination) + run(["node", str(tsc), "-p", str(destination / "tsconfig.json")], root) + run(["npm", "pack", "--dry-run", "--ignore-scripts"], destination, capture_output=True) + version = packages[package_name]["version"] + if not execute: + summarize(f"Built and checked {package_name}@{version}; dry run, nothing was published.") + elif already_published(package_name, version): + summarize(f"{package_name}@{version} is already on npm; skipped.") + else: + run(["npm", "publish", "--access", "public", "--ignore-scripts"], destination) + published.append(f"{package_name}@{version}") + summarize(f"Published {package_name}@{version}.") + if execute: + summarize("Published to npm: " + (", ".join(published) if published else "nothing new.")) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--root", type=Path, default=Path.cwd()) + parser.add_argument("--targets", default="") + parser.add_argument( + "--execute", + action="store_true", + help="publish; without it the packages are only built and checked", + ) + args = parser.parse_args() + root = args.root.resolve() + try: + targets = select_targets(root / TOOLING / "packages.yaml", args.targets) + publish(root, targets, args.execute) + except (ValueError, KeyError, OSError, subprocess.CalledProcessError) as error: + if isinstance(error, subprocess.CalledProcessError) and error.stderr: + print(error.stderr) + parser.exit(1, f"npm publication failed: {error}\n") + + +if __name__ == "__main__": + main() diff --git a/tooling/extension-bindings/ci/test_publish_npm.py b/tooling/extension-bindings/ci/test_publish_npm.py new file mode 100644 index 0000000..40b061e --- /dev/null +++ b/tooling/extension-bindings/ci/test_publish_npm.py @@ -0,0 +1,123 @@ +"""Exercise the npm publication script.""" + +from __future__ import annotations + +import json +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch + +from generation import select_targets +from testsupport import RepoTestCase, load_script + +script = load_script("publish-npm.py") + + +class NpmPublicationTests(RepoTestCase): + def setUp(self) -> None: + super().setUp() + for key, name in (("owner", "@example/owner"), ("addon", "@example/addon")): + self.packages[key]["languages"]["typescript"] = { + "coordinate": name, + "name": name, + "version": "0.1.0", + "sourceDirectory": f"bindings/{key}/typescript", + "languageVersion": "20.19.6", + "publicationMode": "registry", + "registryId": "npm", + } + self.write_catalog() + self.package("addon", "@example/addon") + self.package("owner", "@example/owner", {"@example/addon": "^0.1.0"}) + self.write("tooling/extension-bindings/emitters/typescript/node_modules/typescript/bin/tsc", "") + self.targets = select_targets(self.catalog, "owner:typescript,addon:typescript") + + def package(self, key: str, name: str, dependencies: dict | None = None, version: str = "0.1.0") -> None: + self.write( + f"bindings/{key}/typescript/package.json", + json.dumps({"name": name, "version": version, "dependencies": dependencies or {}}), + ) + self.write(f"bindings/{key}/typescript/tsconfig.json", "{}") + + def commands(self, calls) -> list[list[str]]: + return [call.args[0] for call in calls] + + def test_dependencies_are_published_before_their_dependents(self) -> None: + packages = { + "@example/owner": {"dependencies": {"@example/addon": "^0.1.0", "left-pad": "^1.0.0"}}, + "@example/addon": {}, + "@example/leaf": {"dependencies": {"@example/owner": "^0.1.0"}}, + } + self.assertEqual( + script.publication_order(packages), + ["@example/addon", "@example/owner", "@example/leaf"], + ) + packages["@example/addon"] = {"dependencies": {"@example/leaf": "^0.1.0"}} + with self.assertRaisesRegex(ValueError, "cycle"): + script.publication_order(packages) + + def test_trusted_publishing_needs_a_recent_npm(self) -> None: + for version in ("11.5.1\n", "11.6.0", "12.0.0-pre.1"): + script.check_npm_version(version) + for version in ("10.8.2", "11.5.0"): + with self.assertRaisesRegex(ValueError, "11.5.1 or later"): + script.check_npm_version(version) + + def test_a_package_must_match_its_catalog_entry(self) -> None: + self.package("owner", "@example/other") + catalog = {"packages": self.packages} + with self.assertRaisesRegex(ValueError, "catalog says"): + script.read_package(self.root, catalog, self.targets[1]) + + def test_a_dry_run_builds_and_checks_but_never_publishes(self) -> None: + with patch.object(script, "run") as run, patch.object( + script, "already_published" + ) as published: + script.publish(self.root, self.targets, False) + commands = self.commands(run.call_args_list) + self.assertFalse(any(command[:2] == ["npm", "publish"] for command in commands)) + self.assertEqual(sum(command[:2] == ["npm", "pack"] for command in commands), 2) + builds = [command[3] for command in commands if command[0] == "node"] + self.assertTrue(builds[0].endswith("node_modules/@example/addon/tsconfig.json")) + self.assertTrue(builds[1].endswith("node_modules/@example/owner/tsconfig.json")) + published.assert_not_called() + + def test_publication_skips_versions_that_are_already_on_npm(self) -> None: + def run(command, cwd, **options): + return SimpleNamespace(stdout="11.6.0\n") + + with patch.object(script, "run", side_effect=run) as runner, patch.object( + script, "already_published", side_effect=lambda name, version: name == "@example/addon" + ): + script.publish(self.root, self.targets, True) + publishes = [ + call for call in runner.call_args_list if call.args[0][:2] == ["npm", "publish"] + ] + self.assertEqual(len(publishes), 1) + self.assertEqual( + publishes[0].args[0], ["npm", "publish", "--access", "public", "--ignore-scripts"] + ) + self.assertTrue(str(publishes[0].args[1]).endswith("node_modules/@example/owner")) + + def test_publication_refuses_an_old_npm(self) -> None: + with patch.object( + script, "run", return_value=SimpleNamespace(stdout="10.8.2\n") + ), self.assertRaisesRegex(ValueError, "npm 11.5.1"): + script.publish(self.root, self.targets, True) + + def test_publication_needs_the_emitter_dependencies(self) -> None: + (self.root / "tooling/extension-bindings/emitters/typescript/node_modules/typescript/bin/tsc").unlink() + with self.assertRaisesRegex(ValueError, "npm ci"): + script.publish(self.root, self.targets, False) + + def test_other_languages_are_not_published(self) -> None: + targets = select_targets(self.catalog, "owner:go") + with patch.object(script, "run") as run: + script.publish(self.root, targets, True) + run.assert_not_called() + + +if __name__ == "__main__": + import unittest + + unittest.main()