diff --git a/.github/actions/binding-tools/action.yml b/.github/actions/binding-tools/action.yml index 78d8c11..5f5f243 100644 --- a/.github/actions/binding-tools/action.yml +++ b/.github/actions/binding-tools/action.yml @@ -2,7 +2,7 @@ name: Prepare binding tools description: Select targets, install the exact compilers and prerequisites, and build the shared generator inputs: targets: - description: Comma-separated package-key:go or package-key:python targets, or all + description: Comma-separated package-key:go, package-key:python, or package-key:typescript targets, or all required: true publication: description: Include the committed package dependency closure for publication @@ -17,6 +17,12 @@ outputs: has-python: description: Whether a Python target is selected value: ${{ steps.configure.outputs.has-python }} + typescript-version: + description: Exact catalog Node.js version, empty when no TypeScript target is selected + value: ${{ steps.configure.outputs.typescript-version }} + has-typescript: + description: Whether a TypeScript target is selected + value: ${{ steps.configure.outputs.has-typescript }} source-commit: description: Checked-out source commit value: ${{ steps.configure.outputs.source-commit }} @@ -53,6 +59,7 @@ runs: python tooling/extension-bindings/ci/generation.py configure --targets "$INPUT_TARGETS" python tooling/extension-bindings/ci/generate-go.py configure --targets "$INPUT_TARGETS" python tooling/extension-bindings/ci/generate-python.py configure --targets "$INPUT_TARGETS" + python tooling/extension-bindings/ci/generate-typescript.py configure --targets "$INPUT_TARGETS" fi - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 with: @@ -110,6 +117,18 @@ runs: printf 'RC_PYTHON_PROFILER_BIN=%s\n' "$tools/python-venv/bin/runtimeconditions-python-profiler" printf 'RC_PYTHON_PROFILER_ARTIFACT=%s\n' "$wheel" } >> "$GITHUB_ENV" + - name: Set up the pinned Node.js + if: steps.configure.outputs.has-typescript == 'true' + uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6.5.0 + with: + node-version: ${{ steps.configure.outputs.typescript-version }} + cache: npm + cache-dependency-path: tooling/extension-bindings/emitters/typescript/package-lock.json + - name: Build the TypeScript emitter from its pinned dependencies + if: steps.configure.outputs.has-typescript == 'true' + shell: bash + working-directory: tooling/extension-bindings/emitters/typescript + run: npm ci --ignore-scripts && npm run build - name: Build shared generator shell: bash run: go -C tooling/extension-bindings/orchestrator build -trimpath -buildvcs=false -o "$RUNNER_TEMP/binding-tools/rc" ./cmd/rc diff --git a/.github/workflows/binding-generate.yml b/.github/workflows/binding-generate.yml index 47dd044..4de9d4b 100644 --- a/.github/workflows/binding-generate.yml +++ b/.github/workflows/binding-generate.yml @@ -1,4 +1,4 @@ -name: Generate and commit Go and Python bindings +name: Generate and commit Go, Python and TypeScript bindings on: push: @@ -10,18 +10,20 @@ on: - 'catalog/**' - 'bindings/**/go/**' - 'bindings/**/python/**' + - 'bindings/**/typescript/**' - 'tooling/extension-bindings/packages.yaml' - 'tooling/extension-bindings/toolchain.lock.yaml' - 'tooling/extension-bindings/model/**' - 'tooling/extension-bindings/normalizer/**' - 'tooling/extension-bindings/emitters/go/**' - 'tooling/extension-bindings/emitters/python/**' + - 'tooling/extension-bindings/emitters/typescript/**' - 'tooling/extension-bindings/orchestrator/**' - 'tooling/extension-bindings/ci/**' workflow_dispatch: inputs: targets: - description: 'Comma-separated package-key:go or package-key:python targets, or all; empty uses automatic targets' + description: 'Comma-separated package-key:go, package-key:python, or package-key:typescript targets, or all; empty uses automatic targets' type: string default: '' @@ -40,7 +42,7 @@ jobs: permissions: contents: write env: - INPUT_TARGETS: ${{ inputs.targets || vars.BINDING_GENERATION_TARGETS || 'common-integrations:go,env-configuration:go,common-integrations:python,env-configuration:python' }} + INPUT_TARGETS: ${{ inputs.targets || vars.BINDING_GENERATION_TARGETS || 'common-integrations:go,env-configuration:go,common-integrations:python,env-configuration:python,common-integrations:typescript,env-configuration:typescript' }} GOTOOLCHAIN: local steps: - name: Check out current main @@ -59,6 +61,7 @@ jobs: shell: bash env: HAS_PYTHON: ${{ steps.tools.outputs.has-python }} + HAS_TYPESCRIPT: ${{ steps.tools.outputs.has-typescript }} run: | set -euo pipefail python -m unittest discover -s tooling/extension-bindings/ci -p 'test_*.py' @@ -71,6 +74,9 @@ jobs: tooling/extension-bindings/emitters/python/tests \ --ignore=tooling/extension-bindings/emitters/python/tests/test_packages.py fi + if [[ "$HAS_TYPESCRIPT" == true ]]; then + npm --prefix tooling/extension-bindings/emitters/typescript test + fi - name: Generate and verify bindings env: @@ -81,6 +87,8 @@ jobs: --rc "$RC" --core-schema "$CORE_SCHEMA" python tooling/extension-bindings/ci/generate-python.py update --targets "$INPUT_TARGETS" \ --rc "$RC" --core-schema "$CORE_SCHEMA" + python tooling/extension-bindings/ci/generate-typescript.py update --targets "$INPUT_TARGETS" \ + --rc "$RC" --core-schema "$CORE_SCHEMA" - name: Commit verified generated source to main env: diff --git a/.github/workflows/binding-publish-npm.yml b/.github/workflows/binding-publish-npm.yml index cdb1cae..daa24f2 100644 --- a/.github/workflows/binding-publish-npm.yml +++ b/.github/workflows/binding-publish-npm.yml @@ -1,4 +1,4 @@ -name: Publish TypeScript bindings (placeholder, publishes nothing) +name: Publish TypeScript bindings on: workflow_dispatch: @@ -8,7 +8,7 @@ on: type: string default: '' dry_run: - description: 'Must stay true: the workflow is a placeholder and verifies and publishes nothing' + description: 'Build and check the packages without publishing them to npm' type: boolean default: true @@ -23,26 +23,49 @@ jobs: publish: if: github.ref == 'refs/heads/main' runs-on: ubuntu-24.04 - timeout-minutes: 15 + timeout-minutes: 20 + permissions: + contents: read + # Trusted publishing: npm accepts this workflow's OIDC identity, so no npm token is stored. + id-token: write + env: + INPUT_TARGETS: ${{ inputs.targets }} + EXECUTE: ${{ !inputs.dry_run }} steps: - - name: Reject real publication - if: ${{ !inputs.dry_run }} - run: | - echo "::error::npm publication is not implemented. Run with dry_run enabled; nothing was published." - exit 1 - - name: Record that nothing was verified or published - env: - TARGETS: ${{ inputs.targets }} - DRY_RUN: ${{ inputs.dry_run }} + - name: Check out repository + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + with: + persist-credentials: false + + - name: Set up Python for the publication helper + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6 + with: + python-version: '3.12.10' + + - name: Install the helper dependency + run: python -m pip install --disable-pip-version-check PyYAML==6.0.3 + + - name: Set up Node.js for npm trusted publishing + uses: actions/setup-node@249970729cb0ef3589644e2896645e5dc5ba9c38 # v6.5.0 + with: + node-version: '24.11.0' + registry-url: 'https://registry.npmjs.org' + package-manager-cache: false + + - name: Install the pinned emitter dependencies, which provide the TypeScript compiler + working-directory: tooling/extension-bindings/emitters/typescript + run: npm ci --ignore-scripts + + - name: Test the publication helper + run: python -m unittest discover -s tooling/extension-bindings/ci -p 'test_publish_npm.py' + + - name: Build and check the committed packages, then publish them + shell: bash run: | set -euo pipefail - { - echo "## Placeholder: nothing was verified or published" - echo - echo "npm publication is not implemented. This run did not verify, build, or publish any package." - echo - echo "- targets: \`${TARGETS:-}\`" - echo "- dry_run: \`${DRY_RUN}\`" - } >> "$GITHUB_STEP_SUMMARY" - echo "::notice::Placeholder only. No package was verified or published." + args=() + if [[ "$EXECUTE" == true ]]; then + args+=(--execute) + fi + python tooling/extension-bindings/ci/publish-npm.py --targets "$INPUT_TARGETS" "${args[@]}" diff --git a/tooling/extension-bindings/ci/generate-typescript.py b/tooling/extension-bindings/ci/generate-typescript.py new file mode 100644 index 0000000..4a687bc --- /dev/null +++ b/tooling/extension-bindings/ci/generate-typescript.py @@ -0,0 +1,275 @@ +"""Configure, update, and check TypeScript bindings for the Actions workflow.""" + +from __future__ import annotations + +import argparse +import hashlib +import json +import os +import re +import shutil +import subprocess +import tempfile +from pathlib import Path + +import yaml + +from generation import ( + Target, + exact_version, + of_language, + select_targets, + summarize, + write_outputs, +) + +LANGUAGE = "typescript" +TOOLING = Path("tooling/extension-bindings") +EMITTER = TOOLING / "emitters/typescript" +TARGET_API = "runtimeconditions.io/typescript-package-target/v1alpha1" +TARGET_KIND = "RuntimeConditionsTypeScriptPackageTarget" +FILE_MANIFEST = "runtimeconditions.file-manifest.yaml" +MODEL_FILE = "runtimeconditions.binding-model.yaml" +EXTENSION_FILE = "runtimeconditions.extension.yaml" +RESOURCES = "runtimeconditions" + + +def sha256(data: bytes) -> str: + return hashlib.sha256(data).hexdigest() + + +def emitter_digest(emitter: Path) -> str: + """Identify the emitter by its source, so the digest does not depend on a build.""" + names = ["package.json", "package-lock.json", "package-target.schema.yaml"] + for folder, pattern in (("src", "*.ts"), ("templates", "*")): + names += sorted( + path.relative_to(emitter).as_posix() + for path in (emitter / folder).glob(pattern) + if path.is_file() + ) + files = {name: sha256((emitter / name).read_bytes()) for name in names} + return sha256(json.dumps(files, sort_keys=True, separators=(",", ":")).encode()) + + +def file_manifest(directory: Path) -> str: + """List every file except the manifest itself, in the format the Go tooling writes.""" + files = { + path.relative_to(directory).as_posix(): sha256(path.read_bytes()) + for path in directory.rglob("*") + if path.is_file() and path.name != FILE_MANIFEST + } + lines = [ + "apiVersion: runtimeconditions.io/file-manifest/v1alpha1", + "kind: RuntimeConditionsFileManifest", + "files:", + ] + for name in sorted(files, key=lambda item: item.encode("utf-8")): + if not re.fullmatch(r"[A-Za-z0-9_./-]+", name): + raise ValueError(f"unexpected generated file name: {name}") + lines.append(f" {name}: {files[name]}") + return "\n".join(lines) + "\n" + + +def package_target( + catalog: dict, key: str, model: dict, emitter_sha256: str +) -> dict: + """Build the emitter's package target from the catalog and the normalized model.""" + package = catalog["packages"][key] + config = package["languages"][LANGUAGE] + root = package["rootExtension"] + dependencies = [] + for edge in sorted( + item["to"] + for item in model.get("dependencyEdges") or [] + if item["from"] == root + ): + providers = [ + other["languages"][LANGUAGE] + for other in catalog["packages"].values() + if other["rootExtension"] == edge and LANGUAGE in other["languages"] + ] + if len(providers) != 1: + raise ValueError( + f"{key}:{LANGUAGE}: direct dependency {edge} requires exactly one " + f"{LANGUAGE} catalog provider (found {len(providers)})" + ) + dependencies.append( + { + "extension": edge, + "packageName": providers[0]["name"], + "version": str(providers[0]["version"]), + } + ) + target = { + "apiVersion": TARGET_API, + "kind": TARGET_KIND, + "packageKey": key, + "rootExtension": root, + "packageName": config["name"], + "version": str(config["version"]), + "sourceDirectory": config["sourceDirectory"], + "minimumNodeVersion": ".".join(str(config["languageVersion"]).split(".")[:2]), + "publicationMode": config["publicationMode"], + "repositoryUrl": catalog["repositoryUrl"], + "dependencies": dependencies, + "emitterSha256": emitter_sha256, + } + if "registryId" in config: + target["registryId"] = config["registryId"] + return target + + +def root_extension_source(root: Path, lock: dict, extension: str) -> bytes: + """Read the root extension from the catalog, checking it against the dependency lock.""" + entries = [item for item in lock["extensions"] if item["id"] == extension] + if len(entries) != 1: + raise ValueError(f"the dependency lock must list {extension} exactly once") + backend, _, locator = entries[0]["sourceLocator"].partition(":") + if backend != "catalog": + raise ValueError(f"{extension}: only catalog extension sources are supported") + data = (root / "catalog" / locator).read_bytes() + if sha256(data) != entries[0]["sourceSha256"]: + raise ValueError(f"{extension}: catalog source differs from the dependency lock") + return data + + +def tools(root: Path) -> tuple[str, Path]: + node = os.environ.get("RC_BINDINGS_NODE", "node") + main = Path( + os.environ.get("RC_TYPESCRIPT_EMITTER", root / EMITTER / "dist/src/main.js") + ) + if not main.is_file(): + raise ValueError( + f"build the TypeScript emitter first (npm ci && npm run build in {root / EMITTER}): " + f"{main} is missing" + ) + return node, main + + +def generate( + root: Path, catalog: dict, key: str, rc: Path, core: Path, work: Path +) -> Path: + """Generate one verified package tree beneath work and return its directory.""" + node, main = tools(root) + tooling = root / TOOLING + model_path, lock_path = work / f"{key}.model.yaml", work / f"{key}.lock.yaml" + subprocess.run( + [ + str(rc.resolve()), + "bindings", + "normalize", + "--packages", + str(tooling / "packages.yaml"), + "--toolchain-lock", + str(tooling / "toolchain.lock.yaml"), + "--core-schema", + str(core.resolve()), + "--target", + f"{key}:{LANGUAGE}", + "--output", + str(model_path), + "--dependency-lock-output", + str(lock_path), + ], + cwd=root, + check=True, + stdout=subprocess.DEVNULL, + ) + model = yaml.safe_load(model_path.read_text(encoding="utf-8")) + lock = yaml.safe_load(lock_path.read_text(encoding="utf-8")) + target = package_target(catalog, key, model, emitter_digest(root / EMITTER)) + target_path = work / f"{key}.target.yaml" + target_path.write_text(yaml.safe_dump(target, sort_keys=False), encoding="utf-8") + package = work / key + common = ["--model", str(model_path), "--package-config", str(target_path)] + subprocess.run( + [node, str(main), "emit", *common, "--output", str(package)], + cwd=root, + check=True, + stdout=subprocess.DEVNULL, + ) + subprocess.run( + [node, str(main), "verify", *common, "--input", str(package)], + cwd=root, + check=True, + stdout=subprocess.DEVNULL, + ) + resources = package / RESOURCES + shutil.copyfile(model_path, resources / MODEL_FILE) + (resources / EXTENSION_FILE).write_bytes( + root_extension_source(root, lock, target["rootExtension"]) + ) + (package / FILE_MANIFEST).write_text(file_manifest(package), encoding="utf-8") + return package + + +def tree(directory: Path) -> dict[str, bytes]: + return { + path.relative_to(directory).as_posix(): path.read_bytes() + for path in sorted(directory.rglob("*")) + if path.is_file() + } + + +def differences(expected: Path, actual: Path) -> list[str]: + left, right = tree(expected), tree(actual) if actual.is_dir() else {} + return [ + name + for name in sorted(set(left) | set(right)) + if left.get(name) != right.get(name) + ] + + +def run( + root: Path, targets: list[Target], rc: Path, core: Path, synchronize: bool +) -> None: + selected = of_language(targets, LANGUAGE) + if not selected: + summarize("No TypeScript targets selected; nothing to do.") + return + catalog = yaml.safe_load( + (root / TOOLING / "packages.yaml").read_text(encoding="utf-8") + ) + with tempfile.TemporaryDirectory(prefix="typescript-bindings-") as name: + for key, directory, _ in selected: + package = generate(root, catalog, key.split(":")[0], rc, core, Path(name)) + destination = root / directory + if synchronize: + shutil.rmtree(destination, ignore_errors=True) + shutil.copytree(package, destination) + elif changed := differences(package, destination): + raise ValueError( + f"{key}: generated source differs from {directory}: " + + ", ".join(changed) + ) + verb = "Verified and synchronized" if synchronize else "Verified" + summarize(f"{verb} {len(selected)} TypeScript binding targets.") + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("command", choices=["configure", "update", "check"]) + parser.add_argument("--root", type=Path, default=Path.cwd()) + parser.add_argument("--targets", default="") + parser.add_argument("--rc", type=Path) + parser.add_argument("--core-schema", type=Path) + args = parser.parse_args() + root = args.root.resolve() + try: + targets = select_targets(root / TOOLING / "packages.yaml", args.targets) + if args.command == "configure": + version = exact_version(targets, LANGUAGE) + write_outputs( + typescript_version=version, + has_typescript="true" if version else "false", + ) + else: + if args.rc is None or args.core_schema is None: + raise ValueError(f"{args.command} requires --rc and --core-schema") + run(root, targets, args.rc, args.core_schema, args.command == "update") + except (ValueError, KeyError, OSError, subprocess.CalledProcessError) as error: + parser.exit(1, f"binding generation failed: {error}\n") + + +if __name__ == "__main__": + main() diff --git a/tooling/extension-bindings/ci/generation.py b/tooling/extension-bindings/ci/generation.py index 193aca6..b1cba1d 100644 --- a/tooling/extension-bindings/ci/generation.py +++ b/tooling/extension-bindings/ci/generation.py @@ -10,7 +10,7 @@ import yaml -LANGUAGES = {"go": "Go", "python": "Python"} +LANGUAGES = {"go": "Go", "python": "Python", "typescript": "TypeScript"} Target = tuple[str, str, str] @@ -34,11 +34,19 @@ def exact_version(targets: list[Target], language: str) -> str: return versions.pop() if versions else "" +def join_labels(labels: list[str]) -> str: + if len(labels) < 3: + return " and ".join(labels) + return ", ".join(labels[:-1]) + " and " + labels[-1] + + def describe(targets: list[Target]) -> str: - return " and ".join( - label - for language, label in LANGUAGES.items() - if of_language(targets, language) + return join_labels( + [ + label + for language, label in LANGUAGES.items() + if of_language(targets, language) + ] ) @@ -79,7 +87,7 @@ def select_targets( ) if not keys or len(set(keys)) != len(keys): raise ValueError("select at least one binding target without duplicates") - enabled = " and ".join(LANGUAGES[language] for language in languages) + enabled = join_labels([LANGUAGES[language] for language in languages]) for key in keys: if key not in targets: raise ValueError( diff --git a/tooling/extension-bindings/ci/publish-npm.py b/tooling/extension-bindings/ci/publish-npm.py new file mode 100644 index 0000000..be1dea0 --- /dev/null +++ b/tooling/extension-bindings/ci/publish-npm.py @@ -0,0 +1,143 @@ +"""Build the committed TypeScript bindings and publish them to npm with trusted publishing.""" + +from __future__ import annotations + +import argparse +import json +import os +import shutil +import subprocess +import tempfile +from pathlib import Path + +import yaml + +from generation import Target, of_language, select_targets, summarize + +LANGUAGE = "typescript" +TOOLING = Path("tooling/extension-bindings") +EMITTER = TOOLING / "emitters/typescript" +MINIMUM_NPM = (11, 5, 1) + + +def read_package(root: Path, catalog: dict, target: Target) -> dict: + """Read a committed package.json and check it against the catalog entry.""" + key, directory, _ = target + package = json.loads((root / directory / "package.json").read_text(encoding="utf-8")) + expected = catalog["packages"][key.split(":")[0]]["languages"][LANGUAGE] + if package["name"] != expected["name"] or package["version"] != str(expected["version"]): + raise ValueError( + f"{key}: {directory}/package.json is " + f"{package['name']}@{package['version']}, but the catalog says " + f"{expected['name']}@{expected['version']}" + ) + return package + + +def publication_order(packages: dict[str, dict]) -> list[str]: + """Order package names so every selected dependency comes before its dependents.""" + ordered: list[str] = [] + state: dict[str, int] = {} + + def visit(name: str) -> None: + if state.get(name) == 2: + return + if state.get(name) == 1: + raise ValueError(f"package dependency cycle at {name}") + state[name] = 1 + for dependency in sorted(packages[name].get("dependencies", {})): + if dependency in packages: + visit(dependency) + state[name] = 2 + ordered.append(name) + + for name in sorted(packages): + visit(name) + return ordered + + +def check_npm_version(version: str) -> None: + numbers = tuple(int(part) for part in version.strip().split("-")[0].split(".")[:3]) + if numbers < MINIMUM_NPM: + raise ValueError( + "trusted publishing needs npm " + + ".".join(map(str, MINIMUM_NPM)) + + f" or later, but npm {version.strip()} is installed" + ) + + +def run(command: list[str], cwd: Path, **options) -> subprocess.CompletedProcess: + return subprocess.run(command, cwd=cwd, check=True, text=True, **options) + + +def already_published(name: str, version: str) -> bool: + result = subprocess.run( + ["npm", "view", f"{name}@{version}", "version"], + capture_output=True, + text=True, + ) + return result.returncode == 0 and result.stdout.strip() == version + + +def publish(root: Path, targets: list[Target], execute: bool) -> None: + selected = of_language(targets, LANGUAGE) + if not selected: + summarize("No TypeScript targets selected; nothing to publish.") + return + tsc = root / EMITTER / "node_modules/typescript/bin/tsc" + if not tsc.is_file(): + raise ValueError(f"install the emitter dependencies first (npm ci in {root / EMITTER}): {tsc} is missing") + if execute: + check_npm_version(run(["npm", "--version"], root, capture_output=True).stdout) + catalog = yaml.safe_load((root / TOOLING / "packages.yaml").read_text(encoding="utf-8")) + packages = {} + directories = {} + for target in selected: + package = read_package(root, catalog, target) + packages[package["name"]] = package + directories[package["name"]] = root / target[1] + published = [] + with tempfile.TemporaryDirectory(prefix="npm-bindings-") as name: + # Each package is built where its dependencies resolve, from the same + # committed tree a consumer would receive. + workspace = Path(name) + for package_name in publication_order(packages): + destination = workspace / "node_modules" / package_name + shutil.copytree(directories[package_name], destination) + run(["node", str(tsc), "-p", str(destination / "tsconfig.json")], root) + run(["npm", "pack", "--dry-run", "--ignore-scripts"], destination, capture_output=True) + version = packages[package_name]["version"] + if not execute: + summarize(f"Built and checked {package_name}@{version}; dry run, nothing was published.") + elif already_published(package_name, version): + summarize(f"{package_name}@{version} is already on npm; skipped.") + else: + run(["npm", "publish", "--access", "public", "--ignore-scripts"], destination) + published.append(f"{package_name}@{version}") + summarize(f"Published {package_name}@{version}.") + if execute: + summarize("Published to npm: " + (", ".join(published) if published else "nothing new.")) + + +def main() -> None: + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--root", type=Path, default=Path.cwd()) + parser.add_argument("--targets", default="") + parser.add_argument( + "--execute", + action="store_true", + help="publish; without it the packages are only built and checked", + ) + args = parser.parse_args() + root = args.root.resolve() + try: + targets = select_targets(root / TOOLING / "packages.yaml", args.targets) + publish(root, targets, args.execute) + except (ValueError, KeyError, OSError, subprocess.CalledProcessError) as error: + if isinstance(error, subprocess.CalledProcessError) and error.stderr: + print(error.stderr) + parser.exit(1, f"npm publication failed: {error}\n") + + +if __name__ == "__main__": + main() diff --git a/tooling/extension-bindings/ci/test_generate_typescript.py b/tooling/extension-bindings/ci/test_generate_typescript.py new file mode 100644 index 0000000..b6d63b5 --- /dev/null +++ b/tooling/extension-bindings/ci/test_generate_typescript.py @@ -0,0 +1,202 @@ +"""Exercise the TypeScript generation script.""" + +from __future__ import annotations + +import hashlib +from pathlib import Path +from unittest.mock import patch + +from generation import select_targets +from testsupport import RepoTestCase, load_script, run_script + +script = load_script("generate-typescript.py") + +ROOT_EXTENSION = "https://example.test/owner/v1/extension.yaml" +ADDON_EXTENSION = "https://example.test/addon/v1/extension.yaml" + + +def typescript(name: str, directory: str, version: str = "0.1.0") -> dict: + return { + "coordinate": name, + "name": name, + "version": version, + "sourceDirectory": directory, + "languageVersion": "20.19.6", + "publicationMode": "registry", + "registryId": "npm", + } + + +class TypeScriptGenerationTests(RepoTestCase): + def setUp(self) -> None: + super().setUp() + self.packages["owner"]["languages"]["typescript"] = typescript( + "@example/owner", "bindings/owner/typescript" + ) + self.packages["owner"]["rootExtension"] = ROOT_EXTENSION + self.packages["addon"]["languages"]["typescript"] = typescript( + "@example/addon", "bindings/addon/typescript" + ) + self.packages["addon"]["rootExtension"] = ADDON_EXTENSION + self.write_catalog() + self.catalog_document = { + "repositoryUrl": "https://github.com/example/extensions", + "packages": self.packages, + } + + def test_configure_reports_the_node_version_and_whether_typescript_is_selected(self) -> None: + output = self.root.parent / "github-output" + for targets, version, has_typescript in [ + ("owner:typescript", "20.19.6", "true"), + ("owner:go", "", "false"), + ("owner:go,owner:typescript", "20.19.6", "true"), + ]: + with self.subTest(targets=targets): + values = run_script( + "generate-typescript.py", + self.root, + output, + "configure", + "--targets", + targets, + ) + self.assertEqual( + values, + {"typescript-version": version, "has-typescript": has_typescript}, + ) + + def test_the_emitter_digest_follows_its_sources_and_not_its_build(self) -> None: + emitter = self.root / "emitter" + for name, data in { + "package.json": "{}", + "package-lock.json": "{}", + "package-target.schema.yaml": "type: object\n", + "src/plan.ts": "export {};\n", + "templates/package.json.tmpl": "{}", + }.items(): + path = emitter / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(data, encoding="utf-8") + first = script.emitter_digest(emitter) + (emitter / "dist").mkdir() + (emitter / "dist/plan.js").write_text("built\n", encoding="utf-8") + self.assertEqual(script.emitter_digest(emitter), first) + (emitter / "src/plan.ts").write_text("export const x = 1;\n", encoding="utf-8") + self.assertNotEqual(script.emitter_digest(emitter), first) + + def test_the_file_manifest_lists_every_file_but_itself_in_the_go_format(self) -> None: + package = self.root / "package" + (package / "src").mkdir(parents=True) + (package / "src/b.ts").write_text("b\n", encoding="utf-8") + (package / "a.json").write_text("a\n", encoding="utf-8") + (package / script.FILE_MANIFEST).write_text("stale\n", encoding="utf-8") + digest = lambda text: hashlib.sha256(text.encode()).hexdigest() + self.assertEqual( + script.file_manifest(package), + "apiVersion: runtimeconditions.io/file-manifest/v1alpha1\n" + "kind: RuntimeConditionsFileManifest\n" + "files:\n" + f" a.json: {digest('a' + chr(10))}\n" + f" src/b.ts: {digest('b' + chr(10))}\n", + ) + + def test_the_package_target_comes_from_the_catalog_and_the_model(self) -> None: + model = {"dependencyEdges": [{"from": ROOT_EXTENSION, "to": ADDON_EXTENSION}]} + target = script.package_target(self.catalog_document, "owner", model, "f" * 64) + self.assertEqual(target["packageName"], "@example/owner") + self.assertEqual(target["sourceDirectory"], "bindings/owner/typescript") + self.assertEqual(target["minimumNodeVersion"], "20.19") + self.assertEqual(target["registryId"], "npm") + self.assertEqual(target["repositoryUrl"], "https://github.com/example/extensions") + self.assertEqual(target["emitterSha256"], "f" * 64) + self.assertEqual( + target["dependencies"], + [{"extension": ADDON_EXTENSION, "packageName": "@example/addon", "version": "0.1.0"}], + ) + + def test_a_package_without_dependency_edges_has_no_dependencies(self) -> None: + target = script.package_target(self.catalog_document, "addon", {}, "f" * 64) + self.assertEqual(target["dependencies"], []) + + def test_a_dependency_needs_exactly_one_typescript_provider(self) -> None: + del self.packages["addon"]["languages"]["typescript"] + model = {"dependencyEdges": [{"from": ROOT_EXTENSION, "to": ADDON_EXTENSION}]} + with self.assertRaisesRegex(ValueError, "found 0"): + script.package_target(self.catalog_document, "owner", model, "f" * 64) + + def test_the_root_extension_is_read_from_the_catalog_and_checked_against_the_lock(self) -> None: + source = b"extension: definition\n" + self.write("catalog/owner/owner-v1.yaml", source.decode()) + lock = { + "extensions": [ + { + "id": ROOT_EXTENSION, + "sourceBackend": "catalog", + "sourceLocator": "catalog:owner/owner-v1.yaml", + "sourceSha256": hashlib.sha256(source).hexdigest(), + } + ] + } + self.assertEqual(script.root_extension_source(self.root, lock, ROOT_EXTENSION), source) + lock["extensions"][0]["sourceSha256"] = "0" * 64 + with self.assertRaisesRegex(ValueError, "differs from the dependency lock"): + script.root_extension_source(self.root, lock, ROOT_EXTENSION) + lock["extensions"][0]["sourceLocator"] = "https:example.test/owner.yaml" + with self.assertRaisesRegex(ValueError, "only catalog extension sources"): + script.root_extension_source(self.root, lock, ROOT_EXTENSION) + with self.assertRaisesRegex(ValueError, "exactly once"): + script.root_extension_source(self.root, lock, ADDON_EXTENSION) + + def test_generation_needs_the_built_emitter(self) -> None: + with patch.dict("os.environ", {}, clear=True), self.assertRaisesRegex( + ValueError, "build the TypeScript emitter first" + ): + script.tools(self.root) + built = self.root / "main.js" + built.write_text("", encoding="utf-8") + with patch.dict("os.environ", {"RC_TYPESCRIPT_EMITTER": str(built)}, clear=True): + self.assertEqual(script.tools(self.root), ("node", built)) + + def generated(self, files: dict[str, str]): + def generate(root, catalog, key, rc, core, work): + package = work / key + for name, data in files.items(): + path = package / name + path.parent.mkdir(parents=True, exist_ok=True) + path.write_text(data, encoding="utf-8") + return package + + return generate + + def test_update_replaces_the_committed_tree_and_check_compares_it(self) -> None: + self.write("tooling/extension-bindings/packages.yaml", self.catalog.read_text(encoding="utf-8")) + self.write("bindings/owner/typescript/stale.ts", "old generated source\n") + targets = select_targets(self.catalog, "owner:typescript") + files = {"package.json": "{}\n", "src/bindings.ts": "new\n"} + with patch.object(script, "generate", self.generated(files)): + with self.assertRaisesRegex(ValueError, "stale.ts"): + script.run(self.root, targets, Path("rc"), Path("core.yaml"), False) + script.run(self.root, targets, Path("rc"), Path("core.yaml"), True) + script.run(self.root, targets, Path("rc"), Path("core.yaml"), False) + committed = self.root / "bindings/owner/typescript" + self.assertFalse((committed / "stale.ts").exists()) + self.assertEqual((committed / "src/bindings.ts").read_text(encoding="utf-8"), "new\n") + + def test_update_leaves_other_languages_alone(self) -> None: + targets = select_targets(self.catalog, "owner:go") + with patch.object(script, "generate") as generate: + script.run(self.root, targets, Path("rc"), Path("core.yaml"), True) + generate.assert_not_called() + self.assertTrue((self.root / "bindings/owner/go/stale.go").exists()) + + def test_typescript_targets_carry_their_directory_and_exact_node_version(self) -> None: + self.assertIn( + ("owner:typescript", "bindings/owner/typescript", "20.19.6"), + select_targets(self.catalog, ""), + ) + + +if __name__ == "__main__": + import unittest + + unittest.main() diff --git a/tooling/extension-bindings/ci/test_publish_npm.py b/tooling/extension-bindings/ci/test_publish_npm.py new file mode 100644 index 0000000..40b061e --- /dev/null +++ b/tooling/extension-bindings/ci/test_publish_npm.py @@ -0,0 +1,123 @@ +"""Exercise the npm publication script.""" + +from __future__ import annotations + +import json +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import patch + +from generation import select_targets +from testsupport import RepoTestCase, load_script + +script = load_script("publish-npm.py") + + +class NpmPublicationTests(RepoTestCase): + def setUp(self) -> None: + super().setUp() + for key, name in (("owner", "@example/owner"), ("addon", "@example/addon")): + self.packages[key]["languages"]["typescript"] = { + "coordinate": name, + "name": name, + "version": "0.1.0", + "sourceDirectory": f"bindings/{key}/typescript", + "languageVersion": "20.19.6", + "publicationMode": "registry", + "registryId": "npm", + } + self.write_catalog() + self.package("addon", "@example/addon") + self.package("owner", "@example/owner", {"@example/addon": "^0.1.0"}) + self.write("tooling/extension-bindings/emitters/typescript/node_modules/typescript/bin/tsc", "") + self.targets = select_targets(self.catalog, "owner:typescript,addon:typescript") + + def package(self, key: str, name: str, dependencies: dict | None = None, version: str = "0.1.0") -> None: + self.write( + f"bindings/{key}/typescript/package.json", + json.dumps({"name": name, "version": version, "dependencies": dependencies or {}}), + ) + self.write(f"bindings/{key}/typescript/tsconfig.json", "{}") + + def commands(self, calls) -> list[list[str]]: + return [call.args[0] for call in calls] + + def test_dependencies_are_published_before_their_dependents(self) -> None: + packages = { + "@example/owner": {"dependencies": {"@example/addon": "^0.1.0", "left-pad": "^1.0.0"}}, + "@example/addon": {}, + "@example/leaf": {"dependencies": {"@example/owner": "^0.1.0"}}, + } + self.assertEqual( + script.publication_order(packages), + ["@example/addon", "@example/owner", "@example/leaf"], + ) + packages["@example/addon"] = {"dependencies": {"@example/leaf": "^0.1.0"}} + with self.assertRaisesRegex(ValueError, "cycle"): + script.publication_order(packages) + + def test_trusted_publishing_needs_a_recent_npm(self) -> None: + for version in ("11.5.1\n", "11.6.0", "12.0.0-pre.1"): + script.check_npm_version(version) + for version in ("10.8.2", "11.5.0"): + with self.assertRaisesRegex(ValueError, "11.5.1 or later"): + script.check_npm_version(version) + + def test_a_package_must_match_its_catalog_entry(self) -> None: + self.package("owner", "@example/other") + catalog = {"packages": self.packages} + with self.assertRaisesRegex(ValueError, "catalog says"): + script.read_package(self.root, catalog, self.targets[1]) + + def test_a_dry_run_builds_and_checks_but_never_publishes(self) -> None: + with patch.object(script, "run") as run, patch.object( + script, "already_published" + ) as published: + script.publish(self.root, self.targets, False) + commands = self.commands(run.call_args_list) + self.assertFalse(any(command[:2] == ["npm", "publish"] for command in commands)) + self.assertEqual(sum(command[:2] == ["npm", "pack"] for command in commands), 2) + builds = [command[3] for command in commands if command[0] == "node"] + self.assertTrue(builds[0].endswith("node_modules/@example/addon/tsconfig.json")) + self.assertTrue(builds[1].endswith("node_modules/@example/owner/tsconfig.json")) + published.assert_not_called() + + def test_publication_skips_versions_that_are_already_on_npm(self) -> None: + def run(command, cwd, **options): + return SimpleNamespace(stdout="11.6.0\n") + + with patch.object(script, "run", side_effect=run) as runner, patch.object( + script, "already_published", side_effect=lambda name, version: name == "@example/addon" + ): + script.publish(self.root, self.targets, True) + publishes = [ + call for call in runner.call_args_list if call.args[0][:2] == ["npm", "publish"] + ] + self.assertEqual(len(publishes), 1) + self.assertEqual( + publishes[0].args[0], ["npm", "publish", "--access", "public", "--ignore-scripts"] + ) + self.assertTrue(str(publishes[0].args[1]).endswith("node_modules/@example/owner")) + + def test_publication_refuses_an_old_npm(self) -> None: + with patch.object( + script, "run", return_value=SimpleNamespace(stdout="10.8.2\n") + ), self.assertRaisesRegex(ValueError, "npm 11.5.1"): + script.publish(self.root, self.targets, True) + + def test_publication_needs_the_emitter_dependencies(self) -> None: + (self.root / "tooling/extension-bindings/emitters/typescript/node_modules/typescript/bin/tsc").unlink() + with self.assertRaisesRegex(ValueError, "npm ci"): + script.publish(self.root, self.targets, False) + + def test_other_languages_are_not_published(self) -> None: + targets = select_targets(self.catalog, "owner:go") + with patch.object(script, "run") as run: + script.publish(self.root, targets, True) + run.assert_not_called() + + +if __name__ == "__main__": + import unittest + + unittest.main() diff --git a/tooling/extension-bindings/orchestrator/plan.go b/tooling/extension-bindings/orchestrator/plan.go index 93b439f..9364686 100644 --- a/tooling/extension-bindings/orchestrator/plan.go +++ b/tooling/extension-bindings/orchestrator/plan.go @@ -20,12 +20,18 @@ type BuildPlan struct { Targets []Target `yaml:"targets" json:"targets"` } +// nativeLanguages are the languages rc itself emits, verifies, and packages. Other catalog +// languages are generated by their own scripts, so only an explicit target selects them. +var nativeLanguages = map[string]bool{"go": true, "python": true} + func (p *Project) SelectTargets(keys []string) (BuildPlan, error) { selected := map[string]bool{} if len(keys) == 0 { for key, set := range p.Catalog.Packages { for lang := range set.Languages { - selected[key+":"+lang] = true + if nativeLanguages[lang] { + selected[key+":"+lang] = true + } } } } else { diff --git a/tooling/extension-bindings/orchestrator/plan_test.go b/tooling/extension-bindings/orchestrator/plan_test.go index 1d70b1e..ebde6cc 100644 --- a/tooling/extension-bindings/orchestrator/plan_test.go +++ b/tooling/extension-bindings/orchestrator/plan_test.go @@ -204,3 +204,25 @@ func TestCatalogRejectsUnsafeModuleCoordinate(t *testing.T) { t.Fatal("unsafe module coordinate accepted") } } + +func TestDefaultSelectionLeavesLanguagesWithoutANativeAdapterToTheirOwnScripts(t *testing.T) { + p, _ := testProject(t, "go") + set := p.Catalog.Packages["new-channel"] + set.Languages["typescript"] = LanguageTarget{Coordinate: "@example/new-channel", Name: "@example/new-channel", Version: "0.7.0", SourceDirectory: "bindings/new-channel/typescript", LanguageVersion: "20.19.6", PublicationMode: "github-tag"} + plan, err := p.SelectTargets(nil) + if err != nil { + t.Fatal(err) + } + for _, target := range plan.Targets { + if target.Language == "typescript" { + t.Fatalf("default selection included %s", target.Key) + } + } + if len(plan.Targets) != 1 { + t.Fatalf("default selection %v", plan.Targets) + } + plan, err = p.SelectTargets([]string{"new-channel:typescript"}) + if err != nil || len(plan.Targets) != 1 || plan.Targets[0].Language != "typescript" { + t.Fatalf("explicit selection %v %v", plan.Targets, err) + } +} diff --git a/tooling/extension-bindings/packages.yaml b/tooling/extension-bindings/packages.yaml index 15e0348..7943e4e 100644 --- a/tooling/extension-bindings/packages.yaml +++ b/tooling/extension-bindings/packages.yaml @@ -41,6 +41,14 @@ packages: languageVersion: 3.12.10 publicationMode: registry registryId: pypi + typescript: + coordinate: '@runtimeconditions/common-integrations' + name: '@runtimeconditions/common-integrations' + version: 0.1.0 + sourceDirectory: bindings/common-integrations/typescript + languageVersion: 20.19.6 + publicationMode: registry + registryId: npm env-configuration: rootExtension: https://runtimeconditions.io/extensions/env-configuration/v1alpha1/runtimeconditions.extension.yaml coreProfileSchemaVersion: 0.2.0 @@ -60,6 +68,14 @@ packages: languageVersion: 3.12.10 publicationMode: registry registryId: pypi + typescript: + coordinate: '@runtimeconditions/env-configuration' + name: '@runtimeconditions/env-configuration' + version: 0.1.0 + sourceDirectory: bindings/env-configuration/typescript + languageVersion: 20.19.6 + publicationMode: registry + registryId: npm google-analytics: rootExtension: https://runtimeconditions.io/extensions/google/analytics/0.1.0/runtimeconditions.extension.yaml coreProfileSchemaVersion: 0.2.0