Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 25 additions & 3 deletions .github/workflows/bindings-test.yml
Original file line number Diff line number Diff line change
Expand Up @@ -10,21 +10,43 @@ permissions:
contents: read

jobs:
# The api fixture's checks, each beside the script generated from its transcript, which the
# tree does not hold: the builder unpacks them over its checkout as api-checks/.
transcripts:
name: api fixture's checks
runs-on: ubuntu-24.04
timeout-minutes: 30
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
with:
persist-credentials: false
- uses: taiki-e/install-action@3a3daafd3b7f23b8f65d9f5ba2db203fb59b79e2 # v2.90.2
with:
tool: just
- run: just ffi transcripts generated/api-checks
- uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
with:
name: api-checks
path: generated
if-no-files-found: error

api:
name: api fixture
needs: transcripts
uses: ./.github/workflows/bindings.yml
with:
manifest-path: ffi/fixtures/api/Cargo.toml
features: doubled
example: transcript
python-check: ffi/tests/api.py
checks-artifact: api-checks
python-check: api-checks/python
# The small crate, so the second build costs little: moq's stand-in leaves it to its release.
reproduce: true
# api.cpp rather than api.c, whose broken cases name items only wasm32 has.
c-check: ffi/tests/api.cpp
c-check: api-checks/cpp
web-manifest-path: ffi/fixtures/api/Cargo.toml
scope: kixel-ai
web-check: ffi/tests/api.ts
web-check: api-checks/web
# A module path a proxy could serve. No go-check, as no swift-check: api.go needs the
# wasm32 items this export lacks, so the consumer only loads the runtime.
go-module: example.com/api
Expand Down
51 changes: 37 additions & 14 deletions .github/workflows/bindings.yml
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ on:
default: ''
type: string
python-check:
description: A program run on each installed wheel from a clean directory
description: A program run on each installed wheel from a clean directory, or a directory holding it as check.py beside the modules it imports
default: ''
type: string
swift-check:
Expand All @@ -33,7 +33,7 @@ on:
default: false
type: boolean
c-check:
description: A C or C++ program built by clean CMake and pkg-config consumers of each C archive
description: A C or C++ program built by clean CMake and pkg-config consumers of each C archive, or a directory holding it as check.c or check.cpp beside the headers it includes
default: ''
type: string
web-manifest-path:
Expand All @@ -45,7 +45,7 @@ on:
default: ''
type: string
web-check:
description: A TypeScript module run on the installed npm package, importing the binding as ./web/<module>.ts does beside the export
description: A TypeScript module run on the installed npm package, importing the binding as ./web/<module>.ts does beside the export, or a directory holding it as check.ts beside the modules it imports
default: ''
type: string
web-features:
Expand All @@ -64,6 +64,10 @@ on:
description: The main.go of a program, importing the module's packages, run against it
default: ''
type: string
checks-artifact:
description: An artifact of the caller's run unpacked over its checkout before the checks are read, for checks it generates
default: ''
type: string
outputs:
version:
description: The package's Cargo version
Expand Down Expand Up @@ -363,6 +367,13 @@ jobs:
python3 -I rust2/ffi/release/package.py sdist --manifest-path "source/$MANIFEST" --export export --out again/dist
fi
python3 -I rust2/ffi/release/package.py same dist again/dist
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
if: inputs.checks-artifact != ''
with:
name: ${{ inputs.checks-artifact }}
path: source
# A Python or C check lands as a directory holding it as check.<ext>, whichever form the
# input takes: a file is copied into one, and a directory whole.
- name: Checks
env:
PYTHON_CHECK: ${{ inputs.python-check }}
Expand All @@ -371,10 +382,13 @@ jobs:
GO_CHECK: ${{ inputs.go-check }}
run: |
mkdir -p dist/checks
if [[ -n "$PYTHON_CHECK" ]]; then cp "source/$PYTHON_CHECK" dist/checks/check.py; fi
stage() {
if [[ -d "source/$1" ]]; then cp -R "source/$1" "$2"; else mkdir "$2" && cp "source/$1" "$2/$3"; fi
}
if [[ -n "$PYTHON_CHECK" ]]; then stage "$PYTHON_CHECK" dist/checks/python check.py; fi
if [[ -n "$SWIFT_CHECK" ]]; then cp "source/$SWIFT_CHECK" dist/checks/main.swift; fi
# Its extension says whether it is C or C++.
if [[ -n "$C_CHECK" ]]; then mkdir dist/checks/c && cp "source/$C_CHECK" dist/checks/c/; fi
# A file's extension says whether it is C or C++.
if [[ -n "$C_CHECK" ]]; then stage "$C_CHECK" dist/checks/c "check.${C_CHECK##*.}"; fi
if [[ -n "$GO_CHECK" ]]; then cp "source/$GO_CHECK" dist/checks/main.go; fi
- uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2
with:
Expand Down Expand Up @@ -540,8 +554,8 @@ jobs:
"$py" -m pip install --no-index --no-deps dist/wheels/*.whl
args=()
if [[ -f dist/native.txt ]]; then args+=(--native dist/native.txt); fi
if [[ -f dist/checks/check.py ]]; then
"$py" -I rust2/ffi/release/package.py consume-python --check dist/checks/check.py "${args[@]}"
if [[ -d dist/checks/python ]]; then
"$py" -I rust2/ffi/release/package.py consume-python --check dist/checks/python "${args[@]}"
else
"$py" -I rust2/ffi/release/package.py consume-python --import "$(cat dist/lib.txt)"
fi
Expand Down Expand Up @@ -605,8 +619,8 @@ jobs:
tag=$(basename "${wheels[0]}" .whl | cut -d- -f3)
[[ "$tag" =~ ^cp3([0-9]+)$ ]] || { echo "::error::no CPython floor in ${wheels[0]}"; exit 1; }
echo "version=3.${BASH_REMATCH[1]}" >> "$GITHUB_OUTPUT"
args=(--check dist/checks/check.py)
if [[ ! -f dist/checks/check.py ]]; then args=(--import "$(cat dist/lib.txt)"); fi
args=(--check dist/checks/python)
if [[ ! -d dist/checks/python ]]; then args=(--import "$(cat dist/lib.txt)"); fi
if [[ -f native/native.txt ]]; then args+=(--native native/native.txt); fi
echo "args=${args[*]}" >> "$GITHUB_OUTPUT"
# Alpine's own Python and rustup's Rust, whose host is the musl target the export names.
Expand Down Expand Up @@ -722,8 +736,7 @@ jobs:
[[ ${#archives[@]} == 1 ]] || { echo "::error::expected one C archive, found ${archives[*]}"; exit 1; }
args=(--archive "${archives[0]}")
if [[ -f dist/native.txt ]]; then args+=(--native dist/native.txt); fi
checks=(dist/checks/c/*)
if [[ ${#checks[@]} == 1 ]]; then args+=(--check "${checks[0]}"); fi
if [[ -d dist/checks/c ]]; then args+=(--check dist/checks/c); fi
python -I rust2/ffi/release/package.py consume-c "${args[@]}"

# The web binding's npm and JSR packages: web-manifest-path exported for wasm32-wasip1
Expand Down Expand Up @@ -753,6 +766,11 @@ jobs:
if: needs.facts.outputs.web-fixture == 'true'
working-directory: source
run: just ffi standin "" web
- uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
if: inputs.checks-artifact != ''
with:
name: ${{ inputs.checks-artifact }}
path: source
# npm packs, and Node's WASI runs the native example.
- uses: actions/setup-node@949feb2413d6458794dcd2491c4babbbce0c15c1 # v7.1.0
with:
Expand Down Expand Up @@ -782,7 +800,12 @@ jobs:
features=$WEB_FEATURES example=$WEB_EXAMPLE
fi
python3 -I rust2/ffi/release/package.py web --manifest-path "source/$WEB_MANIFEST" --features "$features" --scope "$SCOPE" --version "$VERSION" --example "$example" --out web
if [[ -n "$WEB_CHECK" ]]; then cp "source/$WEB_CHECK" web/check.ts; fi
# A directory whole, or a file copied into one as check.ts.
if [[ -n "$WEB_CHECK" && -d "source/$WEB_CHECK" ]]; then
cp -R "source/$WEB_CHECK" web/check
elif [[ -n "$WEB_CHECK" ]]; then
mkdir web/check && cp "source/$WEB_CHECK" web/check/check.ts
fi
# The build again, from a new export, which must pack the same tarball.
if [[ "$REPRODUCE" == true ]]; then
python3 -I rust2/ffi/release/package.py web --manifest-path "source/$WEB_MANIFEST" --features "$features" --scope "$SCOPE" --version "$VERSION" --example "" --out web-again
Expand Down Expand Up @@ -822,7 +845,7 @@ jobs:
- name: Install and run
run: |
args=()
if [[ -f web/check.ts ]]; then args+=(--check web/check.ts); fi
if [[ -d web/check ]]; then args+=(--check web/check); fi
if [[ -f web/native.txt ]]; then args+=(--native web/native.txt); fi
python3 -I rust2/ffi/release/package.py consume-web --dist web "${args[@]}"

Expand Down
3 changes: 3 additions & 0 deletions ffi/justfile
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,9 @@ corpus-update:
protocol:
cargo test --locked --test fixtures protocol_nightly -- --ignored

# Write the api fixture's Python, C++, and web checks, each beside the script tests/fixtures/transcript generates, into <dir>/python, <dir>/cpp, and <dir>/web, the directories the release builder's check inputs take.
transcripts dir:
out="$(cd {{ quote(invocation_directory()) }} && mkdir -p {{ quote(dir) }} && cd {{ quote(dir) }} && pwd)" && RUST2_TRANSCRIPTS="$out" cargo test --locked --test fixtures -- --ignored --exact transcripts
# Run the api fixture's web test in headless Chromium, Firefox, and on Linux WebKit, in the bench shell, which carries them: nix develop .#bench.
browsers:
cargo test --locked --test fixtures -- --ignored --exact api_browsers
Expand Down
8 changes: 7 additions & 1 deletion ffi/release/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -64,6 +64,11 @@ TypeScript module importing the binding as `./web/<module>.ts` does beside an ex
whose imports the consumer rewrites to the package's name. Its crate's version must be
the release's. `features` and `example` apply to it when it is the crate's own; a web
crate apart from it takes `web-features` and `web-example` instead.
A `python-check`, `c-check`, or `web-check` may instead name a directory holding the program
as `check.py`, `check.c` or `check.cpp`, or `check.ts`, beside the files it reads, which each
consumer places beside it: a check that imports a module or includes a header of its own.
A check its tests generate comes through `checks-artifact`, an artifact of the caller's run
the builder unpacks over its checkout before reading the checks.

## What the builder does

Expand Down Expand Up @@ -193,7 +198,8 @@ crate apart from it takes `web-features` and `web-example` instead.

rust2 runs it on every manual Check and nightly through
[`bindings-test.yml`](/.github/workflows/bindings-test.yml): on the api fixture, whose
wheels, C++ consumers, and web package must print its native transcript, and on moq's
wheels, C++ consumers, and web package must print its native transcript through the checks
`just ffi transcripts` writes, each test beside the script generated from it, and on moq's
stand-in facade at the exporter's moq pin, through `moq/moq.py`, `moq/moq.swift`,
`moq/moq.cpp`, and `moq/moq.go`, with its wasm32 stand-in as the web package.

Expand Down
74 changes: 55 additions & 19 deletions ffi/release/package.py
Original file line number Diff line number Diff line change
Expand Up @@ -328,9 +328,11 @@ def workspace_manifest(path):
def relocate(manifest, moves, strict=False):
"""A manifest with each local package's absolute path moved, as `moves` maps them, checked
by reading it back: nothing else changed, and when `strict`, every `path` is one moved."""
# One pass, so a path already moved is never matched again by another package's key.
spelled = {f"path = {json.dumps(old, ensure_ascii=False)}": f"path = {json.dumps(new)}" for old, new in moves.items()}
text = manifest
for old, new in moves.items():
text = text.replace(f"path = {json.dumps(old, ensure_ascii=False)}", f"path = {json.dumps(new)}")
if spelled:
text = re.sub("|".join(map(re.escape, spelled)), lambda m: spelled[m.group(0)], manifest)

def moved(value):
if isinstance(value, dict):
Expand Down Expand Up @@ -423,7 +425,17 @@ def moves(at):
for root in roots:
at = place(root.parent)
files[f"{at}/Cargo.toml"] = (relocate(workspace_manifest(root), moves(at)).encode(), False)
generated = relocate((export / "Cargo.toml").read_text(encoding="utf-8"), moves("binding"), strict=True)
# The generated manifest names each package relative to the export, where a relative path
# reads right, and absolute elsewhere.
binding = moves("binding")
for d in placed:
try:
binding[Path(os.path.relpath(d, export)).as_posix()] = binding[str(d)]
except ValueError:
# On another Windows drive than the export, no relative path reaches it; only the
# absolute one names it.
pass
generated = relocate((export / "Cargo.toml").read_text(encoding="utf-8"), binding, strict=True)
files["binding/Cargo.toml"] = (generated.encode(), False)
for path in [export / "Cargo.lock", export / "build.rs", export / "python" / f"{info['lib']}.py",
*sorted(p for p in (export / "src").rglob("*") if p.is_file())]:
Expand Down Expand Up @@ -1089,12 +1101,20 @@ def owned(member):
return path


def check_program_in(check, names):
"""The program of a check directory: the one file of `names` it holds, beside the files it reads."""
found = [Path(check) / name for name in names if (Path(check) / name).is_file()]
if len(found) != 1:
raise SystemExit(f"{check} must hold exactly one of {', '.join(names)}, beside the files it reads")
return found[0]


def c_check(lib, check, work):
"""The consumer's program: the caller's check, or else one that links a function of each header."""
"""The consumer's program: the caller's check, its directory copied whole so that its
quoted includes resolve beside it, or else one that links a function of each header."""
if check:
source = work / Path(check).name
shutil.copy(check, source)
return source
shutil.copytree(check, work / "check")
return check_program_in(work / "check", ("check.c", "check.cpp", "check.cc", "check.cxx"))
source = work / "smoke.cpp"
source.write_text(f'#include "{lib}.h"\n#include "{lib}.hpp"\n\n'
f"int main() {{\n\tvoid (*volatile fn)({lib}_buf) = {lib}_buf_free;\n"
Expand Down Expand Up @@ -1212,10 +1232,16 @@ def compare(native, actual, what):


def consume_python(check, native, module):
"""Run a check program, or else an import, on the installed wheel from an empty directory."""
program = [str(Path(check).resolve())] if check else ["-c", f"import {module}"]
with tempfile.TemporaryDirectory() as clean:
result = subprocess.run([sys.executable, "-I", "-X", "dev", "-W", "error", *program],
"""Run a check program, or else an import, on the installed wheel from an empty directory.
The check's directory is copied apart, and it alone joins the path, where the check's own
modules are found: `-E -s` is `-I` but for that."""
with tempfile.TemporaryDirectory() as clean, tempfile.TemporaryDirectory() as scripts:
if check:
shutil.copytree(check, scripts, dirs_exist_ok=True)
program = [str(check_program_in(scripts, ("check.py",)))]
else:
program = ["-I", "-c", f"import {module}"]
result = subprocess.run([sys.executable, "-E", "-s", "-X", "dev", "-W", "error", *program],
capture_output=True, cwd=clean)
sys.stderr.buffer.write(result.stderr)
if result.returncode:
Expand Down Expand Up @@ -2018,7 +2044,7 @@ def web(args):


def check_program(check, name):
"""A check written beside an export, importing `./web/<module>.ts`, as it imports the package."""
"""A check's module written beside an export, importing `./web/<module>.ts`, as it imports the package."""
text = Path(check).read_text(encoding="utf-8")
# Whitespace and comments, which may separate the keyword from its specifier.
gap = r"(?:\s|/\*[\s\S]*?\*/|//[^\n]*\n)*"
Expand All @@ -2041,26 +2067,36 @@ def consume_web(dist, check, native):
if len(tarballs) != 1:
raise SystemExit(f"{dist} holds {len(tarballs)} npm tarballs")
name = json.loads((dist / "npm" / "package.json").read_text(encoding="utf-8"))["name"]
program = check_program(check, name) if check else (
f"import * as binding from {json.dumps(name)};\n\nawait binding.load();\n")
if check:
check_program_in(check, ("check.ts",))
expected = Path(native).read_bytes() if native and check else None
with tempfile.TemporaryDirectory() as clean:
clean = Path(clean)
# The check's directory whole, each TypeScript module in it importing the package;
# `check.ts` runs, reaching the rest by path. The project's own files then replace any
# it holds of the same names.
if check:
shutil.copytree(check, clean, dirs_exist_ok=True)
for module in clean.rglob("*.ts"):
module.write_text(check_program(module, name), encoding="utf-8")
else:
(clean / "check.ts").write_text(
f"import * as binding from {json.dumps(name)};\n\nawait binding.load();\n", encoding="utf-8")
manifest = {"type": "module", "private": True, "dependencies": {name: f"file:{tarballs[0]}"},
"devDependencies": BUNDLERS}
(clean / "package.json").write_text(json.dumps(manifest, indent=2) + "\n", encoding="utf-8")
run("npm", "install", "--no-audit", "--no-fund", "--ignore-scripts", cwd=clean, capture_output=True)
(clean / "check.ts").write_text(program, encoding="utf-8")
# The bundles target the browser, whose `document` Vite's preload polyfill reads;
# each runs on Node from where the bundler wrote it, a relative base reaching its module.
(clean / "index.html").write_text('<!doctype html><script type="module" src="./check.ts"></script>\n', encoding="utf-8")
(clean / "vite.config.js").write_text(
'export default { base: "./", build: { target: "es2024", modulePreload: { polyfill: false } } };\n', encoding="utf-8")
run("node", "--no-warnings", "--input-type=module", "-e",
'import { stripTypeScriptTypes } from "node:module"; import { readFileSync, writeFileSync } from "node:fs";'
' writeFileSync("check.js", stripTypeScriptTypes(readFileSync("check.ts", "utf8")));', cwd=clean)
# webpack reads the check's TypeScript through Node's own type stripping, as Node runs it.
(clean / "strip-types.cjs").write_text(
'module.exports = (source) => require("node:module").stripTypeScriptTypes(source);\n', encoding="utf-8")
(clean / "webpack.config.js").write_text(
'export default { mode: "production", entry: "./check.js", target: ["web", "es2024"],'
'export default { mode: "production", entry: "./check.ts", target: ["web", "es2024"],'
r' module: { rules: [{ test: /\.ts$/, exclude: /node_modules/, loader: new URL("./strip-types.cjs", import.meta.url).pathname }] },'
' experiments: { outputModule: true },'
' output: { module: true, path: new URL("./webpack", import.meta.url).pathname, filename: "check.js" } };\n',
encoding="utf-8")
Expand Down
Loading