From af6792f46362d64572a2c25c8d67c7db6e7a8bb8 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Thu, 29 Oct 2026 14:32:23 -0700 Subject: [PATCH 1/4] chore(quest): claim quest/m2/ts-std-build.md Quest-Claim: 87e6d34c667dfd46 From db398da9e3918a3322e82d87ccafaea9195c2330 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Thu, 29 Oct 2026 18:27:04 -0700 Subject: [PATCH 2/4] chore(quest): plan checking std once per toolchain as a Nix sysroot --- quest/m2/README.md | 2 +- quest/m2/ts-std-build.md | 31 ++++++++++++++++++------------- 2 files changed, 19 insertions(+), 14 deletions(-) diff --git a/quest/m2/README.md b/quest/m2/README.md index 11ba5abcd..b548d7dc9 100644 --- a/quest/m2/README.md +++ b/quest/m2/README.md @@ -30,6 +30,6 @@ The fork's feature quests folded into the [exporter](/quest/m1/ffi-export/README - [Async in the TypeScript translator](/quest/m2/ts-async.md) - moq-net's and kio's async code translates with Rust's lazy futures - [Shared ownership](/quest/m2/ts-shared.md) - `Arc`/`Rc`/`Weak` keep exact counts, and locks and atomics become plain values - [Translate moq-net's model to TypeScript](/quest/m2/ts-model/README.md) - the model translates with kio, moq-clock, and moq-pattern, for moq to compare against WASM -- [Translate against the prebuilt std](/quest/m2/ts-std-build.md) - measure whether `-Zbuild-std` is still needed now that panics abort, and drop it if not +- [Check std once per toolchain](/quest/m2/ts-std-build.md) - check std as a Nix derivation the translator takes as its sysroot, instead of in every cold target directory - [Waiting: Swift 6.4 in nixpkgs-unstable](/quest/m2/uniffi-swift-arm.md) - the nightly UniFFI fixtures move back to arm - [Waiting: ubrn reads UniFFI 0.32](/quest/m2/ubrn-release.md) - the binding benchmark's browser runner pins a ubrn release, not an open PR diff --git a/quest/m2/ts-std-build.md b/quest/m2/ts-std-build.md index c3ee432ef..a09c07fa5 100644 --- a/quest/m2/ts-std-build.md +++ b/quest/m2/ts-std-build.md @@ -1,21 +1,26 @@ -# [S] Translate against the prebuilt std +# [S] Check std once per toolchain ## Goal -Decide, by measurement, whether rust2-ts still needs `-Zbuild-std` now that it aborts on -panic ([#187](https://github.com/kixelated/rust2/pull/187)), and drop it if not, making the -translator's builds faster and simpler. +rust2-ts checks std from source in every cold target directory, through `-Zbuild-std`, so +its translations see std's un-inlined MIR: the prebuilt std's optimized MIR changes +snapshots and the census and refuses `LocalKey`'s methods (measured in +[#237](https://github.com/kixelated/rust2/pull/237)). Check std once per toolchain +instead, as a Nix derivation the translator takes as its sysroot, so CI's runners, which +keep `/nix/store`, check it only when the nightly changes. Snapshots and moq's census stay +byte-identical. ## Plan -Suggested by #187 on 2026-10-28. +Decided on #237 on 2026-10-29. -- Translate the fixtures and moq's census against the prebuilt std and diff the census and - every snapshot against today's. Equal output, with the differential tests passing, is - the bar. -- The risk: the prebuilt std is built with optimizations that may inline or rewrite calls - the translator expects to see by path (std's iterators, formatting, and the calls in - `library.rs`), so its MIR may differ from what `-Zbuild-std` gives. -- Measure the build time saved. +- A derivation checks std with `-Zbuild-std` from the toolchain's `rust-src`, offline + from its vendored crates, with the flags translation uses, and keeps the `.rmeta` files + as a sysroot. +- The dev shell and the `ts` package point the translator at it; the wrapper passes it to + rustc as `--sysroot` for the target's crates only. +- Without one, outside Nix, the translator falls back to `-Zbuild-std`, from one shared + list of flags; a nightly job runs the suite that way, so the two stay identical. +- Measure cold and warm CPU-s for a fixture and for the suite. -Public API: none, unless the toolchain the driver needs changes. +Public API: none; the `ts` package and dev shell set a new environment variable. From a1ac23811bdbee4f4a3f719cd56f376540d7ee52 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Thu, 29 Oct 2026 19:24:24 -0700 Subject: [PATCH 3/4] perf(ts): check std once per toolchain as a Nix sysroot The flake checks std for wasm32 with -Zbuild-std once per toolchain, with the profile translation reads its MIR at, and the dev shell and ts package pass it as RUST2_TS_SYSROOT, which the wrapper gives the target's crates as --sysroot. Without it, each target directory checks std with -Zbuild-std as before, from the same profile, now in ts/src/profile.toml. A nightly step runs the fixtures that way. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/nightly.yml | 5 +++ flake.nix | 27 +++++++++++++ quest/m2/README.md | 1 - quest/m2/ts-std-build.md | 26 ------------ ts/README.md | 4 +- ts/src/cli.rs | 75 ++++++++++++++++++----------------- ts/src/config.rs | 2 + ts/src/driver.rs | 8 +++- ts/src/profile.toml | 17 ++++++++ 9 files changed, 100 insertions(+), 65 deletions(-) delete mode 100644 quest/m2/ts-std-build.md create mode 100644 ts/src/profile.toml diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 0eb6b6383..28d855b4e 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -38,6 +38,11 @@ jobs: - name: Build run: nix build .#ts + # The fixtures against std checked in their own target directories, as outside Nix, + # whose snapshots hold the flake's sysroot to the same output. + - name: Fixtures without the sysroot + run: nix develop --command env RUST2_TS_SYSROOT= just ts test --test fixtures + uniffi-fixtures: name: UniFFI fixtures # x86_64: nixpkgs marks Swift 6.3 broken on aarch64-linux, where it fails to link with diff --git a/flake.nix b/flake.nix index c67f46bbc..fc15b2ca8 100644 --- a/flake.nix +++ b/flake.nix @@ -72,6 +72,31 @@ # the repo toolchain. ts/justfile puts it first for its recipes. tsToolchain = pkgs.rust-bin.fromRustupToolchainFile ./ts/rust-toolchain.toml; quest-cli = quest.packages.${system}.default; + # std checked once per toolchain for the TypeScript driver's target, with the + # profile translation reads its MIR at, as the sysroot rust2-ts takes instead of + # checking std with -Zbuild-std in every target directory. Offline, from the + # crates rust-src vendors. + rust2-ts-std = pkgs.stdenv.mkDerivation { + name = "rust2-ts-std"; + dontUnpack = true; + nativeBuildInputs = [ tsToolchain ]; + buildPhase = '' + export CARGO_HOME=$TMPDIR/cargo CARGO_UNSTABLE_PROFILE_RUSTFLAGS=true + mkdir -p sysroot/src && cd sysroot + printf '[package]\nname = "sysroot"\nversion = "0.0.0"\nedition = "2024"\n' > Cargo.toml + touch src/lib.rs + # The crates ts/src/cli.rs's STD names. + cargo check --offline --target wasm32-unknown-unknown -Zbuild-std=core,alloc,std,panic_abort \ + --config "source.crates-io.replace-with = 'vendored'" \ + --config "source.vendored.directory = '${tsToolchain}/lib/rustlib/src/rust/library/vendor'" \ + --config ${./ts/src/profile.toml} + ''; + installPhase = '' + lib=$out/lib/rustlib/wasm32-unknown-unknown/lib + mkdir -p $lib + find target/wasm32-unknown-unknown -name 'lib*.rmeta' ! -name 'libsysroot-*' -exec cp -t $lib {} + + ''; + }; # The driver links its nightly's rustc libraries, and runs cargo from it too. rust2-ts = (pkgs.makeRustPlatform { @@ -101,6 +126,7 @@ # tsc checks bindings, from the caller's PATH first. postInstall = '' wrapProgram $out/bin/rust2-ts --set-default CARGO ${tsToolchain}/bin/cargo \ + --set-default RUST2_TS_SYSROOT ${rust2-ts-std} \ --suffix PATH : ${pkgs.typescript}/bin ''; # The tests need node, tsc, and wasm builds: `just ts test` runs them. @@ -275,6 +301,7 @@ kotlin ]; RUST2_TS_TOOLCHAIN = tsToolchain; + RUST2_TS_SYSROOT = rust2-ts-std; KOTLINX_COROUTINES = "${kotlinx-coroutines}"; }; # `just ts bench` adds Chrome and Firefox as engines, and wasm-bindgen for moq-wasm's diff --git a/quest/m2/README.md b/quest/m2/README.md index b548d7dc9..372185fcc 100644 --- a/quest/m2/README.md +++ b/quest/m2/README.md @@ -30,6 +30,5 @@ The fork's feature quests folded into the [exporter](/quest/m1/ffi-export/README - [Async in the TypeScript translator](/quest/m2/ts-async.md) - moq-net's and kio's async code translates with Rust's lazy futures - [Shared ownership](/quest/m2/ts-shared.md) - `Arc`/`Rc`/`Weak` keep exact counts, and locks and atomics become plain values - [Translate moq-net's model to TypeScript](/quest/m2/ts-model/README.md) - the model translates with kio, moq-clock, and moq-pattern, for moq to compare against WASM -- [Check std once per toolchain](/quest/m2/ts-std-build.md) - check std as a Nix derivation the translator takes as its sysroot, instead of in every cold target directory - [Waiting: Swift 6.4 in nixpkgs-unstable](/quest/m2/uniffi-swift-arm.md) - the nightly UniFFI fixtures move back to arm - [Waiting: ubrn reads UniFFI 0.32](/quest/m2/ubrn-release.md) - the binding benchmark's browser runner pins a ubrn release, not an open PR diff --git a/quest/m2/ts-std-build.md b/quest/m2/ts-std-build.md deleted file mode 100644 index a09c07fa5..000000000 --- a/quest/m2/ts-std-build.md +++ /dev/null @@ -1,26 +0,0 @@ -# [S] Check std once per toolchain - -## Goal - -rust2-ts checks std from source in every cold target directory, through `-Zbuild-std`, so -its translations see std's un-inlined MIR: the prebuilt std's optimized MIR changes -snapshots and the census and refuses `LocalKey`'s methods (measured in -[#237](https://github.com/kixelated/rust2/pull/237)). Check std once per toolchain -instead, as a Nix derivation the translator takes as its sysroot, so CI's runners, which -keep `/nix/store`, check it only when the nightly changes. Snapshots and moq's census stay -byte-identical. - -## Plan - -Decided on #237 on 2026-10-29. - -- A derivation checks std with `-Zbuild-std` from the toolchain's `rust-src`, offline - from its vendored crates, with the flags translation uses, and keeps the `.rmeta` files - as a sysroot. -- The dev shell and the `ts` package point the translator at it; the wrapper passes it to - rustc as `--sysroot` for the target's crates only. -- Without one, outside Nix, the translator falls back to `-Zbuild-std`, from one shared - list of flags; a nightly job runs the suite that way, so the two stay identical. -- Measure cold and warm CPU-s for a fixture and for the suite. - -Public API: none; the `ts` package and dev shell set a new environment variable. diff --git a/ts/README.md b/ts/README.md index 2bd2e4054..3f3004b4e 100644 --- a/ts/README.md +++ b/ts/README.md @@ -7,7 +7,9 @@ rust2-ts --config ts.toml (--out dir | --census file.md) [--manifest-path Cargo. ``` It runs `cargo check` on the package with itself as `RUSTC_WRAPPER`, for -`wasm32-unknown-unknown`, with std built from source for it with every function's MIR, which needs the toolchain's `rust-src`. Every crate compiles as rustc +`wasm32-unknown-unknown`, against std checked from source with every function's MIR. The flake's +package and dev shell check it once per toolchain and pass it as `RUST2_TS_SYSROOT`; without +that, each target directory checks it with `-Zbuild-std`, which needs the toolchain's `rust-src`. Every crate compiles as rustc would; the configured library is also translated, after type and borrow checking, starting from its roots and following what they reach. Every crate it depends on but std's translates with it as one program, as far as the roots reach: each compiles with every function's MIR, and its types and diff --git a/ts/src/cli.rs b/ts/src/cli.rs index 85a21859b..d35b3365c 100644 --- a/ts/src/cli.rs +++ b/ts/src/cli.rs @@ -1,5 +1,6 @@ //! The command line: run cargo on the configured package with this binary as rustc. +use std::path::Path; use std::process::ExitCode; use rust2_driver::cargo::{Args, Check, Output}; @@ -8,7 +9,7 @@ use crate::config::{self, Config}; /// Translation type-checks for this target: its 32-bit `usize` is what a JS number /// holds exactly, and its `cfg`s are the browser's. -const TARGET: &str = "wasm32-unknown-unknown"; +pub const TARGET: &str = "wasm32-unknown-unknown"; const USAGE: &str = "usage: rust2-ts --config (--out | --census ) [--manifest-path ] [-- ] @@ -44,11 +45,9 @@ fn try_run(args: &[String]) -> Result { output, ("CARGO_UNSTABLE_PROFILE_RUSTFLAGS", "true".as_ref()), ]; - let mut cargo = std_from_source(); - cargo.push("--config".into()); - cargo.push(format!("profile.dev.rustflags={}", toml_list(FLAGS))); - // The profile those flags and std's are set on, which cargo refuses to combine with - // one the cargo args pick. + let mut cargo = profile_args()?; + // The profile `PROFILE` sets, which cargo refuses to combine with one the cargo args + // pick. cargo.extend(["--profile".into(), "dev".into()]); cargo.extend(args.cargo); Check { @@ -62,39 +61,43 @@ fn try_run(args: &[String]) -> Result { .run() } -/// What every crate compiles with, through the profile, which cargo's fingerprint sees, -/// for a crate cached without them to compile again: the package and every crate it -/// depends on translate as one program. -/// -/// Every function's MIR, which a `cargo check` encodes for no dependency's functions -/// otherwise. Without the checks of undefined behavior that debug builds add around raw -/// pointers, such as a box's: JS has no pointers to check. With every call where the -/// source has it, as a release build's inlining would not keep it, since a call is what a -/// binding or a std function's translation stands in for. And with `rust2_ts` set, so a -/// crate can give translation a safe twin of its `unsafe`. std's crates have their own -/// below, and the rest of std's take these to no effect, since none translates. -const FLAGS: &[&str] = &[ - "-Zalways-encode-mir", - "-Zub-checks=no", - "-Zinline-mir=no", - "--cfg=rust2_ts", -]; +/// The cargo config of the profile every crate compiles with, one `--config` per line. +const PROFILE: &str = include_str!("profile.toml"); -fn toml_list(items: &[&str]) -> String { - let items: Vec = items.iter().map(|item| format!("{item:?}")).collect(); - format!("[{}]", items.join(", ")) +/// The std crates `-Zbuild-std` checks, which flake.nix's sysroot holds too. +const STD: &str = "-Zbuild-std=core,alloc,std,panic_abort"; + +/// Cargo args that check std from source, unless [`sysroot`] has it checked already, and +/// set [`PROFILE`]. +fn profile_args() -> Result, String> { + let mut args = Vec::new(); + if sysroot()?.is_none() { + args.push(STD.to_string()); + } + for line in PROFILE + .lines() + .filter(|line| !line.is_empty() && !line.starts_with('#')) + { + args.extend(["--config".to_string(), line.to_string()]); + } + Ok(args) } -/// Cargo args that build std from source with every function's MIR, so the bodies of -/// std's combinators lower as the crate's do: the target's prebuilt std encodes the MIR -/// of few, optimized. Set through the profile, which cargo's fingerprint sees. -fn std_from_source() -> Vec { - let mut args = vec!["-Zbuild-std=core,alloc,std,panic_abort".to_string()]; - for krate in ["core", "alloc", "std"] { - args.push("--config".into()); - args.push(format!( - "profile.dev.package.{krate}.rustflags=[\"-Zalways-encode-mir\"]" +/// The sysroot of std checked with [`PROFILE`] for [`TARGET`], which rustc takes for the +/// target's crates instead of a `-Zbuild-std` in every target directory, if set. +pub fn sysroot() -> Result, String> { + let dir = match std::env::var(config::env::SYSROOT) { + Ok(dir) if !dir.is_empty() => dir, + Err(std::env::VarError::NotUnicode(_)) => return Err(format!("{} is not UTF-8", config::env::SYSROOT)), + _ => return Ok(None), + }; + let lib = Path::new(&dir).join("lib/rustlib").join(TARGET).join("lib"); + if !lib.is_dir() { + return Err(format!( + "{} is {dir}, which holds no std for {TARGET} in {}", + config::env::SYSROOT, + lib.display() )); } - args + Ok(Some(dir)) } diff --git a/ts/src/config.rs b/ts/src/config.rs index a61296b91..f81f42ad8 100644 --- a/ts/src/config.rs +++ b/ts/src/config.rs @@ -80,4 +80,6 @@ pub mod env { pub const OUT: &str = "RUST2_TS_OUT"; /// Where to write a census instead of translating, absolute. pub const CENSUS: &str = "RUST2_TS_CENSUS"; + /// A sysroot of std checked for translation, which flake.nix builds once per toolchain. + pub const SYSROOT: &str = "RUST2_TS_SYSROOT"; } diff --git a/ts/src/driver.rs b/ts/src/driver.rs index 8b0cf9b3d..12952baa4 100644 --- a/ts/src/driver.rs +++ b/ts/src/driver.rs @@ -9,11 +9,17 @@ use rustc_interface::interface::Compiler; use rustc_middle::ty::TyCtxt; use crate::config::{self, Config}; -use crate::{lower, program}; +use crate::{cli, lower, program}; /// Run as rustc, with `args` as cargo passed them: the wrapped rustc's path first. pub fn run(args: Vec) -> ExitCode { rust2_driver::rustc::run("rust2-ts", args, |args| { + // The target's crates only: build scripts and proc macros run on the host, with its std. + if args.windows(2).any(|w| w[0] == "--target" && w[1] == cli::TARGET) + && let Some(sysroot) = cli::sysroot()? + { + args.extend(["--sysroot".into(), sysroot]); + } Ok(Translate::for_args(args)?.map(|job| Box::new(job) as rust2_driver::rustc::Job)) }) } diff --git a/ts/src/profile.toml b/ts/src/profile.toml new file mode 100644 index 000000000..8ee682f7e --- /dev/null +++ b/ts/src/profile.toml @@ -0,0 +1,17 @@ +# The profile every crate compiles with, one `--config` per line, which cargo's +# fingerprint sees, for a crate cached without it to compile again: the package and every +# crate it depends on translate as one program. flake.nix checks std with it too. +# +# Every function's MIR, which a `cargo check` encodes for no dependency's functions +# otherwise. Without the checks of undefined behavior that debug builds add around raw +# pointers, such as a box's: JS has no pointers to check. With every call where the source +# has it, as a release build's inlining would not keep it, since a call is what a binding +# or a std function's translation stands in for. And with `rust2_ts` set, so a crate can +# give translation a safe twin of its `unsafe`. +profile.dev.rustflags = ["-Zalways-encode-mir", "-Zub-checks=no", "-Zinline-mir=no", "--cfg=rust2_ts"] +# std's own crates, with every function's MIR, so the bodies of std's combinators lower as +# the crate's do: the target's prebuilt std encodes the MIR of few, optimized. The rest of +# std's take the flags above to no effect, since none translates. +profile.dev.package.core.rustflags = ["-Zalways-encode-mir"] +profile.dev.package.alloc.rustflags = ["-Zalways-encode-mir"] +profile.dev.package.std.rustflags = ["-Zalways-encode-mir"] From 4232079782d461ce4384d8fcc48268fabdad8983 Mon Sep 17 00:00:00 2001 From: Luke Curley <432854+kixelated@users.noreply.github.com> Date: Thu, 29 Oct 2026 19:38:27 -0700 Subject: [PATCH 4/4] fix(ts): resolve a relative RUST2_TS_SYSROOT before cargo runs the wrapper elsewhere Co-Authored-By: Claude Opus 5.5 --- ts/src/cli.rs | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/ts/src/cli.rs b/ts/src/cli.rs index d35b3365c..c73eff048 100644 --- a/ts/src/cli.rs +++ b/ts/src/cli.rs @@ -39,13 +39,16 @@ fn try_run(args: &[String]) -> Result { Output::Dir(out) => (config::env::OUT, out.as_os_str()), Output::Census(census) => (config::env::CENSUS, census.as_os_str()), }; - // `cargo clean` reads the profiles too. + let sysroot = sysroot()?; + // `cargo clean` reads the profiles too. The sysroot is absolute for the wrapper, which + // cargo runs elsewhere, and empty for none. let env = [ (config::env::CONFIG, args.config.as_os_str()), output, ("CARGO_UNSTABLE_PROFILE_RUSTFLAGS", "true".as_ref()), + (config::env::SYSROOT, sysroot.as_deref().unwrap_or("").as_ref()), ]; - let mut cargo = profile_args()?; + let mut cargo = profile_args(sysroot.is_none()); // The profile `PROFILE` sets, which cargo refuses to combine with one the cargo args // pick. cargo.extend(["--profile".into(), "dev".into()]); @@ -67,11 +70,10 @@ const PROFILE: &str = include_str!("profile.toml"); /// The std crates `-Zbuild-std` checks, which flake.nix's sysroot holds too. const STD: &str = "-Zbuild-std=core,alloc,std,panic_abort"; -/// Cargo args that check std from source, unless [`sysroot`] has it checked already, and -/// set [`PROFILE`]. -fn profile_args() -> Result, String> { +/// Cargo args that set [`PROFILE`], and check std from source if `std`. +fn profile_args(std: bool) -> Vec { let mut args = Vec::new(); - if sysroot()?.is_none() { + if std { args.push(STD.to_string()); } for line in PROFILE @@ -80,17 +82,23 @@ fn profile_args() -> Result, String> { { args.extend(["--config".to_string(), line.to_string()]); } - Ok(args) + args } /// The sysroot of std checked with [`PROFILE`] for [`TARGET`], which rustc takes for the -/// target's crates instead of a `-Zbuild-std` in every target directory, if set. +/// target's crates instead of a `-Zbuild-std` in every target directory, if set: +/// absolute. pub fn sysroot() -> Result, String> { let dir = match std::env::var(config::env::SYSROOT) { Ok(dir) if !dir.is_empty() => dir, Err(std::env::VarError::NotUnicode(_)) => return Err(format!("{} is not UTF-8", config::env::SYSROOT)), _ => return Ok(None), }; + let dir = std::path::absolute(&dir) + .map_err(|err| format!("resolving {}: {err}", config::env::SYSROOT))? + .into_os_string() + .into_string() + .expect("a UTF-8 path resolves to a UTF-8 path"); let lib = Path::new(&dir).join("lib/rustlib").join(TARGET).join("lib"); if !lib.is_dir() { return Err(format!(