diff --git a/ci/BUILD.bazel b/ci/BUILD.bazel new file mode 100644 index 00000000000..67ccad94f43 --- /dev/null +++ b/ci/BUILD.bazel @@ -0,0 +1,14 @@ +load("@rules_python//python:defs.bzl", "py_test") + +py_test( + name = "v8_update_test", + size = "small", + srcs = [ + "jsonc.py", + "v8_update.py", + "v8_update_test.py", + ], + # v8_update.py imports jsonc.py, and the test imports v8_update.py, as top-level modules. + imports = ["."], + main = "v8_update_test.py", +) diff --git a/ci/v8_update.py b/ci/v8_update.py index 23929f9a9e0..c0f4992360c 100644 --- a/ci/v8_update.py +++ b/ci/v8_update.py @@ -12,6 +12,7 @@ import subprocess import sys import tempfile +import tomllib from pathlib import Path from urllib.request import urlopen @@ -43,6 +44,11 @@ ) RUST_ICU_VERSION = re.compile(r'(?<="RUST_ICU_MAJOR_VERSION_NUMBER": ")\d+(?=")') +# V8 builds these crates from Chromium's third_party/rust. Every crate in their +# runtime dependency trees must resolve to the version Chromium builds V8 with. +CHROMIUM_CRATE_ROOTS = ("temporal_capi",) +CARGO_TOML = ROOT / "deps/rust/Cargo.toml" + def run(args, *, cwd=ROOT, check=True, env=None, stdout=None): print("+", shlex.join(map(str, args)), flush=True) @@ -101,6 +107,14 @@ def changed_dependencies(old, target): ) +def _v8_deps_from_gitiles(tag): + with urlopen( + f"https://chromium.googlesource.com/v8/v8/+/refs/tags/{tag}/DEPS?format=TEXT", + timeout=30, + ) as response: + return base64.b64decode(response.read()).decode() + + def _update_aligned_dependencies(): doc = jsonc.loads(DEPS.read_text()) repositories = {repo["name"]: repo for repo in doc.data["repositories"]} @@ -146,6 +160,111 @@ def _update_rust_icu_version(): ) +def _chromium_crate_versions(commit): + """Return {name: [version, ...]} from Chromium's chromium_crates_io/Cargo.lock.""" + with urlopen( + "https://chromium.googlesource.com/chromium/src/third_party/rust/+/" + f"{commit}/chromium_crates_io/Cargo.lock?format=TEXT", + timeout=30, + ) as response: + lock = tomllib.loads(base64.b64decode(response.read()).decode()) + versions = {} + for package in lock["package"]: + versions.setdefault(package["name"], []).append(package["version"]) + return versions + + +def _semver_major(version): + # The components that Cargo's default (caret) requirements hold fixed: those + # up to and including the first nonzero one, as in 2.x.y, 0.2.x and 0.0.3. + parts = version.split(".") + nonzero = next((i for i, part in enumerate(parts) if part != "0"), len(parts) - 1) + return parts[: nonzero + 1] + + +def _chromium_version(chromium, name, version): + """Return Chromium's version of a crate, or None if there isn't exactly one. + + When Chromium locks several versions of the crate, choose the one that is + semver-compatible with `version`. + """ + versions = chromium.get(name, []) + if len(versions) > 1: + versions = [v for v in versions if _semver_major(v) == _semver_major(version)] + return versions[0] if len(versions) == 1 else None + + +def _cargo(*args, check=True, stdout=None): + return run( + ["cargo", *args, "--manifest-path", CARGO_TOML], check=check, stdout=stdout + ).stdout + + +def _chromium_crate_tree(): + """Return the (name, version) runtime dependencies of CHROMIUM_CRATE_ROOTS.""" + args = ["tree", "--target=all", "--edges=normal,no-proc-macro"] + for root in CHROMIUM_CRATE_ROOTS: + args += ["--package", root] + tree = _cargo(*args, "--prefix=none", "--format={p}", stdout=subprocess.PIPE) + packages = (line.split()[:2] for line in tree.splitlines() if line) + return {(name, version.removeprefix("v")) for name, version in packages} + + +def _exact_pins(): + """Return {name: version} for the dependencies that Cargo.toml pins with `=`.""" + dependencies = tomllib.loads(CARGO_TOML.read_text())["dependencies"] + pins = {} + for name, spec in dependencies.items(): + version = spec if isinstance(spec, str) else spec.get("version", "") + if version.startswith("="): + pins[name] = version.removeprefix("=") + return pins + + +def _update_chromium_crates(v8_deps): + commit = _v8_dependency_commit("third_party/rust", v8_deps) + chromium = _chromium_crate_versions(commit) + + # Exact pins in Cargo.toml hold crates that Cargo would otherwise resolve + # newer than Chromium's versions. Move them first, since moving a pinned crate + # can move others along with it. `cargo add` keeps each dependency's features, + # and moves all the pins in one resolution, since one pin's new version may + # require another's. + pins = _exact_pins() + repins = [] + for name, version in sorted(_chromium_crate_tree()): + target = _chromium_version(chromium, name, version) + if pins.get(name) == version and target not in (None, version): + repins.append(f"{name}@={target}") + if repins: + _cargo("add", *repins) + + # Cargo.lock alone holds the rest. If Cargo refuses a version, it prints why, + # and the check below names the crate. + for name, version in _chromium_crate_tree(): + target = _chromium_version(chromium, name, version) + if target not in (None, version): + _cargo("update", f"{name}@{version}", "--precise", target, check=False) + + mismatched = [ + f"{name} {version} (Chromium: {', '.join(chromium.get(name, [])) or 'none'})" + for name, version in sorted(_chromium_crate_tree()) + if _chromium_version(chromium, name, version) != version + ] + if mismatched: + raise ValueError( + f"Crates differ from Chromium's third_party/rust at {commit}: " + f"{', '.join(mismatched)}. If Cargo cannot downgrade a crate by itself, " + f'pin it with ` = {{ version = "=" }}` in {CARGO_TOML}.' + ) + + # Bazel builds the crates from cargo-bazel-lock.json, which must be regenerated + # after Cargo.toml or Cargo.lock changes (see build/deps/rust.MODULE.bazel). + # Regenerate it even if this run changed neither, since an earlier run that + # failed the check above, or a pin added by hand, may have. + run(["bazel", "fetch", "--repo_env=CARGO_BAZEL_REPIN=1", "--repo=@crates_vendor"]) + + def _update_module(tag, patch_names): text = MODULE.read_text() text = re.sub( @@ -273,6 +392,7 @@ def finish_update(target): ] ) _update_rust_icu_version() + _update_chromium_crates((CHECKOUT / "DEPS").read_text()) def parse_args(): @@ -284,6 +404,10 @@ def parse_args(): "check-update", help="check Chrome Beta for a V8 update" ) check.add_argument("--machine-readable", action="store_true") + commands.add_parser( + "rust-crates", + help="align the Rust crates V8 builds with the current V8 version's Chromium", + ) for name in ("update", "finish"): command = commands.add_parser(name) command.add_argument("version", metavar="VERSION") @@ -301,6 +425,9 @@ def main(): return 1 if args.command == "update": return 0 if prepare_update(args.version) else 1 + if args.command == "rust-crates": + _update_chromium_crates(_v8_deps_from_gitiles(read_version())) + return 0 if args.command == "finish": finish_update(args.version) return 0 diff --git a/ci/v8_update_test.py b/ci/v8_update_test.py new file mode 100644 index 00000000000..0add3c3fd53 --- /dev/null +++ b/ci/v8_update_test.py @@ -0,0 +1,152 @@ +"""Tests for the Chromium Rust crate alignment in v8_update.py. + +Cargo, Bazel and the Chromium lockfile are mocked, so the tests check which +versions the helper asks Cargo for, and when it fails. +""" + +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +import v8_update + +V8_DEPS = ( + "'third_party/rust': Var('chromium_url') + '/chromium/src/third_party/rust.git' + '@' + " + f"'{'0' * 40}'," +) + +CARGO_TOML = """\ +[package] +name = "direct-cargo-bazel-deps" + +[dependencies] +icu_calendar = { version = "=2.3.0", default-features = false } +temporal_rs = { version = "=0.2.6", default-features = false, features = ["float64_representable_durations"] } +tinystr = "0.8" +writeable = "=0.6.2" +""" + +# Chromium locks two versions of icu_calendar and of zerovec. +CHROMIUM = { + "icu_calendar": ["1.5.2", "2.2.1"], + "temporal_rs": ["0.2.6"], + "tinystr": ["0.8.2"], + "writeable": ["0.6.2"], + "zerovec": ["0.10.4", "0.11.5"], +} + + +class UpdateChromiumCratesTest(unittest.TestCase): + def setUp(self): + directory = tempfile.TemporaryDirectory() + self.addCleanup(directory.cleanup) + cargo_toml = Path(directory.name) / "Cargo.toml" + cargo_toml.write_text(CARGO_TOML) + for target, value in ( + ("CARGO_TOML", cargo_toml), + ("_chromium_crate_versions", mock.Mock(return_value=CHROMIUM)), + ): + patcher = mock.patch.object(v8_update, target, value) + patcher.start() + self.addCleanup(patcher.stop) + self.cargo = self._patch("_cargo") + self.run_command = self._patch("run") + self.tree = self._patch("_chromium_crate_tree") + + def _patch(self, name): + patcher = mock.patch.object(v8_update, name) + self.addCleanup(patcher.stop) + return patcher.start() + + def test_aligned_tree_changes_nothing(self): + self.tree.return_value = { + ("icu_calendar", "2.2.1"), + ("temporal_rs", "0.2.6"), + ("zerovec", "0.11.5"), + } + v8_update._update_chromium_crates(V8_DEPS) + self.cargo.assert_not_called() + self._assert_repinned_bazel_lockfile() + + def test_moves_exact_pins_with_cargo_add(self): + # The pin moves to the Chromium version in the same semver range, not to 1.5.2. + aligned = {("icu_calendar", "2.2.1"), ("temporal_rs", "0.2.6")} + self.tree.side_effect = [ + {("icu_calendar", "2.3.0"), ("temporal_rs", "0.2.6")}, + aligned, + aligned, + ] + v8_update._update_chromium_crates(V8_DEPS) + self.cargo.assert_called_once_with("add", "icu_calendar@=2.2.1") + self._assert_repinned_bazel_lockfile() + + def test_moves_unpinned_crates_with_cargo_update(self): + self.tree.side_effect = [ + {("tinystr", "0.8.3"), ("zerovec", "0.11.6")}, + {("tinystr", "0.8.3"), ("zerovec", "0.11.6")}, + {("tinystr", "0.8.2"), ("zerovec", "0.11.5")}, + ] + v8_update._update_chromium_crates(V8_DEPS) + self.assertCountEqual( + self.cargo.call_args_list, + [ + mock.call("update", "tinystr@0.8.3", "--precise", "0.8.2", check=False), + mock.call( + "update", "zerovec@0.11.6", "--precise", "0.11.5", check=False + ), + ], + ) + self._assert_repinned_bazel_lockfile() + + def test_fails_when_cargo_refuses_a_version(self): + # Cargo refuses to downgrade zerovec, and Chromium doesn't lock unicode-ident. + tree = {("zerovec", "0.11.6"), ("unicode-ident", "1.0.0")} + self.tree.side_effect = [tree, tree, tree] + with self.assertRaisesRegex( + ValueError, + r"unicode-ident 1\.0\.0 \(Chromium: none\), " + r"zerovec 0\.11\.6 \(Chromium: 0\.10\.4, 0\.11\.5\)", + ): + v8_update._update_chromium_crates(V8_DEPS) + self.cargo.assert_called_once_with( + "update", "zerovec@0.11.6", "--precise", "0.11.5", check=False + ) + self.run_command.assert_not_called() + + def _assert_repinned_bazel_lockfile(self): + self.run_command.assert_called_once_with( + [ + "bazel", + "fetch", + "--repo_env=CARGO_BAZEL_REPIN=1", + "--repo=@crates_vendor", + ] + ) + + +class ChromiumVersionTest(unittest.TestCase): + def test_single_version_is_chosen_across_semver_ranges(self): + # An exact pin can follow Chromium to a new semver range. + self.assertEqual( + v8_update._chromium_version( + {"temporal_rs": ["0.3.0"]}, "temporal_rs", "0.2.6" + ), + "0.3.0", + ) + + def test_duplicate_versions_are_matched_by_semver_range(self): + chromium = { + "a": ["0.0.3", "0.0.4"], + "b": ["0.1.0", "0.2.0"], + "c": ["1.0.0", "2.1.0"], + } + self.assertEqual(v8_update._chromium_version(chromium, "a", "0.0.4"), "0.0.4") + self.assertEqual(v8_update._chromium_version(chromium, "b", "0.2.5"), "0.2.0") + self.assertEqual(v8_update._chromium_version(chromium, "c", "2.0.0"), "2.1.0") + self.assertIsNone(v8_update._chromium_version(chromium, "c", "3.0.0")) + self.assertIsNone(v8_update._chromium_version(chromium, "d", "1.0.0")) + + +if __name__ == "__main__": + unittest.main() diff --git a/deps/rust/Cargo.toml b/deps/rust/Cargo.toml index d278c2bd7b5..62f7a51d584 100644 --- a/deps/rust/Cargo.toml +++ b/deps/rust/Cargo.toml @@ -92,7 +92,8 @@ serde = { version = "1", features = ["derive"] } # V8's Temporal implementation (//deps/rust:temporal_capi). V8 compiles against temporal_capi's # generated C++ headers, so both crates should be pinned to the versions and features the V8 # release expects: those in Chromium's third_party/rust/chromium_crates_io/Cargo.toml, at the -# third_party/rust revision pinned in V8's DEPS. +# third_party/rust revision pinned in V8's DEPS. `python3 ci/v8_update.py rust-crates` updates +# the versions of these and other exact pins in temporal_capi's dependency tree. temporal_capi = { version = "=0.2.6", features = ["zoneinfo64"] } temporal_rs = { version = "=0.2.6", default-features = false, features = ["float64_representable_durations"] } # temporal_rs 0.2.6 accepts icu_calendar 2.3, but Chromium builds it with 2.2.1; this pin keeps diff --git a/docs/v8-updates.md b/docs/v8-updates.md index 89451b53a2f..4199d779c83 100644 --- a/docs/v8-updates.md +++ b/docs/v8-updates.md @@ -91,6 +91,17 @@ from the V8 directory. `rust_icu_sys` if the two values disagree. If `rust_icu_sys` has no `bindgen/lib_.rs` for the new version, update the crate as well. + V8 builds `temporal_capi` (see `deps/rust/Cargo.toml`) from Chromium's + `third_party/rust`, at the revision in V8's `DEPS`. Every crate in its runtime + dependency tree should resolve to the version locked in that revision's + `chromium_crates_io/Cargo.lock`. `python3 ci/v8_update.py rust-crates` moves the exact + (`=`) pins of those crates in `deps/rust/Cargo.toml` with `cargo add`, moves the others + in `deps/rust/Cargo.lock` with `cargo update --precise`, and then regenerates + `deps/rust/cargo-bazel-lock.json` with + `bazel fetch --repo_env=CARGO_BAZEL_REPIN=1 --repo=@crates_vendor`. It needs Python 3.11 + or newer, and `cargo` and `bazel` on `PATH`. If Cargo cannot downgrade a crate on its + own, the helper names it; pin it in `deps/rust/Cargo.toml` and rerun. + 11. Check workerd's tests pass with the updated V8. ```sh @@ -131,7 +142,8 @@ rebases the patches, regenerates `patches/v8/*.patch`, and updates: It also regenerates `com_googlesource_chromium_icu` in `build/deps/gen/deps.MODULE.bazel` and sets `RUST_ICU_MAJOR_VERSION_NUMBER` in `build/deps/rust.MODULE.bazel` and -`src/rust/i18n/BUILD.bazel` to the major version of the new ICU commit (see step 10). +`src/rust/i18n/BUILD.bazel` to the major version of the new ICU commit, and aligns the +Rust crates V8 builds from Chromium's `third_party/rust` (see step 10). If the rebase stops because of conflicts, resolve and continue it in the temporary checkout: