Automate use of V8's version for crates shared by workerd and V8

ci/BUILD.bazel

@@ -0,0 +1,14 @@
+load("@rules_python//python:defs.bzl", "py_test")
+
+py_test(
+    name = "v8_update_test",
+    size = "small",
+    srcs = [
+        "jsonc.py",
+        "v8_update.py",
+        "v8_update_test.py",
+    ],
+    # v8_update.py imports jsonc.py, and the test imports v8_update.py, as top-level modules.
+    imports = ["."],
+    main = "v8_update_test.py",
+)

ci/v8_update.py

@@ -12,6 +12,7 @@
 import subprocess
 import sys
 import tempfile
+import tomllib
 from pathlib import Path
 from urllib.request import urlopen
 
@@ -43,6 +44,11 @@
 )
 RUST_ICU_VERSION = re.compile(r'(?<="RUST_ICU_MAJOR_VERSION_NUMBER": ")\d+(?=")')
 
+# V8 builds these crates from Chromium's third_party/rust. Every crate in their
+# runtime dependency trees must resolve to the version Chromium builds V8 with.
+CHROMIUM_CRATE_ROOTS = ("temporal_capi",)
+CARGO_TOML = ROOT / "deps/rust/Cargo.toml"
+
 
 def run(args, *, cwd=ROOT, check=True, env=None, stdout=None):
     print("+", shlex.join(map(str, args)), flush=True)
@@ -101,6 +107,14 @@ def changed_dependencies(old, target):
     )
 
 
+def _v8_deps_from_gitiles(tag):
+    with urlopen(
+        f"https://chromium.googlesource.com/v8/v8/+/refs/tags/{tag}/DEPS?format=TEXT",
+        timeout=30,
+    ) as response:
+        return base64.b64decode(response.read()).decode()
+
+
 def _update_aligned_dependencies():
     doc = jsonc.loads(DEPS.read_text())
     repositories = {repo["name"]: repo for repo in doc.data["repositories"]}
@@ -146,6 +160,111 @@ def _update_rust_icu_version():
         )
 
 
+def _chromium_crate_versions(commit):
+    """Return {name: [version, ...]} from Chromium's chromium_crates_io/Cargo.lock."""
+    with urlopen(
+        "https://chromium.googlesource.com/chromium/src/third_party/rust/+/"
+        f"{commit}/chromium_crates_io/Cargo.lock?format=TEXT",
+        timeout=30,
+    ) as response:
+        lock = tomllib.loads(base64.b64decode(response.read()).decode())
+    versions = {}
+    for package in lock["package"]:
+        versions.setdefault(package["name"], []).append(package["version"])
+    return versions
+
+
+def _semver_major(version):
+    # The components that Cargo's default (caret) requirements hold fixed: those
+    # up to and including the first nonzero one, as in 2.x.y, 0.2.x and 0.0.3.
+    parts = version.split(".")
+    nonzero = next((i for i, part in enumerate(parts) if part != "0"), len(parts) - 1)
+    return parts[: nonzero + 1]
+
+
+def _chromium_version(chromium, name, version):
+    """Return Chromium's version of a crate, or None if there isn't exactly one.
+
+    When Chromium locks several versions of the crate, choose the one that is
+    semver-compatible with `version`.
+    """
+    versions = chromium.get(name, [])
+    if len(versions) > 1:
+        versions = [v for v in versions if _semver_major(v) == _semver_major(version)]
+    return versions[0] if len(versions) == 1 else None
+
+
+def _cargo(*args, check=True, stdout=None):
+    return run(
+        ["cargo", *args, "--manifest-path", CARGO_TOML], check=check, stdout=stdout
+    ).stdout
+
+
+def _chromium_crate_tree():
+    """Return the (name, version) runtime dependencies of CHROMIUM_CRATE_ROOTS."""
+    args = ["tree", "--target=all", "--edges=normal,no-proc-macro"]
+    for root in CHROMIUM_CRATE_ROOTS:
+        args += ["--package", root]
+    tree = _cargo(*args, "--prefix=none", "--format={p}", stdout=subprocess.PIPE)
+    packages = (line.split()[:2] for line in tree.splitlines() if line)
+    return {(name, version.removeprefix("v")) for name, version in packages}
+
+
+def _exact_pins():
+    """Return {name: version} for the dependencies that Cargo.toml pins with `=`."""
+    dependencies = tomllib.loads(CARGO_TOML.read_text())["dependencies"]
+    pins = {}
+    for name, spec in dependencies.items():
+        version = spec if isinstance(spec, str) else spec.get("version", "")
+        if version.startswith("="):
+            pins[name] = version.removeprefix("=")
+    return pins
+
+
+def _update_chromium_crates(v8_deps):
+    commit = _v8_dependency_commit("third_party/rust", v8_deps)
+    chromium = _chromium_crate_versions(commit)
+
+    # Exact pins in Cargo.toml hold crates that Cargo would otherwise resolve
+    # newer than Chromium's versions. Move them first, since moving a pinned crate
+    # can move others along with it. `cargo add` keeps each dependency's features,
+    # and moves all the pins in one resolution, since one pin's new version may
+    # require another's.
+    pins = _exact_pins()
+    repins = []
+    for name, version in sorted(_chromium_crate_tree()):
+        target = _chromium_version(chromium, name, version)
+        if pins.get(name) == version and target not in (None, version):
+            repins.append(f"{name}@={target}")
+    if repins:
+        _cargo("add", *repins)
+
+    # Cargo.lock alone holds the rest. If Cargo refuses a version, it prints why,
+    # and the check below names the crate.
+    for name, version in _chromium_crate_tree():
+        target = _chromium_version(chromium, name, version)
+        if target not in (None, version):
+            _cargo("update", f"{name}@{version}", "--precise", target, check=False)
+
+    mismatched = [
+        f"{name} {version} (Chromium: {', '.join(chromium.get(name, [])) or 'none'})"
+        for name, version in sorted(_chromium_crate_tree())
+        if _chromium_version(chromium, name, version) != version
+    ]
+    if mismatched:
+        raise ValueError(
+            f"Crates differ from Chromium's third_party/rust at {commit}: "
+            f"{', '.join(mismatched)}. If Cargo cannot downgrade a crate by itself, "
+            f'pin it with `<crate> = {{ version = "=<version>" }}` in {CARGO_TOML}.'
+        )
+
+    # Bazel builds the crates from cargo-bazel-lock.json, which must be regenerated
+    # after Cargo.toml or Cargo.lock changes (see build/deps/rust.MODULE.bazel).
+    # Regenerate it even if this run changed neither, since an earlier run that
+    # failed the check above, or a pin added by hand, may have.
+    run(["bazel", "fetch", "--repo_env=CARGO_BAZEL_REPIN=1", "--repo=@crates_vendor"])
+
+
 def _update_module(tag, patch_names):
     text = MODULE.read_text()
     text = re.sub(
@@ -273,6 +392,7 @@ def finish_update(target):
         ]
     )
     _update_rust_icu_version()
+    _update_chromium_crates((CHECKOUT / "DEPS").read_text())
 
 
 def parse_args():
@@ -284,6 +404,10 @@ def parse_args():
         "check-update", help="check Chrome Beta for a V8 update"
     )
     check.add_argument("--machine-readable", action="store_true")
+    commands.add_parser(
+        "rust-crates",
+        help="align the Rust crates V8 builds with the current V8 version's Chromium",
+    )
     for name in ("update", "finish"):
         command = commands.add_parser(name)
         command.add_argument("version", metavar="VERSION")
@@ -301,6 +425,9 @@ def main():
         return 1
     if args.command == "update":
         return 0 if prepare_update(args.version) else 1
+    if args.command == "rust-crates":
+        _update_chromium_crates(_v8_deps_from_gitiles(read_version()))
+        return 0
     if args.command == "finish":
         finish_update(args.version)
         return 0

ci/v8_update_test.py

@@ -0,0 +1,152 @@
+"""Tests for the Chromium Rust crate alignment in v8_update.py.
+
+Cargo, Bazel and the Chromium lockfile are mocked, so the tests check which
+versions the helper asks Cargo for, and when it fails.
+"""
+
+import tempfile
+import unittest
+from pathlib import Path
+from unittest import mock
+
+import v8_update
+
+V8_DEPS = (
+    "'third_party/rust': Var('chromium_url') + '/chromium/src/third_party/rust.git' + '@' + "
+    f"'{'0' * 40}',"
+)
+
+CARGO_TOML = """\
+[package]
+name = "direct-cargo-bazel-deps"
+
+[dependencies]
+icu_calendar = { version = "=2.3.0", default-features = false }
+temporal_rs = { version = "=0.2.6", default-features = false, features = ["float64_representable_durations"] }
+tinystr = "0.8"
+writeable = "=0.6.2"
+"""
+
+# Chromium locks two versions of icu_calendar and of zerovec.
+CHROMIUM = {
+    "icu_calendar": ["1.5.2", "2.2.1"],
+    "temporal_rs": ["0.2.6"],
+    "tinystr": ["0.8.2"],
+    "writeable": ["0.6.2"],
+    "zerovec": ["0.10.4", "0.11.5"],
+}
+
+
+class UpdateChromiumCratesTest(unittest.TestCase):
+    def setUp(self):
+        directory = tempfile.TemporaryDirectory()
+        self.addCleanup(directory.cleanup)
+        cargo_toml = Path(directory.name) / "Cargo.toml"
+        cargo_toml.write_text(CARGO_TOML)
+        for target, value in (
+            ("CARGO_TOML", cargo_toml),
+            ("_chromium_crate_versions", mock.Mock(return_value=CHROMIUM)),
+        ):
+            patcher = mock.patch.object(v8_update, target, value)
+            patcher.start()
+            self.addCleanup(patcher.stop)
+        self.cargo = self._patch("_cargo")
+        self.run_command = self._patch("run")
+        self.tree = self._patch("_chromium_crate_tree")
+
+    def _patch(self, name):
+        patcher = mock.patch.object(v8_update, name)
+        self.addCleanup(patcher.stop)
+        return patcher.start()
+
+    def test_aligned_tree_changes_nothing(self):
+        self.tree.return_value = {
+            ("icu_calendar", "2.2.1"),
+            ("temporal_rs", "0.2.6"),
+            ("zerovec", "0.11.5"),
+        }
+        v8_update._update_chromium_crates(V8_DEPS)
+        self.cargo.assert_not_called()
+        self._assert_repinned_bazel_lockfile()
+
+    def test_moves_exact_pins_with_cargo_add(self):
+        # The pin moves to the Chromium version in the same semver range, not to 1.5.2.
+        aligned = {("icu_calendar", "2.2.1"), ("temporal_rs", "0.2.6")}
+        self.tree.side_effect = [
+            {("icu_calendar", "2.3.0"), ("temporal_rs", "0.2.6")},
+            aligned,
+            aligned,
+        ]
+        v8_update._update_chromium_crates(V8_DEPS)
+        self.cargo.assert_called_once_with("add", "icu_calendar@=2.2.1")
+        self._assert_repinned_bazel_lockfile()
+
+    def test_moves_unpinned_crates_with_cargo_update(self):
+        self.tree.side_effect = [
+            {("tinystr", "0.8.3"), ("zerovec", "0.11.6")},
+            {("tinystr", "0.8.3"), ("zerovec", "0.11.6")},
+            {("tinystr", "0.8.2"), ("zerovec", "0.11.5")},
+        ]
+        v8_update._update_chromium_crates(V8_DEPS)
+        self.assertCountEqual(
+            self.cargo.call_args_list,
+            [
+                mock.call("update", "tinystr@0.8.3", "--precise", "0.8.2", check=False),
+                mock.call(
+                    "update", "zerovec@0.11.6", "--precise", "0.11.5", check=False
+                ),
+            ],
+        )
+        self._assert_repinned_bazel_lockfile()
+
+    def test_fails_when_cargo_refuses_a_version(self):
+        # Cargo refuses to downgrade zerovec, and Chromium doesn't lock unicode-ident.
+        tree = {("zerovec", "0.11.6"), ("unicode-ident", "1.0.0")}
+        self.tree.side_effect = [tree, tree, tree]
+        with self.assertRaisesRegex(
+            ValueError,
+            r"unicode-ident 1\.0\.0 \(Chromium: none\), "
+            r"zerovec 0\.11\.6 \(Chromium: 0\.10\.4, 0\.11\.5\)",
+        ):
+            v8_update._update_chromium_crates(V8_DEPS)
+        self.cargo.assert_called_once_with(
+            "update", "zerovec@0.11.6", "--precise", "0.11.5", check=False
+        )
+        self.run_command.assert_not_called()
+
+    def _assert_repinned_bazel_lockfile(self):
+        self.run_command.assert_called_once_with(
+            [
+                "bazel",
+                "fetch",
+                "--repo_env=CARGO_BAZEL_REPIN=1",
+                "--repo=@crates_vendor",
+            ]
+        )
+
+
+class ChromiumVersionTest(unittest.TestCase):
+    def test_single_version_is_chosen_across_semver_ranges(self):
+        # An exact pin can follow Chromium to a new semver range.
+        self.assertEqual(
+            v8_update._chromium_version(
+                {"temporal_rs": ["0.3.0"]}, "temporal_rs", "0.2.6"
+            ),
+            "0.3.0",
+        )
+
+    def test_duplicate_versions_are_matched_by_semver_range(self):
+        chromium = {
+            "a": ["0.0.3", "0.0.4"],
+            "b": ["0.1.0", "0.2.0"],
+            "c": ["1.0.0", "2.1.0"],
+        }
+        self.assertEqual(v8_update._chromium_version(chromium, "a", "0.0.4"), "0.0.4")
+        self.assertEqual(v8_update._chromium_version(chromium, "b", "0.2.5"), "0.2.0")
+        self.assertEqual(v8_update._chromium_version(chromium, "c", "2.0.0"), "2.1.0")
+        self.assertIsNone(v8_update._chromium_version(chromium, "c", "3.0.0"))
+        self.assertIsNone(v8_update._chromium_version(chromium, "d", "1.0.0"))
+
+
+if __name__ == "__main__":
+    unittest.main()

deps/rust/Cargo.toml

@@ -92,7 +92,8 @@ serde = { version = "1", features = ["derive"] }
 # V8's Temporal implementation (//deps/rust:temporal_capi). V8 compiles against temporal_capi's
 # generated C++ headers, so both crates should be pinned to the versions and features the V8
 # release expects: those in Chromium's third_party/rust/chromium_crates_io/Cargo.toml, at the
-# third_party/rust revision pinned in V8's DEPS.
+# third_party/rust revision pinned in V8's DEPS. `python3 ci/v8_update.py rust-crates` updates
+# the versions of these and other exact pins in temporal_capi's dependency tree.
 temporal_capi = { version = "=0.2.6", features = ["zoneinfo64"] }
 temporal_rs = { version = "=0.2.6", default-features = false, features = ["float64_representable_durations"] }
 # temporal_rs 0.2.6 accepts icu_calendar 2.3, but Chromium builds it with 2.2.1; this pin keeps

docs/v8-updates.md

@@ -91,6 +91,17 @@ from the V8 directory.
     `rust_icu_sys` if the two values disagree. If `rust_icu_sys` has no `bindgen/lib_<version>.rs`
     for the new version, update the crate as well.
 
+    V8 builds `temporal_capi` (see `deps/rust/Cargo.toml`) from Chromium's
+    `third_party/rust`, at the revision in V8's `DEPS`. Every crate in its runtime
+    dependency tree should resolve to the version locked in that revision's
+    `chromium_crates_io/Cargo.lock`. `python3 ci/v8_update.py rust-crates` moves the exact
+    (`=`) pins of those crates in `deps/rust/Cargo.toml` with `cargo add`, moves the others
+    in `deps/rust/Cargo.lock` with `cargo update --precise`, and then regenerates
+    `deps/rust/cargo-bazel-lock.json` with
+    `bazel fetch --repo_env=CARGO_BAZEL_REPIN=1 --repo=@crates_vendor`. It needs Python 3.11
+    or newer, and `cargo` and `bazel` on `PATH`. If Cargo cannot downgrade a crate on its
+    own, the helper names it; pin it in `deps/rust/Cargo.toml` and rerun.
+
 11. Check workerd's tests pass with the updated V8.
 
      ```sh
@@ -131,7 +142,8 @@ rebases the patches, regenerates `patches/v8/*.patch`, and updates:
 
 It also regenerates `com_googlesource_chromium_icu` in `build/deps/gen/deps.MODULE.bazel`
 and sets `RUST_ICU_MAJOR_VERSION_NUMBER` in `build/deps/rust.MODULE.bazel` and
-`src/rust/i18n/BUILD.bazel` to the major version of the new ICU commit (see step 10).
+`src/rust/i18n/BUILD.bazel` to the major version of the new ICU commit, and aligns the
+Rust crates V8 builds from Chromium's `third_party/rust` (see step 10).
 
 If the rebase stops because of conflicts, resolve and continue it in the temporary
 checkout: