From 8900056acf242164632720c5d3f15b463efa8430 Mon Sep 17 00:00:00 2001 From: Ryan Dahl Date: Sun, 25 Oct 2026 23:57:06 -0700 Subject: [PATCH 01/38] Cluster workerd processes over a shared bucket This is the beginning of merging celld's functionality into workerd. It lets several workerd processes network into a cluster by giving each of them access to the same S3-compatible bucket (`Config.cluster`, `--experimental`). Each Durable Object has one owner at a time, requests reach it from any process, and when its owner dies another process picks it up from the bucket. Objects use the bucket as cold storage with Litestream's algorithm and file formats. For now an object acknowledges a write only once it is fully synced to the bucket; that is much too slow and expensive for most uses. Next we will acknowledge writes once peers in the cluster hold them and buffer the bucket writes. Alarms are also not supported yet. --- deps/rust/Cargo.lock | 17 + deps/rust/Cargo.toml | 4 + src/workerd/server/AGENTS.md | 1 + src/workerd/server/channel-token.c++ | 15 +- src/workerd/server/channel-token.h | 3 +- src/workerd/server/cli/args-test.rs | 4 + src/workerd/server/cli/args.rs | 23 +- src/workerd/server/cli/lib.rs | 2 + src/workerd/server/cluster/BUILD.bazel | 32 + src/workerd/server/cluster/bucket.rs | 233 ++++ src/workerd/server/cluster/capture.rs | 330 +++++ src/workerd/server/cluster/cluster.rs | 364 ++++++ src/workerd/server/cluster/http.rs | 287 +++++ src/workerd/server/cluster/lib.rs | 111 ++ src/workerd/server/cluster/logic/BUILD.bazel | 8 + src/workerd/server/cluster/logic/lease.rs | 145 +++ src/workerd/server/cluster/logic/lib.rs | 16 + .../server/cluster/ltx/client/bucket.rs | 136 ++ src/workerd/server/cluster/ltx/client/mod.rs | 48 + src/workerd/server/cluster/ltx/codec.rs | 354 ++++++ src/workerd/server/cluster/ltx/error.rs | 22 + src/workerd/server/cluster/ltx/ltx.rs | 542 ++++++++ src/workerd/server/cluster/ltx/mod.rs | 92 ++ src/workerd/server/cluster/ltx/replica.rs | 157 +++ src/workerd/server/cluster/ltx_tests.rs | 518 ++++++++ src/workerd/server/cluster/owner.rs | 839 +++++++++++++ src/workerd/server/cluster/ownership.rs | 642 ++++++++++ src/workerd/server/cluster/registry.rs | 316 +++++ src/workerd/server/cluster/s3.rs | 552 +++++++++ .../server/factory/worker-factory-actor.c++ | 71 +- .../server/factory/worker-factory-impl.h | 25 +- .../server/factory/worker-factory-rpc.c++ | 33 +- src/workerd/server/factory/worker-factory.c++ | 13 +- src/workerd/server/factory/worker-factory.h | 8 + src/workerd/server/server/BUILD.bazel | 3 + src/workerd/server/server/actor.rs | 320 ++++- src/workerd/server/server/bridge.rs | 40 + src/workerd/server/server/cluster.rs | 487 ++++++++ src/workerd/server/server/config.rs | 43 + src/workerd/server/server/lib.rs | 1 + src/workerd/server/server/run.rs | 45 + src/workerd/server/server/worker.rs | 2 + src/workerd/server/tests/cluster/BUILD.bazel | 23 + .../server/tests/cluster/cluster-test.mjs | 1098 +++++++++++++++++ .../server/tests/cluster/cluster-test.wd-test | 53 + .../server/tests/cluster/cluster-worker.js | 314 +++++ src/workerd/server/tests/cluster/fake-s3.js | 115 ++ .../server/tests/cluster/fake-s3.wd-test | 31 + src/workerd/server/workerd.capnp | 63 +- src/workerd/util/sqlite-kv-test.c++ | 20 + src/workerd/util/sqlite-kv.c++ | 14 +- src/workerd/util/sqlite-kv.h | 2 +- src/workerd/util/sqlite-metadata-test.c++ | 18 + src/workerd/util/sqlite-metadata.c++ | 14 +- src/workerd/util/sqlite-metadata.h | 2 +- src/workerd/util/sqlite-test.c++ | 33 + src/workerd/util/sqlite.c++ | 139 ++- src/workerd/util/sqlite.h | 63 +- 58 files changed, 8838 insertions(+), 68 deletions(-) create mode 100644 src/workerd/server/cluster/BUILD.bazel create mode 100644 src/workerd/server/cluster/bucket.rs create mode 100644 src/workerd/server/cluster/capture.rs create mode 100644 src/workerd/server/cluster/cluster.rs create mode 100644 src/workerd/server/cluster/http.rs create mode 100644 src/workerd/server/cluster/lib.rs create mode 100644 src/workerd/server/cluster/logic/BUILD.bazel create mode 100644 src/workerd/server/cluster/logic/lease.rs create mode 100644 src/workerd/server/cluster/logic/lib.rs create mode 100644 src/workerd/server/cluster/ltx/client/bucket.rs create mode 100644 src/workerd/server/cluster/ltx/client/mod.rs create mode 100644 src/workerd/server/cluster/ltx/codec.rs create mode 100644 src/workerd/server/cluster/ltx/error.rs create mode 100644 src/workerd/server/cluster/ltx/ltx.rs create mode 100644 src/workerd/server/cluster/ltx/mod.rs create mode 100644 src/workerd/server/cluster/ltx/replica.rs create mode 100644 src/workerd/server/cluster/ltx_tests.rs create mode 100644 src/workerd/server/cluster/owner.rs create mode 100644 src/workerd/server/cluster/ownership.rs create mode 100644 src/workerd/server/cluster/registry.rs create mode 100644 src/workerd/server/cluster/s3.rs create mode 100644 src/workerd/server/server/cluster.rs create mode 100644 src/workerd/server/tests/cluster/BUILD.bazel create mode 100644 src/workerd/server/tests/cluster/cluster-test.mjs create mode 100644 src/workerd/server/tests/cluster/cluster-test.wd-test create mode 100644 src/workerd/server/tests/cluster/cluster-worker.js create mode 100644 src/workerd/server/tests/cluster/fake-s3.js create mode 100644 src/workerd/server/tests/cluster/fake-s3.wd-test diff --git a/deps/rust/Cargo.lock b/deps/rust/Cargo.lock index 50125229587..64f5a6a14f4 100644 --- a/deps/rust/Cargo.lock +++ b/deps/rust/Cargo.lock @@ -587,6 +587,7 @@ dependencies = [ "libm", "libz-rs-sys", "lol_html_c_api", + "lz4_flex", "nix", "notify", "percent-encoding", @@ -594,6 +595,7 @@ dependencies = [ "proc-macro2", "quote", "rand 0.9.5", + "ring", "ruff_python_ast", "ruff_python_parser", "rust_icu_sys", @@ -1396,6 +1398,15 @@ dependencies = [ "thiserror", ] +[[package]] +name = "lz4_flex" +version = "0.11.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "373f5eceeeab7925e0c1098212f2fbc4d416adec9d35051a6ab251e824c1854a" +dependencies = [ + "twox-hash", +] + [[package]] name = "memchr" version = "2.8.3" @@ -2783,6 +2794,12 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" +[[package]] +name = "twox-hash" +version = "2.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "86a801b3cea342a06d468c8710662aa29e5e05e4f5c0d62f00bbb7f2ad7941c2" + [[package]] name = "typenum" version = "1.20.1" diff --git a/deps/rust/Cargo.toml b/deps/rust/Cargo.toml index ec07b6ed352..e66c65d9cda 100644 --- a/deps/rust/Cargo.toml +++ b/deps/rust/Cargo.toml @@ -67,6 +67,8 @@ libc = "0" libm = "0" lol_html_c_api = { git = "https://github.com/cloudflare/lol-html" } hashlink = "0" +# cluster's LTX port (src/workerd/server/cluster/ltx): LTX page compression. +lz4_flex = { version = "0.11", default-features = false, features = ["frame", "safe-decode", "safe-encode"] } # kj-rs-io: `fs` for fcntl (KJ's fd flags), `signal` for ignoring SIGPIPE once per process like # kj::UnixEventPort does. nix = { version = "0", features = ["fs", "signal"] } @@ -77,6 +79,8 @@ pico-args = "0" rand = { version = "0.9", default-features = false, features = ["os_rng", "thread_rng"] } ruff_python_ast = { git = "https://github.com/astral-sh/ruff", tag = "0.12.1" } ruff_python_parser = { git = "https://github.com/astral-sh/ruff", tag = "0.12.1" } +# cluster: SigV4's SHA-256 and HMAC, and its peer MACs. +ring = { version = "0.17", default-features = false } # Use rust_icu's checked-in bindings for the Chromium ICU version linked by workerd. ICU itself is # supplied by Bazel, so bindgen, icu-config, and rust_icu's library-linking features stay disabled. rust_icu_sys = { version = "5.8", default-features = false, features = ["renaming", "icu_version_in_env"] } diff --git a/src/workerd/server/AGENTS.md b/src/workerd/server/AGENTS.md index 6404c3fdbf7..3bdf9463d10 100644 --- a/src/workerd/server/AGENTS.md +++ b/src/workerd/server/AGENTS.md @@ -7,6 +7,7 @@ The `workerd` binary and its server. `:workerd` (`workerd.rs`, this package) is - `cli/` -- the `workerd-cli` crate (declared in this package's `BUILD.bazel`, sources globbed from `cli/`): parses the command line (clap), produces the encoded config (schema files are compiled by this package's `config-compiler.c++`), handles `compile`, `--watch` and re-exec, and hands `serve`, `test` (which `fuzzilli` and `make-pyodide-baseline-snapshot` are built on) and `bench` to the server crate's `entry` module. - `server/` -- the `workerd-server` crate: the single-tenant server. It interprets the config, builds the service graph, binds the sockets, runs the listeners, actors and dynamic workers, drains on SIGTERM, and runs `workerd test` and `workerd bench`. `Error` is `cxx::KjError`; `bridge.rs` is the one module that allows `unsafe`. - `factory/` -- C++: `WorkerFactory` (`worker-factory*`), everything that needs the isolate, driven by the server through the cxx bridge in `server/bridge.rs`; and `bootstrap.{h,c++}`, the C++ half of a command. `with_process_context` sets up the process's logging (the JSON logger, when the config asks for structured logging) on the C++ stack around the Rust command; `bootstrap` sets up perfetto, autogates and V8 on the Rust-owned event loop and builds the `WorkerFactory`, which owns them, from the encoded config and the command-line options for `server/entry.rs`, which owns the loop (`kj_rs_tokio::Runtime`) and blocks on `server_serve` / `server_test` / `server_bench`. +- `cluster/` -- the `cluster` crate and, in `cluster/logic/`, `cluster-logic`: cluster mode (`Config.cluster`, experimental), Durable Objects whose shared state is an S3-compatible bucket. Leases, ownership, LTX capture and restore; pure Rust, driven by `server/cluster.rs`. `cluster-logic` holds decisions only, with no dependencies to do I/O with. This package keeps the config schema (`workerd.capnp`, `compiled-bindings.capnp`), the config compiler, the `WorkerdApi` and the C++ libraries the factory is built from (alarm scheduler, channel tokens, container client, facet tree index, fallback service, actor IDs, JSON logger, Pyodide, V8 platform), and their C++ unit tests. diff --git a/src/workerd/server/channel-token.c++ b/src/workerd/server/channel-token.c++ index e8967b8d77f..0c911faa690 100644 --- a/src/workerd/server/channel-token.c++ +++ b/src/workerd/server/channel-token.c++ @@ -9,6 +9,7 @@ #include #include +#include #include #include @@ -22,8 +23,18 @@ namespace workerd::server { -ChannelTokenHandler::ChannelTokenHandler(Resolver& resolver): resolver(resolver) { - getEntropy(tokenKey); +ChannelTokenHandler::ChannelTokenHandler(Resolver& resolver, kj::Maybe clusterKey) + : resolver(resolver) { + KJ_IF_SOME(key, clusterKey) { + // Labeled so this key never equals another one derived from the cluster key. + auto label = "workerd channel token v1"_kj; + uint outLen = 0; + KJ_ASSERT(HMAC(EVP_sha256(), label.begin(), label.size(), key.asBytes().begin(), key.size(), + tokenKey, &outLen) != nullptr); + KJ_ASSERT(outLen == sizeof(tokenKey)); + } else { + getEntropy(tokenKey); + } SHA256_CTX ctx{}; KJ_ASSERT(SHA256_Init(&ctx)); diff --git a/src/workerd/server/channel-token.h b/src/workerd/server/channel-token.h index 5d2a8f79672..e3fe946e5de 100644 --- a/src/workerd/server/channel-token.h +++ b/src/workerd/server/channel-token.h @@ -60,7 +60,8 @@ class ChannelTokenHandler: public kj::PtrTarget { IoChannelFactory::ChannelTokenUsage usage) = 0; }; - explicit ChannelTokenHandler(Resolver& resolver); + // Instances of a cluster share `clusterKey`, so each decodes the others' tokens. + explicit ChannelTokenHandler(Resolver& resolver, kj::Maybe clusterKey = kj::none); // Helpers to implement `IoChannelFactory::{SubrequestChannel,ActorClassChannel}::getToken()`. // diff --git a/src/workerd/server/cli/args-test.rs b/src/workerd/server/cli/args-test.rs index 09f8bc93061..4787dcaa452 100644 --- a/src/workerd/server/cli/args-test.rs +++ b/src/workerd/server/cli/args-test.rs @@ -65,6 +65,8 @@ fn serve_options() { "--async-trace=trace.ndjson", "--async-trace-stacks=8", "--async-trace-promises", + "--cluster-addr=0.0.0.0:7000", + "--cluster-advertise=10.0.0.4:7000", ]) .command else { @@ -85,6 +87,8 @@ fn serve_options() { assert_eq!(serve_or_test.async_trace_stacks, Some(8)); assert!(serve_or_test.async_trace_promises); assert_eq!(serve.control_fd, Some(3)); + assert_eq!(serve.cluster_address.as_deref(), Some("0.0.0.0:7000")); + assert_eq!(serve.cluster_advertise.as_deref(), Some("10.0.0.4:7000")); assert_eq!( serve.socket_addr_overrides, [Override { diff --git a/src/workerd/server/cli/args.rs b/src/workerd/server/cli/args.rs index 86213e3e0a8..2ae0b173457 100644 --- a/src/workerd/server/cli/args.rs +++ b/src/workerd/server/cli/args.rs @@ -55,7 +55,13 @@ pub enum Command { /// run the server #[command( long_about = "Serve requests based on a config.", - after_long_help = "Serves requests based on the configuration specified in ." + after_long_help = "Serves requests based on the configuration specified in .\n\n\ + Cluster mode (`Config.cluster`, with --experimental): instances that share an \ + S3-compatible bucket serve Durable Objects together. Each `cluster.bucketOptions` \ + setting falls back to an environment variable: AWS_ENDPOINT_URL, AWS_REGION, \ + AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN, or \ + AWS_CREDENTIALS_FILE. The cluster port must stay on a private network; see \ + --cluster-addr." )] Serve { #[command(flatten)] @@ -329,6 +335,21 @@ pub struct ServeArgs { #[arg(long, value_name = "fd", value_parser = parse_control_fd)] pub control_fd: Option, + /// Bind the cluster's peer port to instead of the config's `cluster.address`. The + /// port gives full access to every service in the process to holders of the cluster key, and + /// its traffic is not encrypted: only the cluster's instances should reach it, on a private + /// network (a VPC, WireGuard, Tailscale). A public IP is refused unless the config sets + /// `cluster.unsafePublicAddress`; a wildcard (0.0.0.0) listens on every interface, so + /// firewall the port. + #[arg(long = "cluster-addr", value_name = "addr")] + pub cluster_address: Option, + + /// Advertise to the cluster's other instances instead of the config's + /// `cluster.advertise` or the bound address. A public IP is refused unless the config sets + /// `cluster.unsafePublicAddress`. + #[arg(long = "cluster-advertise", value_name = "addr")] + pub cluster_advertise: Option, + /// Listen on the specified address for debug RPC connections. This exposes a privileged /// interface that allows access to all services in the process. For use by miniflare and local /// development only. diff --git a/src/workerd/server/cli/lib.rs b/src/workerd/server/cli/lib.rs index d6e8714f5f7..e7a31fd1394 100644 --- a/src/workerd/server/cli/lib.rs +++ b/src/workerd/server/cli/lib.rs @@ -471,6 +471,8 @@ fn serve_options( socket_fds, control, debug_port: serve.debug_port, + cluster_address: serve.cluster_address, + cluster_advertise: serve.cluster_advertise, ..run_options(serve_or_test) }; (options, inherited) diff --git a/src/workerd/server/cluster/BUILD.bazel b/src/workerd/server/cluster/BUILD.bazel new file mode 100644 index 00000000000..c4d6432180d --- /dev/null +++ b/src/workerd/server/cluster/BUILD.bazel @@ -0,0 +1,32 @@ +load("//:build/wd_rust_crate.bzl", "wd_rust_crate") + +# Durable Objects whose shared state is an S3-compatible bucket. Pure Rust: +# the server (//src/workerd/server/server) drives it. +wd_rust_crate( + name = "cluster", + proc_macro_deps = ["@crates_vendor//:async-trait"], + # The capture's tests drive a real SQLite. + test_deps = ["@sqlite3"], + visibility = ["//src/workerd/server:__subpackages__"], + deps = [ + "//src/workerd/server/cluster/logic:cluster-logic", + "@crates_vendor//:bytes", + "@crates_vendor//:futures", + "@crates_vendor//:http", + "@crates_vendor//:http-body", + "@crates_vendor//:httpdate", + "@crates_vendor//:hyper-util", + "@crates_vendor//:lz4_flex", + "@crates_vendor//:nix", + "@crates_vendor//:ring", + "@crates_vendor//:rustls", + "@crates_vendor//:rustls-platform-verifier", + "@crates_vendor//:serde", + "@crates_vendor//:serde_json", + "@crates_vendor//:thiserror", + "@crates_vendor//:tokio", + "@crates_vendor//:tokio-rustls", + "@crates_vendor//:tower-service", + "@crates_vendor//:tracing", + ], +) diff --git a/src/workerd/server/cluster/bucket.rs b/src/workerd/server/cluster/bucket.rs new file mode 100644 index 00000000000..5535ff9acd5 --- /dev/null +++ b/src/workerd/server/cluster/bucket.rs @@ -0,0 +1,233 @@ +// Copyright (c) 2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +use std::path::Path; +use std::rc::Rc; + +use crate::Result; +pub use crate::http::Payload; + +/// The identity of one stored version of an object: what a conditional +/// write names to replace exactly that version. An S3 `ETag`. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Version(pub(crate) String); + +/// The precondition of a conditional write. +pub enum Guard { + /// The object must not exist. + Absent, + /// The object must still be this version. + Match(Version), +} + +/// One listed object; `key` is relative to the bucket's prefix. +pub struct Entry { + pub key: String, +} + +/// An object store: S3's semantics for the calls the cluster makes. +#[async_trait::async_trait(?Send)] +pub trait Store { + /// The object and its version; with `to`, the body goes to that file + /// instead. + async fn get(&self, key: &str, to: Option<&Path>) -> Result, Version)>>; + /// `Ok(None)` when the guard failed and nothing was written. + async fn put( + &self, + key: &str, + body: Payload, + guard: Option, + metadata: &[(&str, String)], + ) -> Result>; + async fn list(&self, prefix: &str) -> Result>; + /// Idempotent: deleting an absent key succeeds. + async fn delete(&self, key: &str) -> Result<()>; + /// See `S3::clock_skew_ms`. + fn clock_skew_ms(&self) -> Option { + None + } +} + +/// An object store under a key prefix. Clones share the store. +#[derive(Clone)] +pub struct Bucket { + store: Rc, + prefix: String, +} + +impl Bucket { + pub fn new(store: impl Store + 'static, prefix: &str) -> Self { + Self { + store: Rc::new(store), + prefix: normalize(prefix), + } + } + + /// The same store under another prefix. + #[must_use] + pub fn at(&self, prefix: &str) -> Self { + Self { + store: self.store.clone(), + prefix: normalize(prefix), + } + } + + #[cfg(test)] + pub fn memory(prefix: &str) -> Self { + Self::new(memory::Memory::default(), prefix) + } + + pub fn clock_skew_ms(&self) -> Option { + self.store.clock_skew_ms() + } + + pub(crate) fn prefix(&self) -> &str { + &self.prefix + } + + fn key(&self, key: &str) -> String { + format!("{}{key}", self.prefix) + } + + /// The object and its version, or `None` when it does not exist. + pub async fn get(&self, key: &str) -> Result, Version)>> { + self.store.get(&self.key(key), None).await + } + + /// Write `body` if the guard holds. `Ok(None)` means the guard failed and + /// nothing was written; an error means the write may have landed. + pub async fn put_if(&self, key: &str, body: Vec, guard: Guard) -> Result> { + let body = Payload::Bytes(body.into()); + self.store.put(&self.key(key), body, Some(guard), &[]).await + } + + /// Write `body` unconditionally, with `x-amz-meta-*` metadata. + pub async fn put(&self, key: &str, body: Vec, metadata: &[(&str, String)]) -> Result<()> { + let key = self.key(key); + let body = Payload::Bytes(body.into()); + self.store.put(&key, body, None, metadata).await.map(drop) + } + + /// Writes the object to the file `to`: `false` when it does not exist. + pub async fn get_file(&self, key: &str, to: &Path) -> Result { + let got = self.store.get(&self.key(key), Some(to)).await?; + Ok(got.is_some()) + } + + /// Writes the file `from` unconditionally, with `x-amz-meta-*` metadata. + pub async fn put_file( + &self, + key: &str, + from: &Path, + metadata: &[(&str, String)], + ) -> Result<()> { + let body = Payload::File(from.to_owned()); + self.store + .put(&self.key(key), body, None, metadata) + .await + .map(drop) + } + + /// The keys under `prefix` in lexical order. + pub async fn list(&self, prefix: &str) -> Result> { + let listed = self.store.list(&self.key(prefix)).await?; + Ok(listed + .into_iter() + .filter_map(|entry| { + let key = entry.key.strip_prefix(&self.prefix)?.to_owned(); + Some(Entry { key }) + }) + .collect()) + } + + /// Idempotent: deleting an absent key succeeds. + pub async fn delete(&self, key: &str) -> Result<()> { + self.store.delete(&self.key(key)).await + } +} + +fn normalize(prefix: &str) -> String { + let prefix = prefix.trim_matches('/'); + if prefix.is_empty() { + String::new() + } else { + format!("{prefix}/") + } +} + +#[cfg(test)] +pub mod memory { + use std::cell::Cell; + use std::cell::RefCell; + use std::collections::BTreeMap; + use std::path::Path; + + use super::Entry; + use super::Guard; + use super::Payload; + use super::Store; + use super::Version; + use crate::Result; + + /// S3's semantics for the calls the cluster makes, in memory. + #[derive(Default)] + pub struct Memory { + objects: RefCell, Version)>>, + writes: Cell, + /// As GCS's S3 API treats `If-Match` on a PUT. + pub ignores_guards: Cell, + } + + #[async_trait::async_trait(?Send)] + impl Store for Memory { + async fn get(&self, key: &str, to: Option<&Path>) -> Result, Version)>> { + let got = self.objects.borrow().get(key).cloned(); + match (got, to) { + (Some((body, version)), Some(to)) => { + std::fs::write(to, body).map_err(crate::ltx::Error::from)?; + Ok(Some((Vec::new(), version))) + } + (got, _) => Ok(got), + } + } + + async fn put( + &self, + key: &str, + body: Payload, + guard: Option, + _: &[(&str, String)], + ) -> Result> { + let body = match body { + Payload::Bytes(bytes) => bytes.to_vec(), + Payload::File(path) => std::fs::read(path).map_err(crate::ltx::Error::from)?, + }; + let mut objects = self.objects.borrow_mut(); + let current = objects.get(key).map(|(_, version)| version); + let holds = match guard { + None => true, + Some(Guard::Absent) => current.is_none(), + Some(Guard::Match(version)) => current == Some(&version), + }; + if !holds && !self.ignores_guards.get() { + return Ok(None); + } + self.writes.set(self.writes.get() + 1); + let version = Version(format!("\"{}\"", self.writes.get())); + objects.insert(key.to_owned(), (body, version.clone())); + Ok(Some(version)) + } + + async fn list(&self, prefix: &str) -> Result> { + let objects = self.objects.borrow(); + let keys = objects.keys().filter(|key| key.starts_with(prefix)); + Ok(keys.map(|key| Entry { key: key.clone() }).collect()) + } + + async fn delete(&self, key: &str) -> Result<()> { + self.objects.borrow_mut().remove(key); + Ok(()) + } + } +} diff --git a/src/workerd/server/cluster/capture.rs b/src/workerd/server/cluster/capture.rs new file mode 100644 index 00000000000..bbc39a94ace --- /dev/null +++ b/src/workerd/server/cluster/capture.rs @@ -0,0 +1,330 @@ +// Copyright (c) 2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +//! A Durable Object's commits, cut from the writes `SQLite` makes to its WAL +//! as workerd's VFS reports them: one LTX entry per transaction. +//! +//! `SQLite` writes a frame as its 24-byte header and then its page, at offsets +//! that follow the 32-byte WAL header, and the header of a transaction's +//! last frame carries the database's size, which marks the commit. A write +//! at offset 0 starts a new WAL generation, after a checkpoint has copied +//! every earlier frame into the database file. So the frames since the +//! last commit are buffered, and the write that completes a commit frame +//! completes one transaction. A rolled-back transaction's frames start +//! where the next one's do, which overwrites them. + +use std::collections::BTreeMap; +use std::collections::HashMap; +use std::fs::File; +use std::io::Read; +use std::io::Write; +use std::os::unix::fs::FileExt; +use std::path::Path; +use std::path::PathBuf; + +use crate::Error; +use crate::Result; +use crate::Txid; +use crate::ltx::TXID; +use crate::ltx::codec::Encoder; +use crate::ltx::ltx::HEADER_FLAG_NO_CHECKSUM; +use crate::ltx::ltx::Header; +use crate::ltx::ltx::PageHeader; +use crate::ltx::ltx::VERSION; +use crate::ltx::ltx::encode_file; +use crate::ltx::ltx::lock_pgno; + +const WAL_HEADER: u64 = 32; +const FRAME_HEADER: usize = 24; + +pub struct Capture { + /// The database file. + path: PathBuf, + /// The chain's entries not yet in the bucket or this node's journal, by + /// txid. + entries: BTreeMap>, + last: Txid, + page_size: usize, + /// Where the next transaction's first frame goes: 0 before a WAL header. + start: u64, + /// What was written from `start` on. + frames: Vec, + /// Where this WAL generation's latest committed version of each page + /// is, for a snapshot, and the database's size at the last commit. + pages: HashMap, + commit: u32, + /// The database was deleted: the next commit is a whole image, since a + /// new database's first page reaches the file, not the WAL. + fresh: bool, + /// Why the capture can no longer be trusted, if it cannot. + lost: Option, +} + +impl Capture { + /// Capture the database at `path`, which no connection has open yet: + /// what it holds now is the snapshot at txid 1, held in memory. + #[cfg(test)] + pub fn open(path: &Path) -> Result { + let mut capture = Self::resume(path, 0); + capture.restart()?; + Ok(capture) + } + + /// Continue a chain through `last` over the database at `path`, which + /// holds exactly that chain and no connection has open yet: an object + /// woken where it was evicted. + pub fn resume(path: &Path, last: Txid) -> Self { + Self { + path: path.to_owned(), + entries: BTreeMap::new(), + last, + page_size: 0, + start: 0, + frames: Vec::new(), + pages: HashMap::new(), + commit: 0, + fresh: false, + lost: None, + } + } + + /// Start a new chain from a snapshot of what is committed now. + #[cfg(test)] + pub fn restart(&mut self) -> Result<()> { + self.entries.clear(); + self.last = 0; + if let Some(snapshot) = self.snapshot(1, 1)? { + self.last = 1; + self.entries.insert(1, snapshot); + } + Ok(()) + } + + /// The database's files were deleted: the chain continues with an empty + /// database, and the next commit carries every page. + pub fn reset(&mut self) { + (self.start, self.pages, self.commit) = (0, HashMap::new(), 0); + self.frames.clear(); + self.fresh = true; + if self.last == 0 { + return; + } + let page_size = if self.page_size == 0 { + 4096 + } else { + self.page_size + }; + self.last += 1; + match encode(self.last, self.last, page_size, 0, &[]) { + Ok(entry) => drop(self.entries.insert(self.last, entry)), + Err(e) => self.lost = Some(format!("txid {}: {e}", self.last)), + } + } + + /// The database file overlaid with this WAL generation's committed + /// pages, read back from the WAL, as one entry covering `first` through + /// `last`. `None` for an empty database. + fn snapshot(&mut self, first: Txid, last: Txid) -> Result>> { + let mut image = match std::fs::read(&self.path) { + Ok(bytes) => bytes, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Vec::new(), + Err(e) => return Err(crate::ltx::Error::from(e).into()), + }; + let page_size = match image.get(16..18) { + Some(&[0, 1]) => 65536, + Some(size) => usize::from(u16::from_be_bytes([size[0], size[1]])), + None => self.page_size, + }; + if page_size == 0 { + return Ok(None); + } + self.page_size = page_size; + let mut wal = self.path.as_os_str().to_owned(); + wal.push("-wal"); + let wal = match File::open(wal) { + // Closing the last connection checkpoints the WAL into the + // database file and deletes it: the file holds every page, until + // a reopened connection writes a new WAL header. + Err(e) if e.kind() == std::io::ErrorKind::NotFound => None, + Ok(wal) if wal.metadata().is_ok_and(|m| m.len() == 0) => None, + wal => Some(wal.map_err(crate::ltx::Error::from)?), + }; + if let Some(wal) = wal.filter(|_| !self.pages.is_empty()) { + for (&pgno, &offset) in &self.pages { + let at = (pgno as usize - 1) * page_size; + if image.len() < at + page_size { + image.resize(at + page_size, 0); + } + wal.read_exact_at(&mut image[at..at + page_size], offset) + .map_err(crate::ltx::Error::from)?; + } + image.truncate(self.commit as usize * page_size); + } + let lock = lock_pgno(page_size as u32); + let pages: Vec<(u32, Vec)> = (1..) + .zip(image.chunks_exact(page_size)) + .filter(|(pgno, _)| *pgno != lock) + .map(|(pgno, page)| (pgno, page.to_vec())) + .collect(); + let commit = (image.len() / page_size) as u32; + (commit > 0) + .then(|| encode(first, last, page_size, commit, &pages)) + .transpose() + } + + /// A write `SQLite` made to the WAL. + pub fn write(&mut self, offset: u64, data: &[u8]) { + if offset == 0 { + // A new generation: a checkpoint copied the last one into the + // database file. + if let Some(size) = data.get(8..12) { + self.page_size = u32::from_be_bytes([size[0], size[1], size[2], size[3]]) as usize; + (self.start, self.pages) = (WAL_HEADER, HashMap::new()); + self.frames.clear(); + } + return; + } + if self.start == 0 || offset < self.start { + self.lost = Some(format!("a WAL write at {offset}, before {}", self.start)); + return; + } + let at = usize::try_from(offset - self.start).unwrap_or(usize::MAX); + let end = at.saturating_add(data.len()); + if self.frames.len() < end { + self.frames.resize(end, 0); + } + self.frames[at..end].copy_from_slice(data); + let size = FRAME_HEADER + self.page_size; + if end % size != 0 { + return; + } + let header = &self.frames[end - size..end - size + FRAME_HEADER]; + let commit = u32::from_be_bytes([header[4], header[5], header[6], header[7]]); + if commit != 0 { + self.commit_frames(end, commit); + } + } + + /// The frames before `end` are one committed transaction. + fn commit_frames(&mut self, end: usize, commit: u32) { + let size = FRAME_HEADER + self.page_size; + let mut pages = BTreeMap::new(); + for (i, frame) in self.frames[..end].chunks_exact(size).enumerate() { + let pgno = u32::from_be_bytes([frame[0], frame[1], frame[2], frame[3]]); + self.pages + .insert(pgno, self.start + (i * size + FRAME_HEADER) as u64); + if pgno <= commit { + pages.insert(pgno, frame[FRAME_HEADER..].to_vec()); + } + } + let pages: Vec<(u32, Vec)> = pages.into_iter().collect(); + self.start += end as u64; + self.frames.drain(..end); + self.commit = commit; + // A chain with no snapshot yet starts with one: a new database's + // first page reaches the file, not the WAL, when WAL mode is set. + let entry = if self.last == 0 || self.fresh { + self.fresh = false; + let first = if self.last == 0 { 1 } else { self.last + 1 }; + self.snapshot(first, self.last + 1).and_then(|s| { + s.ok_or_else(|| Error::Replica("a commit to an empty database".into())) + }) + } else { + encode(self.last + 1, self.last + 1, self.page_size, commit, &pages) + }; + self.last += 1; + match entry { + Ok(entry) => drop(self.entries.insert(self.last, entry)), + Err(e) => self.lost = Some(format!("txid {}: {e}", self.last)), + } + } + + /// The last committed txid, or why the capture cannot be trusted. + pub fn last(&self) -> Result { + match &self.lost { + None => Ok(self.last), + Some(why) => Err(Error::Replica(format!("capture lost: {why}"))), + } + } + + pub fn entry(&self, txid: Txid) -> Result> { + self.entries + .get(&txid) + .cloned() + .ok_or_else(|| Error::Replica(format!("no captured entry {txid}"))) + } + + /// The bucket, or this node's journal, holds everything through `txid`. + pub fn forget(&mut self, txid: Txid) { + self.entries.retain(|t, _| *t > txid); + } +} + +/// Encodes the database at `path`, which no connection has open, as the +/// snapshot at txid 1 into `out`, a page at a time: its size, or `None` for +/// an empty database. Closing the last connection checkpointed the WAL into +/// the file. +pub fn snapshot_file(path: &Path, out: &Path) -> Result> { + let io = |e: std::io::Error| Error::from(crate::ltx::Error::from(e)); + let file = match File::open(path) { + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(None), + file => file.map_err(io)?, + }; + let len = file.metadata().map_err(io)?.len(); + let mut reader = std::io::BufReader::new(file); + let mut first = [0; 100]; + if len < first.len() as u64 { + return Ok(None); + } + reader.read_exact(&mut first).map_err(io)?; + let page_size = match u16::from_be_bytes([first[16], first[17]]) { + 1 => 65536, + size => usize::from(size), + }; + let header = Header { + version: VERSION, + flags: HEADER_FLAG_NO_CHECKSUM, + page_size: page_size as u32, + commit: (len / page_size as u64) as u32, + min_txid: TXID(1), + max_txid: TXID(1), + ..Header::default() + }; + let mut writer = std::io::BufWriter::new(File::create(out).map_err(io)?); + let mut encoder = Encoder::new(&mut writer); + encoder.encode_header(header)?; + let mut page = vec![0; page_size]; + page[..first.len()].copy_from_slice(&first); + reader.read_exact(&mut page[first.len()..]).map_err(io)?; + for pgno in 1..=header.commit { + if pgno > 1 { + reader.read_exact(&mut page).map_err(io)?; + } + if pgno != lock_pgno(header.page_size) { + encoder.encode_page(PageHeader { pgno, flags: 0 }, &page)?; + } + } + encoder.close(0)?; + writer.flush().map_err(io)?; + Ok(Some(std::fs::metadata(out).map_err(io)?.len())) +} + +fn encode( + first: Txid, + last: Txid, + page_size: usize, + commit: u32, + pages: &[(u32, Vec)], +) -> Result> { + let header = Header { + version: VERSION, + flags: HEADER_FLAG_NO_CHECKSUM, + page_size: page_size as u32, + commit, + min_txid: TXID(first), + max_txid: TXID(last), + ..Header::default() + }; + Ok(encode_file(&header, pages, 0)?) +} diff --git a/src/workerd/server/cluster/cluster.rs b/src/workerd/server/cluster/cluster.rs new file mode 100644 index 00000000000..237880d0c9f --- /dev/null +++ b/src/workerd/server/cluster/cluster.rs @@ -0,0 +1,364 @@ +// Copyright (c) 2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +//! One instance's view of the cluster: what the server calls. Runs on the +//! server's loop thread, inside its tokio `current_thread` runtime. + +use std::cell::RefCell; +use std::collections::HashMap; +use std::path::Path; +use std::rc::Rc; +use std::time::Duration; +use std::time::SystemTime; +use std::time::UNIX_EPOCH; + +use cluster_logic::lease; +use cluster_logic::lease::Lease; +use cluster_logic::lease::Step; +use tokio::time::timeout; + +use crate::Bucket; +use crate::Claim; +use crate::Error; +use crate::Guard; +use crate::Ownership; +use crate::Registry; +use crate::Result; +use crate::Version; +use crate::hmac_sha256; +use crate::owner::Owners; +use crate::ownership::Owned; +use crate::s3::S3; + +/// The bucket layout's version, the last element of every key's prefix. +const FORMAT: &str = "v1"; + +/// Claims a new object retries when another node opened it first. +const FRESH_ATTEMPTS: usize = 3; + +pub struct Setup { + /// `s3:///`. + pub url: String, + /// `name=value` bucket options (see `S3::new`). + pub options: Vec, + /// This instance's identity, fresh per process. + pub node: String, + /// Where peers reach this instance: `host:port`. + pub address: String, + pub ttl_ms: u64, + /// The cluster secret: MACs every peer call. + pub key: Vec, +} + +/// Where a Durable Object runs. +pub enum Route { + /// Here, at this ownership epoch: take it over with `restore_actor`. + Owned(u64), + /// On another instance. `stale`: the record names a node that looks + /// dead, so the caller should `refresh_route` in the background. + Remote { + node: String, + address: String, + stale: bool, + }, +} + +pub struct Cluster { + registry: Rc, + ownership: Ownership, + owners: Owners, + /// Claims won but not yet taken over, for `restore_actor`. + claims: RefCell>, + bucket: Bucket, + key: Vec, + lease: RefCell, +} + +impl Cluster { + pub fn new(setup: Setup) -> Result> { + let options: Vec<_> = setup + .options + .iter() + .filter_map(|option| option.split_once('=')) + .map(|(key, value)| (key.to_owned(), value.to_owned())) + .collect(); + let (s3, prefix) = S3::new(&setup.url, &options)?; + let bucket = Bucket::new(s3, &format!("{prefix}/{FORMAT}")); + let registry = Rc::new(Registry::new( + bucket.clone(), + &setup.node, + &setup.address, + setup.ttl_ms, + )); + Ok(Rc::new(Self { + owners: Owners::new(bucket.clone()), + ownership: Ownership::new(bucket.clone(), registry.clone()), + bucket, + registry, + claims: RefCell::new(HashMap::new()), + key: setup.key, + lease: RefCell::new(Lease::default()), + })) + } + + pub fn node(&self) -> &str { + self.registry.node() + } + + pub fn key(&self) -> &[u8] { + &self.key + } + + pub async fn renew(&self, now_ms: u64) -> Result { + self.registry.renew(now_ms).await + } + + /// How far this node's wall clock is outside the bucket's, in ms; 0 + /// before any response. + pub fn clock_skew_ms(&self) -> i64 { + self.bucket.clock_skew_ms().unwrap_or(0) + } + + pub async fn deregister(&self) -> Result<()> { + self.registry.deregister().await + } + + /// Live peers' records, as the bucket lists them now. + pub async fn peers(&self, now_ms: u64) -> Result> { + self.registry.peers(now_ms).await + } + + pub async fn acquire_or_route(&self, ns: &str, actor: &str, now_ms: u64) -> Result { + let mut claim = self.ownership.acquire_or_route(ns, actor, now_ms).await?; + // Named open here, but not open here. Evicted here with the bucket + // holding the epoch: resume it (`take_over`); a stale owner that + // writes on cannot acknowledge, since every acknowledgement rereads + // own.json. Otherwise, dropped after a failure or evicted before its + // upload completed: recover it at the next epoch. + if let Claim::Owned(owned) = &claim + && owned.recover.is_none() + && !self.owners.holds(ns, actor) + && self.owners.dormant(ns, actor) != Some(owned.epoch) + { + let next = self.ownership.advance(ns, actor, owned.epoch).await?; + // Changed meanwhile, by another claim here: claim what it left. + claim = match next { + Some(next) => Claim::Owned(next), + None => self.ownership.acquire_or_route(ns, actor, now_ms).await?, + }; + } + // A new object opens here, where losing the race still routes. + for _ in 0..FRESH_ATTEMPTS { + let Claim::Owned(owned) = &claim else { break }; + if owned.recover.is_none_or(|from| from != 0) + || self + .owners + .open_fresh(&self.ownership, ns, actor, owned) + .await? + { + break; + } + claim = self.ownership.acquire_or_route(ns, actor, now_ms).await?; + } + Ok(match claim { + Claim::Owned(owned) => { + let epoch = owned.epoch; + let key = format!("{ns}/{actor}"); + self.claims.borrow_mut().insert(key, owned); + Route::Owned(epoch) + } + Claim::Route { + node, + address, + stale, + } => Route::Remote { + node, + address, + stale, + }, + }) + } + + /// Take a claimed object over: recover what earlier epochs committed + /// into the database at `path`, and open the claimed epoch. + pub async fn restore_actor(&self, ns: &str, actor: &str, epoch: u64, path: &str) -> Result<()> { + let claim = self.claims.borrow_mut().remove(&format!("{ns}/{actor}")); + let owned = claim.filter(|owned| owned.epoch == epoch).ok_or_else(|| { + crate::Error::Replica(format!("{ns}/{actor}: no claim at epoch {epoch}")) + })?; + self.owners + .take_over(&self.ownership, ns, actor, &owned, Path::new(path)) + .await + } + + /// Make an object's captured commits durable in the bucket. + pub async fn sync_actor(&self, ns: &str, actor: &str) -> Result<()> { + self.owners.commit(&self.ownership, ns, actor).await + } + + /// Stop capturing an object that left this node. + pub fn close_actor(&self, ns: &str, actor: &str) { + self.owners.retire(ns, actor); + } + + /// An owned object's database was deleted, before anything writes a + /// new one. + pub fn database_reset(&self, ns: &str, actor: &str) { + self.owners.database_reset(ns, actor); + } + + /// A write SQLite made to a WAL file in namespace `ns`'s storage. + pub fn wal_write(&self, ns: &str, file: &str, offset: u64, data: &[u8]) { + // `.sqlite-wal`, under whatever directory SQLite was given. + let name = file.rsplit('/').next().unwrap_or(file); + if let Some(actor) = name.strip_suffix(".sqlite-wal") { + self.owners.wal_write(ns, actor, offset, data); + } + } + + /// Compact owned objects' chains in the bucket. + pub async fn tier(&self) { + self.owners.tier().await; + } + + /// Keep an object workerd evicted, uploading what the bucket lacks. + pub async fn evict_actor(&self, ns: &str, actor: &str, epoch: u64) { + self.owners.evict(ns, actor, epoch).await; + } + + pub async fn refresh_route(&self, ns: &str, actor: &str, now_ms: u64) -> Result<()> { + self.ownership.refresh_route(ns, actor, now_ms).await + } + + /// The MAC a peer call carries. + pub fn mac(&self, data: &[u8]) -> Vec { + hmac_sha256(&self.key, data) + } +} + +impl Cluster { + /// Keeps this node's lease, until the node must stop + /// serving: answers why. Runs for the life of the process. + pub async fn maintain(&self) -> String { + loop { + if self.lease.borrow().draining { + std::future::pending::<()>().await; + } + let observed = lease::Observed { + now_ms: now_ms(), + skew_ms: self.clock_skew_ms(), + }; + let mut step = self.lease.borrow().before_renew(&observed); + if step == Step::Renew { + let renew = timeout(ms(lease::RENEW_TIMEOUT_MS), self.renew(now_ms())); + let result = match renew.await { + Ok(Ok(until)) => Ok(until), + Ok(Err(e)) => Err(e.to_string()), + Err(_) => Err("renewal timed out".into()), + }; + step = self.lease.borrow_mut().renewed(now_ms(), result); + } + let delay = match step { + Step::Fence(reason) => return reason, + Step::Retry(reason) => { + tracing::warn!("renewing the cluster lease failed; retrying: {reason}"); + lease::RETRY_MS + } + Step::Renew => lease::INTERVAL_MS, + }; + // A stale listing only delays routing to a new peer: its + // failure is logged, never fatal. + match timeout(ms(lease::LIST_TIMEOUT_MS), self.peers(now_ms())).await { + Ok(Ok(_)) => {} + Ok(Err(e)) => tracing::warn!("listing the cluster failed: {e}"), + Err(_) => tracing::warn!("listing the cluster timed out"), + } + tokio::time::sleep(ms(delay)).await; + } + } + + /// Compacts owned objects' chains. Never returns. + pub async fn tier_loop(&self) { + loop { + tokio::time::sleep(TIER_INTERVAL).await; + self.tier().await; + } + } + + /// Stops claiming before the claim reaches the bucket, and parks the + /// lease loop; then `deregister`. + pub fn start_draining(&self) { + self.lease.borrow_mut().draining = true; + self.ownership.may_claim.set(false); + } + + /// The first renewal, before the node serves. + pub async fn join(&self) -> Result<()> { + check_conditional_writes(&self.bucket, self.node()).await?; + let until = self.renew(now_ms()).await?; + self.lease.borrow_mut().renewed(now_ms(), Ok(until)); + Ok(()) + } + + /// Whether this node's lease holds now: an actor database opens only + /// while it does. + pub fn lease_holds(&self) -> bool { + self.lease.borrow().holds(now_ms()) + } +} + +/// How often owned objects' chains are checked for compaction. +const TIER_INTERVAL: Duration = Duration::from_secs(2); + +const fn ms(ms: u64) -> Duration { + Duration::from_millis(ms) +} + +pub fn now_ms() -> u64 { + let since = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default(); + u64::try_from(since.as_millis()).unwrap_or(u64::MAX) +} + +/// Refuse a store that ignores write preconditions: fencing rests on +/// them. GCS's S3 API, for one, ignores `If-Match` on a PUT. +async fn check_conditional_writes(bucket: &Bucket, node: &str) -> Result<()> { + let key = format!("nodes/{node}.probe"); + let stale = Guard::Match(Version("\"0\"".into())); + let created = bucket.put_if(&key, vec![], Guard::Absent).await?; + let again = bucket.put_if(&key, vec![], Guard::Absent).await?; + let replaced = bucket.put_if(&key, vec![], stale).await?; + bucket.delete(&key).await?; + if created.is_none() || again.is_some() || replaced.is_some() { + return Err(Error::Config( + "the bucket ignores conditional writes (If-None-Match, If-Match)".into(), + )); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use futures::executor::block_on; + + use super::check_conditional_writes; + use crate::Bucket; + use crate::bucket::memory::Memory; + + #[test] + fn a_bucket_that_ignores_preconditions_is_refused() { + block_on(async { + assert!( + check_conditional_writes(&Bucket::memory(""), "n") + .await + .is_ok() + ); + let lax = Memory::default(); + lax.ignores_guards.set(true); + let lax = Bucket::new(lax, ""); + assert!(check_conditional_writes(&lax, "n").await.is_err()); + }); + } +} diff --git a/src/workerd/server/cluster/http.rs b/src/workerd/server/cluster/http.rs new file mode 100644 index 00000000000..4e88c5894be --- /dev/null +++ b/src/workerd/server/cluster/http.rs @@ -0,0 +1,287 @@ +// Copyright (c) 2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +//! The bucket's HTTP client: hyper's pooled HTTP/1.1 client over TCP, with +//! rustls and the platform's roots for `https` endpoints. + +use std::future::Future; +use std::future::poll_fn; +use std::io; +use std::io::Read; +use std::io::Write; +use std::path::Path; +use std::path::PathBuf; +use std::pin::Pin; +use std::sync::Arc; +use std::task::Context; +use std::task::Poll; +use std::time::Duration; + +use http::Uri; +use http_body::Body as _; +use http_body::Frame; +use hyper_util::client::legacy; +use hyper_util::client::legacy::connect::Connected; +use hyper_util::client::legacy::connect::Connection; +use hyper_util::rt::TokioExecutor; +use hyper_util::rt::TokioIo; +use rustls::ClientConfig; +use rustls::pki_types::ServerName; +use rustls_platform_verifier::ConfigVerifierExt; +use tokio::io::AsyncRead; +use tokio::io::AsyncWrite; +use tokio::io::ReadBuf; +use tokio::net::TcpStream; +use tokio_rustls::TlsConnector; +use tokio_rustls::client::TlsStream; + +/// A bucket response: only the parts the S3 client reads. +pub struct HttpResponse { + pub status: u16, + pub etag: String, + /// The `Date` header, the bucket's clock. + pub date: String, + /// Empty when a 2xx body went to a file. + pub body: Vec, +} + +/// What a request sends: bytes, or a file read in chunks on the blocking +/// pool. +#[derive(Clone)] +pub enum Payload { + Bytes(bytes::Bytes), + File(PathBuf), +} + +/// The chunks a file is sent and received in. +const CHUNK: usize = 256 << 10; + +pub struct Http { + client: legacy::Client, +} + +impl Http { + pub fn new() -> Result { + let tls = Arc::new(ClientConfig::with_platform_verifier()?); + let client = legacy::Client::builder(TokioExecutor::new()).build(Connector(tls)); + Ok(Self { client }) + } + + /// `headers` alternates names and values. A 2xx body goes to the file + /// `to`, if any. `head` bounds the wait for the response's head, which + /// includes sending the body; `idle`, each wait for the body's next + /// chunk. + pub async fn send( + &self, + method: String, + url: String, + headers: Vec, + payload: Payload, + to: Option<&Path>, + (head, idle): (Duration, Duration), + ) -> Result { + let mut request = http::Request::builder().method(method.as_str()).uri(url); + for pair in headers.chunks(2) { + request = request.header(&pair[0], &pair[1]); + } + let body = match payload { + Payload::Bytes(bytes) => Body::Bytes(Some(bytes)), + Payload::File(path) => Body::file(&path).map_err(|e| e.to_string())?, + }; + let request = request.body(body).map_err(|e| e.to_string())?; + let response = tokio::time::timeout(head, self.client.request(request)) + .await + .map_err(|_| "timed out".to_owned())? + .map_err(|e| e.to_string())?; + let header = |name| { + let value = response.headers().get(name); + value.and_then(|v| v.to_str().ok()).unwrap_or("").to_owned() + }; + let (status, etag, date) = (response.status().as_u16(), header("etag"), header("date")); + let mut incoming = response.into_body(); + let to = to + .filter(|_| (200..300).contains(&status)) + .map(Path::to_owned); + let (tx, mut rx) = tokio::sync::mpsc::channel::(4); + let writer = to.map(|path| { + tokio::task::spawn_blocking(move || -> io::Result<()> { + let mut file = io::BufWriter::new(std::fs::File::create(path)?); + while let Some(chunk) = rx.blocking_recv() { + file.write_all(&chunk)?; + } + file.flush() + }) + }); + let mut body = Vec::new(); + loop { + let frame = poll_fn(|cx| Pin::new(&mut incoming).poll_frame(cx)); + let frame = tokio::time::timeout(idle, frame) + .await + .map_err(|_| "timed out reading the body".to_owned())?; + let Some(frame) = frame else { break }; + let chunk = frame + .map_err(|e| e.to_string())? + .into_data() + .unwrap_or_default(); + if writer.is_none() { + body.extend(chunk); + } else if tx.send(chunk).await.is_err() { + break; + } + } + drop(tx); + if let Some(writer) = writer { + writer + .await + .map_err(|e| e.to_string())? + .map_err(|e| e.to_string())?; + } + Ok(HttpResponse { + status, + etag, + date, + body, + }) + } +} + +/// A request body: bytes in one piece, or a file's chunks as a blocking +/// task reads them. Either way its exact size is known, which S3 needs. +enum Body { + Bytes(Option), + File { + chunks: tokio::sync::mpsc::Receiver>, + size: u64, + }, +} + +impl Body { + fn file(path: &Path) -> io::Result { + let mut file = std::fs::File::open(path)?; + let size = file.metadata()?.len(); + let (tx, chunks) = tokio::sync::mpsc::channel(4); + tokio::task::spawn_blocking(move || { + loop { + let mut chunk = vec![0; CHUNK]; + match file.read(&mut chunk) { + Ok(0) => return, + Ok(n) => chunk.truncate(n), + Err(e) => return drop(tx.blocking_send(Err(e))), + } + if tx.blocking_send(Ok(chunk.into())).is_err() { + return; + } + } + }); + Ok(Self::File { chunks, size }) + } +} + +impl http_body::Body for Body { + type Data = bytes::Bytes; + type Error = io::Error; + + fn poll_frame( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + ) -> Poll, Self::Error>>> { + match self.get_mut() { + Self::Bytes(bytes) => Poll::Ready(bytes.take().map(|data| Ok(Frame::data(data)))), + Self::File { chunks, .. } => { + chunks.poll_recv(cx).map(|c| c.map(|c| c.map(Frame::data))) + } + } + } + + fn size_hint(&self) -> http_body::SizeHint { + match self { + Self::Bytes(bytes) => { + http_body::SizeHint::with_exact(bytes.as_ref().map_or(0, |b| b.len() as u64)) + } + Self::File { size, .. } => http_body::SizeHint::with_exact(*size), + } + } +} + +#[derive(Clone)] +struct Connector(Arc); + +impl tower_service::Service for Connector { + type Response = TokioIo; + type Error = io::Error; + type Future = Pin> + Send>>; + + fn poll_ready(&mut self, _: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + + fn call(&mut self, uri: Uri) -> Self::Future { + let tls = self.0.clone(); + Box::pin(async move { + let invalid = || io::Error::new(io::ErrorKind::InvalidInput, uri.to_string()); + let host = uri.host().ok_or_else(invalid)?.to_owned(); + let https = uri.scheme_str() == Some("https"); + let port = uri.port_u16().unwrap_or(if https { 443 } else { 80 }); + let tcp = TcpStream::connect((host.as_str(), port)).await?; + tcp.set_nodelay(true)?; + if !https { + return Ok(TokioIo::new(Stream::Tcp(tcp))); + } + let name = ServerName::try_from(host).map_err(|_| invalid())?; + let tls = TlsConnector::from(tls).connect(name, tcp).await?; + Ok(TokioIo::new(Stream::Tls(Box::new(tls)))) + }) + } +} + +enum Stream { + Tcp(TcpStream), + Tls(Box>), +} + +impl Connection for Stream { + fn connected(&self) -> Connected { + Connected::new() + } +} + +impl AsyncRead for Stream { + fn poll_read( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &mut ReadBuf<'_>, + ) -> Poll> { + match self.get_mut() { + Self::Tcp(s) => Pin::new(s).poll_read(cx, buf), + Self::Tls(s) => Pin::new(s).poll_read(cx, buf), + } + } +} + +impl AsyncWrite for Stream { + fn poll_write( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &[u8], + ) -> Poll> { + match self.get_mut() { + Self::Tcp(s) => Pin::new(s).poll_write(cx, buf), + Self::Tls(s) => Pin::new(s).poll_write(cx, buf), + } + } + + fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + match self.get_mut() { + Self::Tcp(s) => Pin::new(s).poll_flush(cx), + Self::Tls(s) => Pin::new(s).poll_flush(cx), + } + } + + fn poll_shutdown(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + match self.get_mut() { + Self::Tcp(s) => Pin::new(s).poll_shutdown(cx), + Self::Tls(s) => Pin::new(s).poll_shutdown(cx), + } + } +} diff --git a/src/workerd/server/cluster/lib.rs b/src/workerd/server/cluster/lib.rs new file mode 100644 index 00000000000..c0ecec6d31a --- /dev/null +++ b/src/workerd/server/cluster/lib.rs @@ -0,0 +1,111 @@ +// Copyright (c) 2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +//! A Durable Object cluster whose shared state is an S3-compatible bucket +//! (S3, R2, `MinIO`) instead of a shared filesystem. +//! +//! Everything lives under `v1/` in the configured prefix, so a later format +//! can tell an old bucket from its own. Two records: +//! +//! - `nodes/.json`: one per running instance, with its address and a +//! lease expiry the instance renews. A peer is live while its lease is. +//! - `do///own.json`: one per Durable Object, naming its +//! owner and an ownership epoch. A claim is a conditional write; taking an +//! object over from an owner whose lease expired bumps the epoch, and +//! everything the new owner writes is keyed by that epoch, under +//! `do///ltx/e/`, so a late write from the old +//! owner lands where nothing reads it. +//! +//! The clock is an argument everywhere (`now_ms`), so a test decides time. + +#![deny( + clippy::expect_used, + clippy::panic, + clippy::unreachable, + clippy::unwrap_used +)] + +mod bucket; +mod capture; +pub mod cluster; +mod http; +mod ltx; +#[cfg(test)] +mod ltx_tests; +mod owner; +mod ownership; +mod registry; +pub mod s3; + +pub use bucket::Bucket; +pub use bucket::Guard; +pub use bucket::Store; +pub use bucket::Version; +pub use cluster::Cluster; +pub use cluster::Route; +pub use cluster::Setup; +pub use cluster::now_ms; +pub use owner::Owners; +pub use ownership::Claim; +pub use ownership::Ownership; +pub use registry::NodeRecord; +pub use registry::Registry; + +#[derive(Debug, thiserror::Error)] +pub enum Error { + /// The bucket URL or an option is missing or malformed. + #[error("bucket config: {0}")] + Config(String), + /// A claim or a commit's capture could not proceed. + #[error("replica: {0}")] + Replica(String), + /// A bucket request failed or answered with an unexpected status. + #[error("bucket: {0}")] + Http(String), + #[error("record: {0}")] + Codec(#[from] serde_json::Error), + /// Another writer kept winning the conditional write. + #[error("{0} is contended")] + Contended(String), + /// Someone else rewrote this node's own record: this process no longer + /// holds its identity and must stop serving. + #[error("node record {0} was rewritten by another writer")] + Fenced(String), + /// Capturing, uploading or restoring a database failed. + #[error("replication: {0}")] + Replication(#[from] crate::ltx::Error), +} + +pub type Result = std::result::Result; + +/// An ownership epoch: each takeover of an object bumps it. +pub type Epoch = u64; +/// A transaction id within one epoch's chain, from 1. +pub type Txid = u64; + +pub(crate) fn sha256(data: &[u8]) -> Vec { + ring::digest::digest(&ring::digest::SHA256, data) + .as_ref() + .to_vec() +} + +pub(crate) fn hmac_sha256(key: &[u8], data: &[u8]) -> Vec { + let key = ring::hmac::Key::new(ring::hmac::HMAC_SHA256, key); + ring::hmac::sign(&key, data).as_ref().to_vec() +} + +pub fn hex(bytes: &[u8]) -> String { + use std::fmt::Write; + bytes.iter().fold(String::new(), |mut out, byte| { + let _ = write!(out, "{byte:02x}"); + out + }) +} + +pub fn unhex(text: &str) -> Option> { + (0..text.len()) + .step_by(2) + .map(|i| u8::from_str_radix(text.get(i..i + 2)?, 16).ok()) + .collect() +} diff --git a/src/workerd/server/cluster/logic/BUILD.bazel b/src/workerd/server/cluster/logic/BUILD.bazel new file mode 100644 index 00000000000..7bdc8bd8448 --- /dev/null +++ b/src/workerd/server/cluster/logic/BUILD.bazel @@ -0,0 +1,8 @@ +load("//:build/wd_rust_crate.bzl", "wd_rust_crate") + +# The bucket cluster's decision core. std only: nothing here may do I/O or +# read a clock, so it has no dependencies to do them with. +wd_rust_crate( + name = "cluster-logic", + visibility = ["//src/workerd/server:__subpackages__"], +) diff --git a/src/workerd/server/cluster/logic/lease.rs b/src/workerd/server/cluster/logic/lease.rs new file mode 100644 index 00000000000..660f920f99e --- /dev/null +++ b/src/workerd/server/cluster/logic/lease.rs @@ -0,0 +1,145 @@ +// Copyright (c) 2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +//! An instance's lease in the bucket: when to renew and when to stop +//! serving (fence). +//! +//! The executor runs one round every `INTERVAL_MS` (`RETRY_MS` after a +//! failed renewal): it asks `before_renew`, renews if told to and reports +//! the result to `renewed`. + +/// A node's lease. A peer whose record has not been renewed for a full +/// lease is dead, so the lease spans several rounds. +pub const LEASE_MS: u64 = 20_000; +/// Renewals every 3 s, each bounded at 4 s, a failed one retried every +/// second. +/// +/// A node rides out about 8 s of bucket trouble and fences while its +/// lease holds. R2 stalls writes ~5.5 s at every 5-minute mark; a 2 s +/// bound cancelled them and fenced every node. +pub const INTERVAL_MS: u64 = 3_000; +pub const RENEW_TIMEOUT_MS: u64 = 4_000; +pub const LIST_TIMEOUT_MS: u64 = 2_000; +pub const RETRY_MS: u64 = 1_000; +/// Peers judge this node's lease by their own clocks; one off from the +/// bucket's clock by more than this exits. +pub const MAX_SKEW_MS: i64 = 2_000; +/// Peers judge a lease by their own clocks, each within `MAX_SKEW_MS` of +/// the bucket's, so the owner stops serving this long before it expires. +const SAFETY_MS: u64 = 2 * MAX_SKEW_MS as u64; + +/// What the executor observed before a round's renewal. +pub struct Observed { + pub now_ms: u64, + /// This node's clock less the bucket's. + pub skew_ms: i64, +} + +#[derive(Debug, PartialEq, Eq)] +pub enum Step { + /// Renew now. + Renew, + /// Stop serving: the process exits before peers may take over. + Fence(String), + /// Try again after `RETRY_MS`; the lease still holds. + Retry(String), +} + +#[derive(Default)] +pub struct Lease { + /// The lease expires at this time; 0 before the first renewal. + until_ms: u64, + pub draining: bool, +} + +impl Lease { + pub fn before_renew(&self, o: &Observed) -> Step { + if o.skew_ms.abs() > MAX_SKEW_MS { + return Step::Fence(format!("clock off from the bucket's by {} ms", o.skew_ms)); + } + // A node stalled past its lease (a paused process, a blocked loop) + // may have been taken over: it must not renew and serve on. + if self.until_ms != 0 && !self.holds(o.now_ms + RENEW_TIMEOUT_MS) { + return Step::Fence("the lease lapsed before it could be renewed".into()); + } + Step::Renew + } + + /// The renewal's outcome: the new expiry, or why it failed. `Renew` + /// here means the round succeeded. + pub fn renewed(&mut self, now_ms: u64, result: Result) -> Step { + match result { + Ok(until_ms) => { + self.until_ms = until_ms; + Step::Renew + } + // Fence while the lease still holds, so the node is gone before + // a peer may take over. + Err(e) if !self.holds(now_ms + RENEW_TIMEOUT_MS) => Step::Fence(e), + Err(e) => Step::Retry(e), + } + } + + /// Whether the lease holds at `now_ms`, with peers' clocks allowed + /// for: an actor serves only while it does. + pub fn holds(&self, now_ms: u64) -> bool { + now_ms + SAFETY_MS < self.until_ms + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn seen(now_ms: u64) -> Observed { + Observed { now_ms, skew_ms: 0 } + } + + #[test] + fn renews_and_holds_until_the_safety_margin() { + let mut lease = Lease::default(); + assert_eq!(lease.before_renew(&seen(0)), Step::Renew); + assert_eq!(lease.renewed(0, Ok(LEASE_MS)), Step::Renew); + let last = LEASE_MS - SAFETY_MS; + assert!(lease.holds(last - 1) && !lease.holds(last)); + } + + #[test] + fn a_failed_renewal_retries_until_the_lease_nearly_lapses() { + let mut lease = Lease::default(); + lease.renewed(0, Ok(LEASE_MS)); + assert!(matches!( + lease.renewed(5_000, Err("x".into())), + Step::Retry(_) + )); + let last = LEASE_MS - SAFETY_MS - RENEW_TIMEOUT_MS; + assert!(matches!( + lease.renewed(last - 1, Err("x".into())), + Step::Retry(_) + )); + assert!(matches!( + lease.renewed(last, Err("x".into())), + Step::Fence(_) + )); + } + + #[test] + fn a_node_stalled_past_its_lease_fences_instead_of_renewing() { + let mut lease = Lease::default(); + lease.renewed(0, Ok(LEASE_MS)); + assert_eq!(lease.before_renew(&seen(INTERVAL_MS)), Step::Renew); + let step = lease.before_renew(&seen(LEASE_MS + 1)); + assert!(matches!(step, Step::Fence(_))); + } + + #[test] + fn skew_fences_at_once() { + let lease = Lease::default(); + let o = Observed { + skew_ms: MAX_SKEW_MS + 1, + ..seen(0) + }; + assert!(matches!(lease.before_renew(&o), Step::Fence(_))); + } +} diff --git a/src/workerd/server/cluster/logic/lib.rs b/src/workerd/server/cluster/logic/lib.rs new file mode 100644 index 00000000000..9ba432a6a55 --- /dev/null +++ b/src/workerd/server/cluster/logic/lib.rs @@ -0,0 +1,16 @@ +// Copyright (c) 2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +//! The bucket cluster's decisions, with no I/O: no async, clocks, +//! randomness, locks or dependencies beyond std. Time is an argument. The +//! executor (`cluster`) performs what these types decide. + +#![deny( + clippy::expect_used, + clippy::panic, + clippy::unreachable, + clippy::unwrap_used +)] + +pub mod lease; diff --git a/src/workerd/server/cluster/ltx/client/bucket.rs b/src/workerd/server/cluster/ltx/client/bucket.rs new file mode 100644 index 00000000000..60fbcabede0 --- /dev/null +++ b/src/workerd/server/cluster/ltx/client/bucket.rs @@ -0,0 +1,136 @@ +//! client::bucket — the `ReplicaClient` over the cluster's `Bucket`. +//! +//! Ported (behavior only) from litestream@v0.5.11 `s3/replica_client.go` by +//! way of celld's `celld-ltx`, keeping the invariants from the upstream +//! conformance suite: +//! +//! * key scheme `{path}/{level:04x}/{min}-{max}.ltx` +//! (s3/replica_client.go:629, 677, 1040-1042); +//! * list + seek-skip on `min_txid < seek`, ascending TXID order +//! (s3/replica_client.go:1530-1533); +//! * `NoSuchKey` → `os.ErrNotExist` error mapping +//! (s3/replica_client.go:647-649, 1662-1668). +//! +//! Every file is one PUT. S3 takes up to 5 GiB that way, and this port has no +//! compactor to produce oversized files, so the multipart path stays in celld. + +use std::io::Read; +use std::path::Path; + +use crate::Bucket; +use crate::ltx::TXID; +use crate::ltx::client::ReplicaClient; +use crate::ltx::error::Error; +use crate::ltx::error::Result; +use crate::ltx::ltx::FileInfo; +use crate::ltx::ltx::{self}; +use crate::s3::format_rfc3339_nano; + +/// The standard Litestream S3 metadata key for an LTX header timestamp. +const METADATA_KEY_TIMESTAMP: &str = "litestream-timestamp"; + +/// A replica rooted at the bucket's prefix. +#[derive(Clone)] +pub struct BucketClient { + bucket: Bucket, +} + +impl BucketClient { + pub fn new(bucket: Bucket) -> Self { + Self { bucket } + } + + /// Delete a file a compaction replaced. + pub async fn delete_ltx_file(&self, level: i32, min_txid: TXID, max_txid: TXID) -> Result<()> { + self.bucket + .delete(&Self::ltx_key(level, min_txid, max_txid)) + .await + .map_err(other) + } + + /// `{level:04x}/{min}-{max}.ltx` (s3/replica_client.go:629, 677, 1040-1042). + fn ltx_key(level: i32, min_txid: TXID, max_txid: TXID) -> String { + format!("{level:04x}/{}", ltx::format_filename(min_txid, max_txid)) + } +} + +fn other(error: crate::Error) -> Error { + Error::Other(error.to_string().into()) +} + +fn not_found(key: &str) -> Error { + Error::Io(std::io::Error::new( + std::io::ErrorKind::NotFound, + format!("replica: get object {key}: not found"), + )) +} + +#[async_trait::async_trait(?Send)] +impl ReplicaClient for BucketClient { + async fn ltx_files(&self, level: i32, seek: TXID) -> Result> { + let prefix = format!("{level:04x}/"); + let listed = self.bucket.list(&prefix).await.map_err(other)?; + let mut infos: Vec = listed + .into_iter() + .filter_map(|entry| { + let name = entry.key.rsplit('/').next()?; + let (min_txid, max_txid) = ltx::parse_filename(name).ok()?; + (min_txid >= seek).then(|| FileInfo { + level, + min_txid, + max_txid, + ..Default::default() + }) + }) + .collect(); + infos.sort_by_key(|info| (info.level, info.min_txid.0, info.max_txid.0)); + Ok(infos) + } + + async fn open_ltx_file( + &self, + level: i32, + min_txid: TXID, + max_txid: TXID, + to: &Path, + ) -> Result<()> { + let key = Self::ltx_key(level, min_txid, max_txid); + match self.bucket.get_file(&key, to).await.map_err(other)? { + true => Ok(()), + false => Err(not_found(&key)), + } + } + + async fn write_ltx_file( + &self, + level: i32, + min_txid: TXID, + max_txid: TXID, + from: &Path, + ) -> Result { + // The LTX header timestamp goes in the standard Litestream metadata, + // so Litestream can restore by time without reading every file. + let mut file = std::fs::File::open(from)?; + let mut bytes = [0; ltx::HEADER_SIZE]; + file.read_exact(&mut bytes)?; + let header = ltx::Header::parse(&bytes)?; + let size = file.metadata()?.len(); + let key = Self::ltx_key(level, min_txid, max_txid); + let stamp = format_rfc3339_nano(header.timestamp); + self.bucket + .put_file(&key, from, &[(METADATA_KEY_TIMESTAMP, stamp)]) + .await + .map_err(|e| Error::Other(format!("replica: upload to {key}: {e}").into()))?; + Ok(FileInfo { + level, + min_txid, + max_txid, + size: size as i64, + created_at: Some( + std::time::UNIX_EPOCH + + std::time::Duration::from_millis(header.timestamp.max(0) as u64), + ), + ..Default::default() + }) + } +} diff --git a/src/workerd/server/cluster/ltx/client/mod.rs b/src/workerd/server/cluster/ltx/client/mod.rs new file mode 100644 index 00000000000..f4eba5c368c --- /dev/null +++ b/src/workerd/server/cluster/ltx/client/mod.rs @@ -0,0 +1,48 @@ +//! client — the `ReplicaClient` trait. +//! +//! Ported from litestream@v0.5.11 `replica_client.go`. The trait is the storage +//! abstraction that every backend implements. +//! +//! # I/O +//! Go's `ReplicaClient` uses `io.Reader`/`io.ReadCloser`. LTX files go +//! between the store and files on disk, so no file is held in memory. + +use std::path::Path; + +use async_trait::async_trait; + +use crate::ltx::TXID; +use crate::ltx::error::Result; +use crate::ltx::ltx::FileInfo; + +pub mod bucket; + +/// Client for reading and writing LTX files on a replica backend. +/// +/// Ported from the `ReplicaClient` interface (replica_client.go:19-51). Methods +/// take a compaction `level`: 0 for uploads, `SNAPSHOT_LEVEL` for snapshots. +#[async_trait(?Send)] +pub trait ReplicaClient { + /// Returns all LTX files for `level`, sorted ascending by `min_txid`, that + /// start at or after `seek`. + async fn ltx_files(&self, level: i32, seek: TXID) -> Result>; + + /// Writes an LTX file to the file `to`. Returns an + /// `io::ErrorKind::NotFound` error (wrapped) if it does not exist. + async fn open_ltx_file( + &self, + level: i32, + min_txid: TXID, + max_txid: TXID, + to: &Path, + ) -> Result<()>; + + /// Writes the LTX file `from` to the replica and returns its metadata. + async fn write_ltx_file( + &self, + level: i32, + min_txid: TXID, + max_txid: TXID, + from: &Path, + ) -> Result; +} diff --git a/src/workerd/server/cluster/ltx/codec.rs b/src/workerd/server/cluster/ltx/codec.rs new file mode 100644 index 00000000000..1119ca9bd4a --- /dev/null +++ b/src/workerd/server/cluster/ltx/codec.rs @@ -0,0 +1,354 @@ +//! Streaming LTX encoder and decoder. + +use std::collections::BTreeMap; +use std::io::Read; +use std::io::Write; + +use crate::ltx::CHECKSUM_FLAG; +use crate::ltx::error::Error; +use crate::ltx::error::Result; +use crate::ltx::ltx::CHECKSUM_SIZE; +use crate::ltx::ltx::Crc64; +use crate::ltx::ltx::HEADER_SIZE; +use crate::ltx::ltx::Header; +use crate::ltx::ltx::PAGE_HEADER_FLAG_SIZE; +use crate::ltx::ltx::PAGE_HEADER_SIZE; +use crate::ltx::ltx::PageHeader; +use crate::ltx::ltx::TRAILER_SIZE; +use crate::ltx::ltx::Trailer; +use crate::ltx::ltx::checksum_page; +use crate::ltx::ltx::lock_pgno; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum DecoderState { + Header, + Pages, + Close, + Closed, +} + +pub(crate) struct Decoder { + reader: R, + state: DecoderState, + pub(crate) header: Header, + pub(crate) trailer: Trailer, + hash: Crc64, + rolling_checksum: u64, +} + +impl Decoder { + pub(crate) fn new(reader: R) -> Self { + Self { + reader, + state: DecoderState::Header, + header: Header::default(), + trailer: Trailer::default(), + hash: Crc64::new(), + rolling_checksum: 0, + } + } + + pub(crate) fn decode_header(&mut self) -> Result<()> { + if self.state != DecoderState::Header { + return Err(Error::LTXCorrupted); + } + let mut bytes = [0; HEADER_SIZE]; + self.reader.read_exact(&mut bytes)?; + self.header = Header::parse(&bytes)?; + self.header.validate()?; + self.hash.update(&bytes); + if !self.header.no_checksum() { + self.rolling_checksum = CHECKSUM_FLAG; + } + self.state = DecoderState::Pages; + Ok(()) + } + + pub(crate) fn decode_page(&mut self, data: &mut [u8]) -> Result> { + if self.state == DecoderState::Close { + return Ok(None); + } + if self.state != DecoderState::Pages || data.len() != self.header.page_size as usize { + return Err(Error::LTXCorrupted); + } + + let mut header_bytes = [0; PAGE_HEADER_SIZE]; + self.reader.read_exact(&mut header_bytes)?; + let page = PageHeader::parse(&header_bytes)?; + self.hash.update(&header_bytes); + if page.is_zero() { + self.state = DecoderState::Close; + return Ok(None); + } + page.validate()?; + + if page.flags & PAGE_HEADER_FLAG_SIZE != 0 { + let mut size_bytes = [0; 4]; + self.reader.read_exact(&mut size_bytes)?; + self.hash.update(&size_bytes); + let compressed_size = u32::from_be_bytes(size_bytes) as usize; + if compressed_size > lz4_flex::block::get_maximum_output_size(data.len()) { + return Err(Error::LTXCorrupted); + } + let mut compressed = vec![0; compressed_size]; + self.reader.read_exact(&mut compressed)?; + let n = lz4_flex::block::decompress_into(&compressed, data) + .map_err(|_| Error::LTXCorrupted)?; + if n != data.len() { + return Err(Error::LTXCorrupted); + } + } else { + let mut decoder = lz4_flex::frame::FrameDecoder::new(&mut self.reader); + decoder.read_exact(data).map_err(|_| Error::LTXCorrupted)?; + let mut extra = [0; 1]; + if decoder.read(&mut extra).map_err(|_| Error::LTXCorrupted)? != 0 { + return Err(Error::LTXCorrupted); + } + } + + self.hash.update(data); + if self.header.is_snapshot() + && !self.header.no_checksum() + && page.pgno != lock_pgno(self.header.page_size) + { + self.rolling_checksum = + CHECKSUM_FLAG | (self.rolling_checksum ^ checksum_page(page.pgno, data)); + } + Ok(Some(page)) + } + + pub(crate) fn close(&mut self) -> Result<()> { + if self.state == DecoderState::Closed { + return Ok(()); + } + if self.state != DecoderState::Close { + return Err(Error::LTXCorrupted); + } + + let mut remaining = Vec::new(); + self.reader.read_to_end(&mut remaining)?; + if remaining.len() < 8 + TRAILER_SIZE { + return Err(Error::LTXCorrupted); + } + + let trailer_offset = remaining.len() - TRAILER_SIZE; + let size_offset = trailer_offset - 8; + let index_size = u64::from_be_bytes( + remaining[size_offset..trailer_offset] + .try_into() + .map_err(|_| Error::LTXCorrupted)?, + ) as usize; + if index_size != size_offset { + return Err(Error::LTXCorrupted); + } + verify_page_index(&remaining[..size_offset])?; + + self.trailer = Trailer::parse(&remaining[trailer_offset..])?; + self.trailer.validate(self.header)?; + self.hash + .update(&remaining[..remaining.len() - CHECKSUM_SIZE]); + if CHECKSUM_FLAG | self.hash.sum64() != self.trailer.file_checksum { + return Err(Error::ChecksumMismatch); + } + if self.header.is_snapshot() + && !self.header.no_checksum() + && self.rolling_checksum != self.trailer.post_apply_checksum + { + return Err(Error::ChecksumMismatch); + } + + self.state = DecoderState::Closed; + Ok(()) + } +} + +pub(crate) struct Encoder { + pub(crate) writer: W, + pub(crate) header: Header, + pub(crate) trailer: Trailer, + hash: Crc64, + index: BTreeMap, + bytes_written: u64, + previous_page_number: u32, + header_written: bool, + closed: bool, +} + +impl Encoder { + pub(crate) fn new(writer: W) -> Self { + Self { + writer, + header: Header::default(), + trailer: Trailer::default(), + hash: Crc64::new(), + index: BTreeMap::new(), + bytes_written: 0, + previous_page_number: 0, + header_written: false, + closed: false, + } + } + + pub(crate) fn encode_header(&mut self, header: Header) -> Result<()> { + if self.header_written || self.closed { + return Err(Error::LTXCorrupted); + } + header.validate()?; + self.header = header; + let bytes = header.marshal(); + self.write_hashed(&bytes)?; + self.header_written = true; + Ok(()) + } + + pub(crate) fn encode_page(&mut self, page: PageHeader, data: &[u8]) -> Result<()> { + if !self.header_written + || self.closed + || page.pgno > self.header.commit + || data.len() != self.header.page_size as usize + { + return Err(Error::LTXCorrupted); + } + page.validate()?; + let lock_page = lock_pgno(self.header.page_size); + if page.pgno == lock_page { + return Err(Error::LTXCorrupted); + } + + if self.header.is_snapshot() { + if self.previous_page_number == 0 && page.pgno != 1 { + return Err(Error::LTXCorrupted); + } + let expected = if self.previous_page_number == lock_page - 1 { + self.previous_page_number + 2 + } else { + self.previous_page_number + 1 + }; + if self.previous_page_number != 0 && page.pgno != expected { + return Err(Error::LTXCorrupted); + } + } else if self.previous_page_number >= page.pgno { + return Err(Error::LTXCorrupted); + } + + let offset = self.bytes_written; + if page.flags != 0 { + return Err(Error::LTXCorrupted); + } + self.write_hashed(&page.marshal())?; + // Each page is an LZ4 frame of independent 64 KiB blocks with a + // content checksum, as the pre-v0.5.2 Go encoder writes; the decoder + // also reads v0.5.2's sized blocks. + let frame_info = lz4_flex::frame::FrameInfo::new() + .block_size(lz4_flex::frame::BlockSize::Max64KB) + .block_mode(lz4_flex::frame::BlockMode::Independent) + .content_checksum(true); + let mut encoder = lz4_flex::frame::FrameEncoder::with_frame_info(frame_info, Vec::new()); + encoder.write_all(data)?; + let compressed = encoder.finish().map_err(|_| Error::LTXCorrupted)?; + self.writer.write_all(&compressed)?; + self.bytes_written += compressed.len() as u64; + self.hash.update(data); + + self.previous_page_number = page.pgno; + self.index + .insert(page.pgno, (offset, self.bytes_written - offset)); + Ok(()) + } + + pub(crate) fn close(&mut self, post_apply_checksum: u64) -> Result<()> { + if !self.header_written || self.closed { + return Err(Error::LTXCorrupted); + } + + self.write_hashed(&[0; PAGE_HEADER_SIZE])?; + let index_offset = self.bytes_written; + let mut index_bytes = Vec::new(); + for (&page_number, &(offset, size)) in &self.index { + write_uvarint(&mut index_bytes, page_number as u64); + write_uvarint(&mut index_bytes, offset); + write_uvarint(&mut index_bytes, size); + } + write_uvarint(&mut index_bytes, 0); + self.write_hashed(&index_bytes)?; + self.write_hashed(&(self.bytes_written - index_offset).to_be_bytes())?; + + self.trailer.post_apply_checksum = post_apply_checksum; + self.hash.update(&post_apply_checksum.to_be_bytes()); + self.trailer.file_checksum = CHECKSUM_FLAG | self.hash.sum64(); + self.trailer.validate(self.header)?; + // An empty database's checksum, unless the file carries none. + if self.header.commit == 0 + && !self.header.no_checksum() + && post_apply_checksum != CHECKSUM_FLAG + { + return Err(Error::LTXCorrupted); + } + self.writer.write_all(&self.trailer.marshal())?; + self.bytes_written += TRAILER_SIZE as u64; + self.closed = true; + Ok(()) + } + + fn write_hashed(&mut self, bytes: &[u8]) -> Result<()> { + self.writer.write_all(bytes)?; + self.hash.update(bytes); + self.bytes_written += bytes.len() as u64; + Ok(()) + } +} + +fn verify_page_index(bytes: &[u8]) -> Result<()> { + decode_page_index(bytes).map(|_| ()) +} + +/// Decodes a page index into its `(pgno, frame_offset, frame_size)` entries. +/// The index is varint triples terminated by a `pgno == 0` marker; each entry +/// locates one page's whole frame (header plus LZ4 body) at `frame_offset` in +/// the file, `frame_size` bytes long. Paged restore builds its page map from +/// these, so it can range-read one frame without the object. +pub(crate) fn decode_page_index(bytes: &[u8]) -> Result> { + let mut position = 0; + let mut entries = Vec::new(); + loop { + let page_number = read_uvarint(bytes, &mut position)?; + if page_number == 0 { + break; + } + let offset = read_uvarint(bytes, &mut position)?; + let size = read_uvarint(bytes, &mut position)?; + entries.push((page_number as u32, offset, size)); + } + if position != bytes.len() { + return Err(Error::LTXCorrupted); + } + Ok(entries) +} + +fn read_uvarint(bytes: &[u8], position: &mut usize) -> Result { + let mut value = 0u64; + let mut shift = 0u32; + loop { + let byte = *bytes.get(*position).ok_or(Error::LTXCorrupted)?; + *position += 1; + if byte < 0x80 { + if shift >= 64 || (shift == 63 && byte > 1) { + return Err(Error::LTXCorrupted); + } + return Ok(value | (u64::from(byte) << shift)); + } + value |= u64::from(byte & 0x7f) << shift; + shift += 7; + if shift >= 70 { + return Err(Error::LTXCorrupted); + } + } +} + +fn write_uvarint(bytes: &mut Vec, mut value: u64) { + while value >= 0x80 { + bytes.push(value as u8 | 0x80); + value >>= 7; + } + bytes.push(value as u8); +} diff --git a/src/workerd/server/cluster/ltx/error.rs b/src/workerd/server/cluster/ltx/error.rs new file mode 100644 index 00000000000..8ba743ed720 --- /dev/null +++ b/src/workerd/server/cluster/ltx/error.rs @@ -0,0 +1,22 @@ +//! The LTX error type, after Litestream's sentinel errors (litestream.go:32-37). + +use thiserror::Error; + +pub type Result = std::result::Result; + +#[derive(Debug, Error)] +pub enum Error { + /// The replica's rolling checksum does not match what was expected. + #[error("invalid replica, checksum mismatch")] + ChecksumMismatch, + /// An LTX file's contents are corrupt (bad magic, truncated, bad CRC). + #[error("ltx file corrupted")] + LTXCorrupted, + /// No backup chain reaches the requested transaction. + #[error("transaction not available")] + TxNotAvailable, + #[error("io error: {0}")] + Io(#[from] std::io::Error), + #[error("{0}")] + Other(#[from] Box), +} diff --git a/src/workerd/server/cluster/ltx/ltx.rs b/src/workerd/server/cluster/ltx/ltx.rs new file mode 100644 index 00000000000..94cd854823f --- /dev/null +++ b/src/workerd/server/cluster/ltx/ltx.rs @@ -0,0 +1,542 @@ +//! ltx.rs — LTX (Lite Transaction) file reader/writer + CRC64-ISO checksums. +//! +//! Ported from ltx@v0.5.2 `ltx.go`, `checksum.go`, `encoder.go`, `decoder.go` +//! and litestream@v0.5.11 `v3.go`. `reference/ltx-format.md` describes the +//! authoritative byte layout. +//! +//! The reader decodes a complete LTX file with either v0.5.2 LZ4 blocks or the +//! older LZ4 frames. It verifies the CRC64-ISO file checksum and the rolling +//! snapshot checksum. The v0.5.2 writer emits exact upstream bytes, while the +//! default writer preserves the legacy layout during the staged rollout. + +use std::time::SystemTime; + +use crate::ltx::CHECKSUM_FLAG; +use crate::ltx::Checksum; +use crate::ltx::TXID; +use crate::ltx::error::Error; +use crate::ltx::error::Result; + +// ── Constants (ltx@v0.5.2 ltx.go:18-55) ────────────────────────────────────── + +/// First 4 bytes of every LTX file. +pub const MAGIC: &[u8; 4] = b"LTX1"; +/// Current LTX file format version. +pub const VERSION: i32 = 3; +pub const HEADER_SIZE: usize = 100; +pub const PAGE_HEADER_SIZE: usize = 6; +pub const TRAILER_SIZE: usize = 16; +pub const CHECKSUM_SIZE: usize = 8; + +/// Header flag: checksums are not tracked for this file. +pub const HEADER_FLAG_NO_CHECKSUM: u32 = 1 << 1; +pub const HEADER_FLAG_MASK: u32 = HEADER_FLAG_NO_CHECKSUM; + +/// A four-byte compressed-size field follows the page header, and the page +/// uses raw LZ4 block compression. Files written before ltx v0.5.2 omit this +/// flag and contain one LZ4 frame per page. +pub const PAGE_HEADER_FLAG_SIZE: u16 = 1 << 0; +pub const PAGE_HEADER_FLAG_MASK: u16 = PAGE_HEADER_FLAG_SIZE; + +/// SQLite PENDING_BYTE offset; the lock page derives from it. +pub const PENDING_BYTE: i64 = 0x4000_0000; + +fn corrupt(msg: impl Into) -> Error { + // Wrap a format error as LTXCorrupted, matching litestream's classification + // of malformed LTX content (litestream.go ErrLTXCorrupted). + let _ = msg; + Error::LTXCorrupted +} + +/// Returns the lock page number for a given page size (ltx.go:494). +/// +/// `page_size` is expected to be a validated SQLite page size (a power of two in +/// `[512, 65536]`); for any such value the result is identical to Go's +/// `LockPgno` (`PENDING_BYTE / page_size + 1`). A `page_size` of `0` — which +/// only reaches here via an unvalidated/adversarial header — would make the +/// underlying integer divide panic in both Go and Rust, so we guard it and +/// return `0` (never a real page number) instead of dividing. All in-crate +/// callers validate the header first, mirroring Go's `DecodeHeader`-before- +/// `LockPgno` ordering, so this guard is reached only by a direct external call. +pub fn lock_pgno(page_size: u32) -> u32 { + if page_size == 0 { + return 0; + } + (PENDING_BYTE / page_size as i64) as u32 + 1 +} + +// ── CRC64-ISO (checksum.go:177 `crc64.MakeTable(crc64.ISO)`) ────────────────── + +/// Streaming CRC64-ISO hasher matching Go's `hash/crc64` digest semantics +/// (CRC-64/GO-ISO). celld uses `crc-fast` for SIMD folding; this is Go's own +/// byte-at-a-time table walk (crc64.go `update`), which avoids a dependency. +#[derive(Clone, Default)] +pub struct Crc64 { + crc: u64, +} + +/// Go's `crc64.MakeTable(crc64.ISO)`: the reflected polynomial table. +const CRC64_ISO_TABLE: [u64; 256] = { + let mut table = [0u64; 256]; + let mut i = 0; + while i < 256 { + let mut crc = i as u64; + let mut bit = 0; + while bit < 8 { + crc = if crc & 1 == 1 { + (crc >> 1) ^ 0xD800_0000_0000_0000 + } else { + crc >> 1 + }; + bit += 1; + } + table[i] = crc; + i += 1; + } + table +}; + +impl Crc64 { + pub fn new() -> Self { + Self::default() + } + + pub fn update(&mut self, data: &[u8]) { + let mut crc = !self.crc; + for &b in data { + crc = CRC64_ISO_TABLE[(crc as u8 ^ b) as usize] ^ (crc >> 8); + } + self.crc = !crc; + } + + pub fn sum64(&self) -> u64 { + self.crc + } +} + +/// CRC64 checksum of a single page combined with its page number, with the +/// ChecksumFlag set (checksum.go:106-116). Input is `BE_u32(pgno) ++ data`. +pub fn checksum_page(pgno: u32, data: &[u8]) -> Checksum { + let mut h = Crc64::new(); + h.update(&pgno.to_be_bytes()); + h.update(data); + CHECKSUM_FLAG | h.sum64() +} + +// ── Header / PageHeader / Trailer ───────────────────────────────────────────── + +/// LTX file header (100 bytes). Ported from ltx@v0.5.1 ltx.go:179-326. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub struct Header { + pub version: i32, + pub flags: u32, + pub page_size: u32, + pub commit: u32, + pub min_txid: TXID, + pub max_txid: TXID, + pub timestamp: i64, + pub pre_apply_checksum: Checksum, + pub wal_offset: i64, + pub wal_size: i64, + pub wal_salt1: u32, + pub wal_salt2: u32, + pub node_id: u64, +} + +impl Header { + /// True if this header begins a complete database snapshot (MinTXID == 1). + pub fn is_snapshot(&self) -> bool { + self.min_txid == TXID(1) + } + + /// True if checksum tracking is disabled for this file. + pub fn no_checksum(&self) -> bool { + self.flags & HEADER_FLAG_NO_CHECKSUM != 0 + } + + /// Decodes a header from a 100-byte slice (ltx.go:302-326). All big-endian. + pub fn parse(b: &[u8]) -> Result
{ + if b.len() < HEADER_SIZE { + return Err(corrupt("short header")); + } + if &b[0..4] != MAGIC { + return Err(corrupt("bad magic")); + } + Ok(Header { + version: VERSION, + flags: u32_be(&b[4..]), + page_size: u32_be(&b[8..]), + commit: u32_be(&b[12..]), + min_txid: TXID(u64_be(&b[16..])), + max_txid: TXID(u64_be(&b[24..])), + timestamp: u64_be(&b[32..]) as i64, + pre_apply_checksum: u64_be(&b[40..]), + wal_offset: u64_be(&b[48..]) as i64, + wal_size: u64_be(&b[56..]) as i64, + wal_salt1: u32_be(&b[64..]), + wal_salt2: u32_be(&b[68..]), + node_id: u64_be(&b[72..]), + }) + } + + /// Encodes the header to 100 bytes (ltx.go:283-299). + pub fn marshal(&self) -> [u8; HEADER_SIZE] { + let mut b = [0u8; HEADER_SIZE]; + b[0..4].copy_from_slice(MAGIC); + b[4..8].copy_from_slice(&self.flags.to_be_bytes()); + b[8..12].copy_from_slice(&self.page_size.to_be_bytes()); + b[12..16].copy_from_slice(&self.commit.to_be_bytes()); + b[16..24].copy_from_slice(&self.min_txid.0.to_be_bytes()); + b[24..32].copy_from_slice(&self.max_txid.0.to_be_bytes()); + b[32..40].copy_from_slice(&(self.timestamp as u64).to_be_bytes()); + b[40..48].copy_from_slice(&self.pre_apply_checksum.to_be_bytes()); + b[48..56].copy_from_slice(&(self.wal_offset as u64).to_be_bytes()); + b[56..64].copy_from_slice(&(self.wal_size as u64).to_be_bytes()); + b[64..68].copy_from_slice(&self.wal_salt1.to_be_bytes()); + b[68..72].copy_from_slice(&self.wal_salt2.to_be_bytes()); + b[72..80].copy_from_slice(&self.node_id.to_be_bytes()); + b + } + + /// Validates header invariants (ltx.go:208-267). + pub fn validate(&self) -> Result<()> { + if self.version != VERSION { + return Err(corrupt("invalid version")); + } + if self.flags != (self.flags & HEADER_FLAG_MASK) { + return Err(corrupt("invalid flags")); + } + if !is_valid_page_size(self.page_size) { + return Err(corrupt("invalid page size")); + } + if self.min_txid == TXID(0) { + return Err(corrupt("minimum transaction id required")); + } + if self.max_txid == TXID(0) { + return Err(corrupt("maximum transaction id required")); + } + if self.min_txid > self.max_txid { + return Err(corrupt("transaction ids out of order")); + } + if self.wal_offset < 0 { + return Err(corrupt("wal offset cannot be negative")); + } + if self.wal_size < 0 { + return Err(corrupt("wal size cannot be negative")); + } + if (self.wal_salt1 != 0 || self.wal_salt2 != 0) && self.wal_offset == 0 { + return Err(corrupt("wal offset required if salt exists")); + } + if self.wal_offset == 0 && self.wal_size != 0 { + return Err(corrupt("wal offset required if wal size exists")); + } + if self.is_snapshot() { + if self.pre_apply_checksum != 0 { + return Err(corrupt("pre-apply checksum must be zero on snapshots")); + } + } else if self.no_checksum() { + if self.pre_apply_checksum != 0 { + return Err(corrupt("pre-apply checksum not allowed")); + } + } else { + if self.pre_apply_checksum == 0 { + return Err(corrupt("pre-apply checksum required on non-snapshot files")); + } + if self.pre_apply_checksum & CHECKSUM_FLAG == 0 { + return Err(corrupt("invalid pre-apply checksum format")); + } + } + Ok(()) + } +} + +/// Per-page header (6 bytes). Ported from ltx.go:408-447. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub struct PageHeader { + pub pgno: u32, + pub flags: u16, +} + +impl PageHeader { + pub fn is_zero(&self) -> bool { + self.pgno == 0 && self.flags == 0 + } + + pub fn parse(b: &[u8]) -> Result { + if b.len() < PAGE_HEADER_SIZE { + return Err(corrupt("short page header")); + } + Ok(PageHeader { + pgno: u32_be(&b[0..]), + flags: u16_be(&b[4..]), + }) + } + + pub fn marshal(&self) -> [u8; PAGE_HEADER_SIZE] { + let mut b = [0u8; PAGE_HEADER_SIZE]; + b[0..4].copy_from_slice(&self.pgno.to_be_bytes()); + b[4..6].copy_from_slice(&self.flags.to_be_bytes()); + b + } + + pub fn validate(&self) -> Result<()> { + if self.pgno == 0 { + return Err(corrupt("page number required")); + } + if self.flags != (self.flags & PAGE_HEADER_FLAG_MASK) { + return Err(corrupt("invalid page header flags")); + } + Ok(()) + } +} + +/// File trailer (16 bytes). Ported from ltx.go:348-393. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub struct Trailer { + pub post_apply_checksum: Checksum, + pub file_checksum: Checksum, +} + +impl Trailer { + /// Validates the checksum fields against the header checksum mode. + pub fn validate(&self, header: Header) -> Result<()> { + if header.no_checksum() { + if self.post_apply_checksum != 0 { + return Err(corrupt("post-apply checksum not allowed")); + } + } else if self.post_apply_checksum == 0 || self.post_apply_checksum & CHECKSUM_FLAG == 0 { + return Err(corrupt("invalid post-apply checksum")); + } + + if self.file_checksum == 0 || self.file_checksum & CHECKSUM_FLAG == 0 { + return Err(corrupt("invalid file checksum")); + } + Ok(()) + } + + pub fn parse(b: &[u8]) -> Result { + if b.len() < TRAILER_SIZE { + return Err(corrupt("short trailer")); + } + Ok(Trailer { + post_apply_checksum: u64_be(&b[0..]), + file_checksum: u64_be(&b[8..]), + }) + } + + pub fn marshal(&self) -> [u8; TRAILER_SIZE] { + let mut b = [0u8; TRAILER_SIZE]; + b[0..8].copy_from_slice(&self.post_apply_checksum.to_be_bytes()); + b[8..16].copy_from_slice(&self.file_checksum.to_be_bytes()); + b + } +} + +/// True if `sz` is a power of two in [512, 65536] (ltx.go:399-406). +pub fn is_valid_page_size(sz: u32) -> bool { + let mut i = 512u32; + while i <= 65536 { + if sz == i { + return true; + } + i *= 2; + } + false +} + +/// Formats an LTX filename for a transaction range (ltx.go:487-489). +pub fn format_filename(min_txid: TXID, max_txid: TXID) -> String { + format!("{}-{}.ltx", min_txid, max_txid) +} + +/// Parses a `-.ltx` filename (ltx.go:450-459). +pub fn parse_filename(name: &str) -> Result<(TXID, TXID)> { + let stem = name + .strip_suffix(".ltx") + .ok_or_else(|| corrupt("invalid ltx filename"))?; + let (a, b) = stem + .split_once('-') + .ok_or_else(|| corrupt("invalid ltx filename"))?; + if a.len() != 16 || b.len() != 16 { + return Err(corrupt("invalid ltx filename")); + } + let min = u64::from_str_radix(a, 16).map_err(|_| corrupt("invalid ltx filename"))?; + let max = u64::from_str_radix(b, 16).map_err(|_| corrupt("invalid ltx filename"))?; + Ok((TXID(min), TXID(max))) +} + +// ── FileInfo ────────────────────────────────────────────────────────────────── + +/// Metadata about an LTX file on a replica. Ported from ltx@v0.5.1 ltx.go:571-596. +/// +/// `pre_apply_checksum`/`post_apply_checksum` are populated when known (e.g. by +/// decoding) and are zero when a file is discovered by a bare directory/bucket +/// listing. Listings leave `size` and `created_at` unset, and write results use +/// the LTX header timestamp. +#[derive(Debug, Clone, PartialEq, Eq, Default)] +pub struct FileInfo { + pub level: i32, + pub min_txid: TXID, + pub max_txid: TXID, + pub pre_apply_checksum: Checksum, + pub post_apply_checksum: Checksum, + pub size: i64, + pub created_at: Option, +} + +// ── Decoder ────────────────────────────────────────────────────────────────── + +// ── Encoder (round-trip; byte-fidelity vs the real binary is D1's job) ──────── + +/// Encodes a complete LTX file with the legacy LZ4 frame representation, +/// which the decoder reads alongside the v0.5.2 block +/// representation. +pub fn encode_file( + header: &Header, + pages: &[(u32, Vec)], + post_apply_checksum: Checksum, +) -> Result> { + let mut encoder = crate::ltx::codec::Encoder::new(Vec::new()); + encoder.encode_header(*header)?; + for (page_number, data) in pages { + encoder.encode_page( + PageHeader { + pgno: *page_number, + flags: 0, + }, + data, + )?; + } + encoder.close(post_apply_checksum)?; + Ok(encoder.writer) +} + +/// Merges consecutive LTX files into one covering all their txids, as +/// ltx's compactor does. +pub fn merge_files(files: &[Vec]) -> Result> { + let mut out = Vec::new(); + merge_into(files.iter().map(Vec::as_slice).collect(), &mut out)?; + Ok(out) +} + +/// `merge_files`, from readers to a writer, a page at a time. +pub fn merge_into(inputs: Vec, out: impl std::io::Write) -> Result<()> { + let mut merge = Merge::new(inputs)?; + let mut encoder = crate::ltx::codec::Encoder::new(out); + encoder.encode_header(merge.header)?; + while let Some((pgno, data)) = merge.next()? { + encoder.encode_page(PageHeader { pgno, flags: 0 }, data)?; + } + encoder.close(merge.finish()?) +} + +/// Consecutive LTX files, oldest first, read as one: the latest version of +/// each page, through the last file's commit. Every file keeps its pages in +/// ascending order, so this holds a page per file, never a database. +pub struct Merge { + /// The merged file's header. + pub header: Header, + inputs: Vec>, + page: Vec, +} + +struct Input { + decoder: crate::ltx::codec::Decoder, + /// The page `page` holds; `None` once the file's pages are read. + pgno: Option, + page: Vec, +} + +impl Input { + fn advance(&mut self) -> Result<()> { + self.pgno = self.decoder.decode_page(&mut self.page)?.map(|p| p.pgno); + Ok(()) + } +} + +impl Merge { + pub fn new(readers: Vec) -> Result { + let mut inputs = Vec::new(); + for reader in readers { + let mut decoder = crate::ltx::codec::Decoder::new(reader); + decoder.decode_header()?; + let page = vec![0; decoder.header.page_size as usize]; + inputs.push(Input { + decoder, + pgno: None, + page, + }); + } + let (Some(first), Some(last)) = (inputs.first(), inputs.last()) else { + return Err(corrupt("nothing to merge")); + }; + let (first, last) = (first.decoder.header, last.decoder.header); + if inputs + .iter() + .any(|i| i.decoder.header.page_size != last.page_size) + { + return Err(corrupt("mismatched page sizes")); + } + let header = Header { + flags: first.flags, + min_txid: first.min_txid, + pre_apply_checksum: first.pre_apply_checksum, + ..last + }; + for input in &mut inputs { + input.advance()?; + } + let page = vec![0; header.page_size as usize]; + Ok(Self { + header, + inputs, + page, + }) + } + + /// The next page, in ascending order; `None` after the last. + pub fn next(&mut self) -> Result> { + loop { + let Some(pgno) = self.inputs.iter().filter_map(|i| i.pgno).min() else { + return Ok(None); + }; + let newest = self.inputs.iter_mut().rev().find(|i| i.pgno == Some(pgno)); + if let Some(newest) = newest { + std::mem::swap(&mut self.page, &mut newest.page); + } + for input in &mut self.inputs { + if input.pgno == Some(pgno) { + input.advance()?; + } + } + if pgno <= self.header.commit { + return Ok(Some((pgno, &self.page))); + } + } + } + + /// Checks every file's trailer: the last file's post-apply checksum. + pub fn finish(mut self) -> Result { + for input in &mut self.inputs { + input.decoder.close()?; + } + Ok(self + .inputs + .last() + .map_or(0, |i| i.decoder.trailer.post_apply_checksum)) + } +} + +// ── small byte / varint helpers ────────────────────────────────────────────── + +fn u16_be(b: &[u8]) -> u16 { + u16::from_be_bytes([b[0], b[1]]) +} +fn u32_be(b: &[u8]) -> u32 { + u32::from_be_bytes([b[0], b[1], b[2], b[3]]) +} +fn u64_be(b: &[u8]) -> u64 { + u64::from_be_bytes([b[0], b[1], b[2], b[3], b[4], b[5], b[6], b[7]]) +} + +// ── Tests ───────────────────────────────────────────────────────────────────── diff --git a/src/workerd/server/cluster/ltx/mod.rs b/src/workerd/server/cluster/ltx/mod.rs new file mode 100644 index 00000000000..3661ee9ebcd --- /dev/null +++ b/src/workerd/server/cluster/ltx/mod.rs @@ -0,0 +1,92 @@ +// Ported from celld's `celld-ltx` crate (Apache License 2.0), which was +// seeded from rustyriver and carries: +// +// - rustyriver (https://github.com/mikenomitch/rustyriver): Copyright 2026 The +// rustyriver authors, Apache License 2.0. +// - Litestream (https://github.com/benbjohnson/litestream), tag v0.5.11: +// Copyright (c) Ben Johnson and the Litestream authors, Apache License 2.0. +// - LTX (https://github.com/superfly/ltx), tag v0.5.2: Copyright (c) +// Superfly, Inc., Apache License 2.0. + +//! LTX replication for a `SQLite` database: WAL capture into LTX segments, +//! upload to an object store, and restore. +//! +//! Ported from celld's `celld-ltx` crate (Apache-2.0), itself a from-scratch +//! Rust reimplementation of Litestream v0.5.11 (Apache-2.0, Ben Johnson) with +//! the block format of LTX v0.5.2 (Apache-2.0), by way of rustyriver. Only +//! the LTX format, the replica client and restore are ported; capture is +//! `capture.rs` and compaction is the owner's (`owner.rs`). Paged restore +//! and bundles stay in celld. +//! +//! # Core position and identity helpers +//! +//! Ported from litestream@v0.5.11 litestream.go:18-203 and +//! ltx@v0.5.2 ltx.go:66-145. + +// The modules below keep celld's source nearly line for line so a later +// diff against celld stays readable. They are therefore exempt from the +// pedantic and nursery style groups; the crate's deny lints still apply. +#[expect( + clippy::pedantic, + clippy::nursery, + reason = "ported from celld; kept diffable" +)] +pub mod client; +#[expect( + clippy::pedantic, + clippy::nursery, + reason = "ported from celld; kept diffable" +)] +pub(crate) mod codec; +pub mod error; +#[expect( + clippy::pedantic, + clippy::nursery, + clippy::module_inception, + reason = "ported from celld; kept diffable" +)] +pub mod ltx; +#[expect(clippy::nursery, reason = "ported from celld; kept diffable")] +pub mod replica; + +pub use client::bucket::BucketClient; +pub use error::Error; +pub use replica::SNAPSHOT_LEVEL; +pub use replica::plan; +pub use replica::restore; + +// ── TXID ────────────────────────────────────────────────────────────────────── + +/// A monotonically increasing transaction identifier. +/// +/// Represented as a `u64` and formatted as a zero-padded 16-character +/// lowercase hex string (e.g. `"0000000000000001"`). This matches the +/// on-disk LTX filename convention and `ltx.TXID.String()` in +/// ltx@v0.5.1 ltx.go:142-144. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Default)] +#[expect(clippy::upper_case_acronyms, reason = "Litestream's name for it")] +pub struct TXID(pub u64); + +impl std::fmt::Display for TXID { + /// Formats as a 16-digit lowercase hex string, zero-padded. + /// + /// Ported from `TXID.String()` in ltx@v0.5.1 ltx.go:142. + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{:016x}", self.0) + } +} + +// ── Checksum ────────────────────────────────────────────────────────────────── + +/// A 64-bit LTX rolling / file checksum. +/// +/// The high bit is the `ChecksumFlag` sentinel used by the LTX format to +/// distinguish a real checksum from an absent one. +/// +/// Ported from ltx@v0.5.1 ltx.go (type alias `Checksum uint64`). +pub type Checksum = u64; + +/// Flag bit set on every valid non-zero LTX checksum. +/// +/// Ported from `ChecksumFlag` in ltx@v0.5.1 ltx.go:55. +pub const CHECKSUM_FLAG: Checksum = 1 << 63; diff --git a/src/workerd/server/cluster/ltx/replica.rs b/src/workerd/server/cluster/ltx/replica.rs new file mode 100644 index 00000000000..d2597d85a9c --- /dev/null +++ b/src/workerd/server/cluster/ltx/replica.rs @@ -0,0 +1,157 @@ +//! Restore from a replica, ported from litestream@v0.5.11 `replica.go`. +//! Uploading is the owner's (`owner.rs`). +//! +//! Uploads live at L0; a compaction writes a snapshot from TXID 1 at +//! `SNAPSHOT_LEVEL` (litestream's L9). A restore is the newest snapshot and +//! the L0 files that extend it, merged the way `ltx.NewCompactor` and +//! `Decoder.DecodeDatabaseTo` do (ltx@v0.5.1). + +use std::fs::File; +use std::io::BufReader; +use std::io::BufWriter; +use std::io::Write; +use std::path::Path; +use std::path::PathBuf; + +use futures::StreamExt; +use futures::TryStreamExt; +use futures::stream; + +use crate::ltx::TXID; +use crate::ltx::client::ReplicaClient; +use crate::ltx::error::Error; +use crate::ltx::error::Result; +use crate::ltx::ltx::{self}; + +/// Downloads in flight during a restore, to hide the store's round trips. +const RESTORE_DOWNLOAD_CONCURRENCY: usize = 16; + +/// The level of snapshots, as litestream's `SnapshotLevel`. +pub const SNAPSHOT_LEVEL: i32 = 9; + +/// The files a restore through `txid` (`TXID(0)`: the latest) reads, in +/// order: the newest snapshot, then each L0 file that extends what came +/// before. A file inside the range already reached is skipped: tiering and +/// bucket-acknowledged commits upload merged ranges concurrently, so an +/// older, narrower range can land beside a newer, wider one, and applying it +/// after would bring back old pages (lost writes on the fleet). A file that +/// extends the range carries the newest version of every page it holds. Not +/// checked for gaps. +pub async fn plan(client: &C, txid: TXID) -> Result> { + let mut infos = client.ltx_files(SNAPSHOT_LEVEL, TXID(0)).await?; + infos.extend(client.ltx_files(0, TXID(0)).await?); + if txid != TXID(0) { + infos.retain(|info| info.max_txid <= txid); + } + infos.sort_by_key(|info| (info.min_txid, std::cmp::Reverse(info.max_txid))); + let mut reached = TXID(0); + infos.retain(|info| { + let extends = info.max_txid > reached; + reached = reached.max(info.max_txid); + extends + }); + Ok(infos) +} + +/// Restores the replica into `output_path`, which must not exist, up to +/// `txid` (`TXID(0)`: the latest). The plan (`plan`) must be contiguous +/// from TXID 1 (replica.go:1454-1526). The files are spooled to +/// `.restore/` and merged a page at a time into `.tmp`, +/// which is renamed into place (replica.go:657-694), on the blocking pool. +pub async fn restore(client: &C, output_path: &Path, txid: TXID) -> Result<()> { + if output_path.exists() { + return Err(Error::Other( + format!( + "cannot restore, output path already exists: {}", + output_path.display() + ) + .into(), + )); + } + let infos = plan(client, txid).await?; + let mut reached = TXID(0); + for info in &infos { + if info.min_txid.0 > reached.0 + 1 { + return Err(Error::Other( + format!( + "non-contiguous ltx files: have up to {reached} but next file starts at {}", + info.min_txid + ) + .into(), + )); + } + reached = reached.max(info.max_txid); + } + if infos.is_empty() || (txid != TXID(0) && reached < txid) { + return Err(Error::TxNotAvailable); + } + let spool = with_suffix(output_path, ".restore"); + std::fs::create_dir_all(&spool)?; + let files: Vec<_> = (0..infos.len()) + .map(|i| spool.join(i.to_string())) + .collect(); + let downloaded = stream::iter(infos.iter().zip(&files)) + .map(|(info, file)| client.open_ltx_file(info.level, info.min_txid, info.max_txid, file)) + .buffer_unordered(RESTORE_DOWNLOAD_CONCURRENCY) + .try_collect::<()>() + .await; + if let Err(e) = downloaded { + let _ = std::fs::remove_dir_all(&spool); + return Err(e); + } + let output = output_path.to_owned(); + let result = blocking(move || write_image(&files, &output)).await; + let _ = std::fs::remove_dir_all(&spool); + result +} + +/// Merges the LTX `files` (a snapshot, then what extends it) into the +/// database at `output`: pages `1..=commit`, the lock page zero-filled +/// (decoder.go:236-254). A page missing from the merge means the chain is +/// incomplete. There is no fsync: a restored image is never read after a +/// crash, since the takeover that follows one deletes it. +fn write_image(files: &[PathBuf], output: &Path) -> Result<()> { + let readers = files + .iter() + .map(|f| Ok(BufReader::new(File::open(f)?))) + .collect::>>()?; + let mut merge = ltx::Merge::new(readers)?; + let (page_size, commit) = (merge.header.page_size, merge.header.commit); + let lock = ltx::lock_pgno(page_size); + let zero = vec![0; page_size as usize]; + let tmp = with_suffix(output, ".tmp"); + let mut out = BufWriter::new(File::create(&tmp)?); + let mut next = 1; + let fill = |out: &mut BufWriter, next: &mut u32, to: u32| -> Result<()> { + for pgno in *next..to { + if pgno != lock { + return Err(Error::Other(format!("restore lacks page {pgno}").into())); + } + out.write_all(&zero)?; + } + *next = to.max(*next); + Ok(()) + }; + while let Some((pgno, data)) = merge.next()? { + fill(&mut out, &mut next, pgno)?; + out.write_all(data)?; + next = pgno + 1; + } + fill(&mut out, &mut next, commit + 1)?; + merge.finish()?; + out.flush()?; + drop(out); + Ok(std::fs::rename(&tmp, output)?) +} + +fn with_suffix(path: &Path, suffix: &str) -> PathBuf { + let mut path = path.as_os_str().to_owned(); + path.push(suffix); + path.into() +} + +async fn blocking(f: impl FnOnce() -> Result + Send + 'static) -> Result { + tokio::task::spawn_blocking(f) + .await + .map_err(|e| Error::Other(format!("blocking task: {e}").into()))? +} diff --git a/src/workerd/server/cluster/ltx_tests.rs b/src/workerd/server/cluster/ltx_tests.rs new file mode 100644 index 00000000000..8cdc9a52059 --- /dev/null +++ b/src/workerd/server/cluster/ltx_tests.rs @@ -0,0 +1,518 @@ +// Copyright (c) 2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +//! The capture against real `SQLite`: a test VFS wraps the default one and +//! reports WAL writes to a `Capture`, as workerd's VFS does, and whatever +//! the capture cut must restore to what the application sees. + +use std::cell::RefCell; +use std::ffi::CString; +use std::ffi::c_char; +use std::ffi::c_int; +use std::ffi::c_void; +use std::path::Path; +use std::path::PathBuf; +use std::rc::Rc; +use std::sync::OnceLock; + +use crate::Bucket; +use crate::Error; +use crate::capture::Capture; +use crate::ltx::BucketClient; +use crate::ltx::TXID; +use crate::ltx::client::ReplicaClient; +use crate::ltx::restore; +use crate::owner::Owners; +use crate::ownership::Claim; +use crate::ownership::Ownership; +use crate::registry::Registry; + +/// `sqlite3_vfs`, with the members the wrapper replaces typed. +#[repr(C)] +#[derive(Clone, Copy)] +struct Vfs { + version: c_int, + file_size: c_int, + max_path: c_int, + next: *mut Self, + name: *const c_char, + app_data: *mut c_void, + open: unsafe extern "C" fn(*mut Self, *const c_char, *mut File, c_int, *mut c_int) -> c_int, + rest: [*const c_void; 15], +} + +/// `sqlite3_io_methods`, likewise, followed by the method it replaces. +#[repr(C)] +#[derive(Clone, Copy)] +struct Methods { + version: c_int, + close: *const c_void, + read: *const c_void, + write: unsafe extern "C" fn(*mut File, *const c_void, c_int, i64) -> c_int, + rest: [*const c_void; 15], + wrapped_write: unsafe extern "C" fn(*mut File, *const c_void, c_int, i64) -> c_int, +} + +#[repr(C)] +struct File { + methods: *const Methods, +} + +unsafe extern "C" { + fn sqlite3_vfs_find(name: *const c_char) -> *mut Vfs; + fn sqlite3_vfs_register(vfs: *mut Vfs, default: c_int) -> c_int; + fn sqlite3_open_v2( + path: *const c_char, + db: *mut *mut c_void, + flags: c_int, + vfs: *const c_char, + ) -> c_int; + fn sqlite3_exec( + db: *mut c_void, + sql: *const c_char, + cb: *const c_void, + arg: *mut c_void, + err: *mut *mut c_char, + ) -> c_int; + fn sqlite3_prepare_v2( + db: *mut c_void, + sql: *const c_char, + n: c_int, + stmt: *mut *mut c_void, + tail: *mut *const c_char, + ) -> c_int; + fn sqlite3_step(stmt: *mut c_void) -> c_int; + fn sqlite3_column_int64(stmt: *mut c_void, col: c_int) -> i64; + fn sqlite3_finalize(stmt: *mut c_void) -> c_int; + fn sqlite3_close(db: *mut c_void) -> c_int; +} + +/// Just enough of `SQLite` for the tests: run SQL, read one row of integers. +struct Connection(*mut c_void); + +impl Connection { + fn open(path: &str) -> Self { + let path = CString::new(path).unwrap(); + let mut db = std::ptr::null_mut(); + // OPEN_READWRITE | OPEN_CREATE | OPEN_URI. + // SAFETY: the arguments are valid for the call. + let rc = unsafe { sqlite3_open_v2(path.as_ptr(), &raw mut db, 0x46, std::ptr::null()) }; + assert_eq!(rc, 0); + Self(db) + } + + fn execute(&self, sql: &str) { + let sql = CString::new(sql).unwrap(); + // SAFETY: `self.0` is an open connection. + let rc = unsafe { + sqlite3_exec( + self.0, + sql.as_ptr(), + std::ptr::null(), + std::ptr::null_mut(), + std::ptr::null_mut(), + ) + }; + assert_eq!(rc, 0, "{sql:?}"); + } + + fn row(&self, sql: &str, columns: c_int) -> Vec { + let sql = CString::new(sql).unwrap(); + let mut stmt = std::ptr::null_mut(); + // SAFETY: `self.0` is an open connection and `stmt` is finalized. + unsafe { + assert_eq!( + sqlite3_prepare_v2( + self.0, + sql.as_ptr(), + -1, + &raw mut stmt, + std::ptr::null_mut() + ), + 0 + ); + assert_eq!(sqlite3_step(stmt), 100); + let row = (0..columns) + .map(|c| sqlite3_column_int64(stmt, c)) + .collect(); + sqlite3_finalize(stmt); + row + } + } +} + +impl Drop for Connection { + fn drop(&mut self) { + // SAFETY: `self.0` is an open connection, closed once. + unsafe { sqlite3_close(self.0) }; + } +} + +const SQLITE_OPEN_WAL: c_int = 0x0008_0000; + +thread_local! { + static CAPTURE: RefCell> = const { RefCell::new(None) }; +} + +/// The wrapped VFS, and where a file's "is a WAL" flag sits past the +/// wrapped file's own state. +struct Wrapped { + vfs: usize, + flag_at: usize, +} + +fn wrapped() -> &'static Wrapped { + static WRAPPED: OnceLock = OnceLock::new(); + // SAFETY: the default VFS lives for the process, and the copy + // registered here is leaked, as `SQLite` requires. + WRAPPED.get_or_init(|| unsafe { + let native = sqlite3_vfs_find(std::ptr::null()); + let flag_at = usize::try_from((*native).file_size) + .unwrap() + .next_multiple_of(8); + let mut vfs = *native; + vfs.name = c"capture-test".as_ptr(); + vfs.file_size = c_int::try_from(flag_at + 8).unwrap(); + vfs.open = open; + assert_eq!(sqlite3_vfs_register(Box::leak(Box::new(vfs)), 0), 0); + Wrapped { + vfs: native as usize, + flag_at, + } + }) +} + +unsafe fn is_wal(file: *mut File) -> *mut bool { + // SAFETY: the flag is within the `file_size` bytes `SQLite` allocated. + unsafe { file.cast::().add(wrapped().flag_at).cast() } +} + +unsafe extern "C" fn open( + _: *mut Vfs, + name: *const c_char, + file: *mut File, + flags: c_int, + out: *mut c_int, +) -> c_int { + // SAFETY: `SQLite` passes a file of the wrapper's size; the patched + // method table is leaked, so it outlives the file. + unsafe { + let native = wrapped().vfs as *mut Vfs; + let rc = ((*native).open)(native, name, file, flags, out); + if rc == 0 && !(*file).methods.is_null() { + *is_wal(file) = flags & SQLITE_OPEN_WAL != 0; + let mut methods = *(*file).methods; + methods.wrapped_write = methods.write; + methods.write = write; + (*file).methods = Box::leak(Box::new(methods)); + } + rc + } +} + +unsafe extern "C" fn write(file: *mut File, data: *const c_void, n: c_int, offset: i64) -> c_int { + // SAFETY: `data` holds `n` bytes, and the file was opened by `open`. + unsafe { + let rc = ((*(*file).methods).wrapped_write)(file, data, n, offset); + if rc == 0 && *is_wal(file) { + let data = std::slice::from_raw_parts(data.cast::(), usize::try_from(n).unwrap()); + let offset = u64::try_from(offset).unwrap(); + CAPTURE.with_borrow_mut(|c| c.as_mut().unwrap().write(offset, data)); + } + rc + } +} + +fn scratch(name: &str) -> PathBuf { + let root = std::env::var_os("TEST_TMPDIR").map_or_else(std::env::temp_dir, PathBuf::from); + let dir = root.join(format!("capture-{name}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + dir +} + +/// The application's connection, through the capturing VFS, with the +/// capture already watching. +fn open_app(path: &Path) -> Connection { + wrapped(); + CAPTURE.set(Some(Capture::open(path).unwrap())); + let app = Connection::open(&format!("file:{}?vfs=capture-test", path.display())); + app.execute("PRAGMA journal_mode=WAL; CREATE TABLE t (id INTEGER PRIMARY KEY, v TEXT)"); + app +} + +fn insert(app: &Connection, from: i64, to: i64) { + let mut sql = String::from("BEGIN;"); + for id in from..to { + sql += &format!( + "INSERT INTO t (id, v) VALUES ({id}, '{}');", + "x".repeat(200) + ); + } + sql += "COMMIT;"; + app.execute(&sql); +} + +/// The table as one comparable value: row count, id sum, and value bytes. +fn rows(conn: &Connection) -> (i64, i64, i64) { + let r = conn.row( + "SELECT count(*), coalesce(sum(id), 0), coalesce(sum(length(v)), 0) FROM t", + 3, + ); + (r[0], r[1], r[2]) +} + +fn runtime() -> tokio::runtime::Runtime { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap() +} + +/// The last captured txid. +fn last() -> u64 { + CAPTURE.with_borrow(|c| c.as_ref().unwrap().last().unwrap()) +} + +/// Upload the captured entries `first..=last` merged into one file, as +/// tiering does. +fn upload(store: &Bucket, first: u64, last: u64) { + upload_at(store, 0, first, last); +} + +fn upload_at(store: &Bucket, level: i32, first: u64, last: u64) { + let files: Vec> = CAPTURE.with_borrow(|c| { + let c = c.as_ref().unwrap(); + (first..=last).map(|t| c.entry(t).unwrap()).collect() + }); + let from = scratch(&format!("upload-{level}-{first}-{last}")).join("ltx"); + std::fs::write(&from, crate::ltx::ltx::merge_files(&files).unwrap()).unwrap(); + let client = BucketClient::new(store.at("cell")); + runtime() + .block_on(client.write_ltx_file(level, TXID(first), TXID(last), &from)) + .unwrap(); +} + +fn restored_rows(store: &Bucket, out: &Path) -> (i64, i64, i64) { + let client = BucketClient::new(store.at("cell")); + runtime().block_on(restore(&client, out, TXID(0))).unwrap(); + rows(&Connection::open(out.to_str().unwrap())) +} + +#[test] +fn restore_reproduces_the_application_rows() { + let dir = scratch("roundtrip"); + let app = open_app(&dir.join("app.db")); + let store = Bucket::memory(""); + insert(&app, 0, 100); + let shipped = last(); + upload(&store, 1, shipped); + insert(&app, 100, 250); + app.execute("DELETE FROM t WHERE id % 7 = 0"); + upload(&store, shipped + 1, last()); + let want = rows(&app); + assert_eq!(want.0, 250 - 36); + assert_eq!(restored_rows(&store, &dir.join("restored.db")), want); +} + +#[test] +fn restore_reads_the_newest_snapshot_and_what_extends_it() { + let dir = scratch("snapshot"); + let app = open_app(&dir.join("app.db")); + let store = Bucket::memory(""); + insert(&app, 0, 100); + let a = last(); + upload(&store, 1, a); + upload_at(&store, crate::ltx::SNAPSHOT_LEVEL, 1, a); + insert(&app, 100, 200); + let b = last(); + upload(&store, a + 1, b); + upload_at(&store, crate::ltx::SNAPSHOT_LEVEL, 1, b); + app.execute("DELETE FROM t WHERE id % 3 = 0"); + upload(&store, b + 1, last()); + let client = BucketClient::new(store.at("cell")); + let plan = runtime() + .block_on(crate::ltx::plan(&client, TXID(0))) + .unwrap(); + let plan: Vec<_> = plan + .iter() + .map(|f| (f.level, f.min_txid.0, f.max_txid.0)) + .collect(); + assert_eq!( + plan, + [(crate::ltx::SNAPSHOT_LEVEL, 1, b), (0, b + 1, last())] + ); + assert_eq!(restored_rows(&store, &dir.join("restored.db")), rows(&app)); +} + +#[test] +fn a_rolled_back_transaction_leaves_nothing() { + let dir = scratch("rollback"); + let app = open_app(&dir.join("app.db")); + let store = Bucket::memory(""); + insert(&app, 0, 10); + let wal = || std::fs::metadata(dir.join("app.db-wal")).unwrap().len(); + let before = wal(); + // A small cache spills the doomed transaction's frames into the WAL. + app.execute("PRAGMA cache_size=2; BEGIN;"); + app.execute( + "WITH RECURSIVE s(x) AS (SELECT 1000 UNION ALL SELECT x + 1 FROM s WHERE x < 3000) \ + INSERT INTO t SELECT x, hex(randomblob(100)) FROM s", + ); + assert!(wal() > before + 100 * 4096, "no frames spilled"); + app.execute("ROLLBACK;"); + insert(&app, 10, 20); + upload(&store, 1, last()); + assert_eq!(rows(&app).0, 20); + assert_eq!(restored_rows(&store, &dir.join("restored.db")), rows(&app)); +} + +#[test] +fn checkpoints_and_a_restart_keep_the_chain_whole() { + let dir = scratch("checkpoint"); + let app = open_app(&dir.join("app.db")); + let store = Bucket::memory(""); + // Frequent checkpoints: the WAL starts over many times. + app.execute("PRAGMA wal_autocheckpoint=8"); + for from in (0..600).step_by(50) { + insert(&app, from, from + 50); + } + app.execute("PRAGMA wal_checkpoint(TRUNCATE)"); + insert(&app, 600, 650); + upload(&store, 1, last()); + assert_eq!(restored_rows(&store, &dir.join("one.db")), rows(&app)); + + // A new chain from a snapshot of the live database (reclaim), after + // commits still in the WAL, then more commits, shrinking at the end. + CAPTURE.with_borrow_mut(|c| c.as_mut().unwrap().restart().unwrap()); + insert(&app, 650, 700); + app.execute("DELETE FROM t WHERE id > 20; VACUUM;"); + insert(&app, 5000, 5010); + let store = Bucket::memory(""); + upload(&store, 1, last()); + assert_eq!(rows(&app).0, 31); + assert_eq!(restored_rows(&store, &dir.join("two.db")), rows(&app)); +} + +#[test] +fn a_restart_after_the_database_closed_reads_the_file() { + let dir = scratch("closed"); + let app = open_app(&dir.join("app.db")); + insert(&app, 0, 50); + let want = rows(&app); + // Closing the last connection checkpoints the WAL and deletes it, so + // the committed pages the capture knows of are in the database file. + drop(app); + assert!(!dir.join("app.db-wal").exists()); + CAPTURE.with_borrow_mut(|c| c.as_mut().unwrap().restart().unwrap()); + // Reopened, before its first write: an empty WAL. + let app = Connection::open(&format!( + "file:{}?vfs=capture-test", + dir.join("app.db").display() + )); + assert_eq!(rows(&app), want); + assert_eq!(std::fs::metadata(dir.join("app.db-wal")).unwrap().len(), 0); + CAPTURE.with_borrow_mut(|c| c.as_mut().unwrap().restart().unwrap()); + let store = Bucket::memory(""); + upload(&store, 1, last()); + assert_eq!(restored_rows(&store, &dir.join("restored.db")), want); +} + +#[test] +fn a_file_the_chain_already_covers_is_not_applied() { + // Tiering and a bucket-acknowledged commit upload concurrently, so a + // stale range can land beside a newer, wider one (lost writes on the + // fleet): 3..=last-4 is inside 2..=last, and must not override it. + let dir = scratch("covered"); + let app = open_app(&dir.join("app.db")); + let store = Bucket::memory(""); + for from in (0..120).step_by(10) { + insert(&app, from, from + 10); + } + let last = last(); + upload(&store, 1, 1); + upload(&store, 2, last); + upload(&store, 3, last - 4); + assert_eq!(restored_rows(&store, &dir.join("restored.db")), rows(&app)); +} + +#[test] +fn a_deleted_database_restores_empty_then_as_recreated() { + // A facet's delete() or a deleteAll() removes the file; the new + // database's first page then reaches the file, not the WAL. + let dir = scratch("reset"); + let path = dir.join("app.db"); + let app = open_app(&path); + insert(&app, 0, 50); + drop(app); + std::fs::remove_file(&path).unwrap(); + CAPTURE.with_borrow_mut(|c| c.as_mut().unwrap().reset()); + let deleted = last(); + let store = Bucket::memory(""); + upload(&store, 1, deleted); + let client = BucketClient::new(store.at("cell")); + let out = dir.join("empty.db"); + runtime().block_on(restore(&client, &out, TXID(0))).unwrap(); + assert_eq!(std::fs::metadata(&out).unwrap().len(), 0); + + let app = Connection::open(&format!("file:{}?vfs=capture-test", path.display())); + app.execute("PRAGMA journal_mode=WAL; CREATE TABLE t (id INTEGER PRIMARY KEY, v TEXT)"); + insert(&app, 100, 110); + upload(&store, deleted + 1, last()); + assert_eq!(rows(&app).0, 10); + assert_eq!(restored_rows(&store, &dir.join("split.db")), rows(&app)); + // Merged into one range, as tiering merges. + let store = Bucket::memory(""); + upload(&store, 1, last()); + assert_eq!(restored_rows(&store, &dir.join("merged.db")), rows(&app)); +} + +#[test] +fn a_superseded_owners_commit_is_not_acknowledged() { + // a owns x and commits while b takes x over: the upload lands in a's + // epoch, but own.json no longer names it, so a must not acknowledge. + let dir = scratch("superseded"); + let path = dir.join("x.db"); + let store = Bucket::memory("").at("cluster"); + let node = |name: &str| { + let registry = Rc::new(Registry::new(store.clone(), name, name, 10_000)); + let ownership = Ownership::new(store.clone(), registry.clone()); + (registry, Owners::new(store.clone()), ownership) + }; + let ((a, a_owners, a_own), (b, b_owners, b_own)) = (node("a"), node("b")); + let claim = + |own: &Ownership, now| match runtime().block_on(own.acquire_or_route("ns", "x", now)) { + Ok(Claim::Owned(owned)) => owned, + other => panic!("{other:?}"), + }; + runtime().block_on(a.renew(0)).unwrap(); + let owned = claim(&a_own, 0); + let rt = runtime(); + assert!( + rt.block_on(a_owners.open_fresh(&a_own, "ns", "x", &owned)) + .unwrap() + ); + rt.block_on(a_owners.take_over(&a_own, "ns", "x", &owned, &path)) + .unwrap(); + + let app = Connection::open(path.to_str().unwrap()); + app.execute("PRAGMA journal_mode=WAL; CREATE TABLE t (id INTEGER PRIMARY KEY, v TEXT)"); + insert(&app, 0, 10); + let wal = std::fs::read(dir.join("x.db-wal")).unwrap(); + a_owners.wal_write("ns", "x", 0, &wal[..32]); + a_owners.wal_write("ns", "x", 32, &wal[32..]); + + // a stops renewing; b takes x over at epoch 2. + rt.block_on(b.renew(9_000)).unwrap(); + let owned = claim(&b_own, 11_000); + assert_eq!(owned.epoch, 2); + rt.block_on(b_owners.take_over(&b_own, "ns", "x", &owned, &dir.join("b.db"))) + .unwrap(); + + let commit = rt.block_on(a_owners.commit(&a_own, "ns", "x")); + assert!( + matches!(&commit, Err(Error::Contended(m)) if m.contains("superseded")), + "{commit:?}" + ); +} diff --git a/src/workerd/server/cluster/owner.rs b/src/workerd/server/cluster/owner.rs new file mode 100644 index 00000000000..6c493d4d2f8 --- /dev/null +++ b/src/workerd/server/cluster/owner.rs @@ -0,0 +1,839 @@ +// Copyright (c) 2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +//! The owner's side of an object: taking it over, the commit path, and +//! keeping its chain in the bucket compact. +//! +//! A commit is acknowledged once the bucket holds it and own.json still +//! names the epoch after the upload. Each epoch's commits form a chain of +//! LTX files under `do///ltx/e`, starting with a snapshot +//! of the state the epoch inherited; a takeover restores the last opened +//! epoch's chain. + +use std::cell::Cell; +use std::cell::RefCell; +use std::collections::HashMap; +use std::path::Path; +use std::path::PathBuf; +use std::rc::Rc; + +use futures::StreamExt; +use futures::stream; + +use crate::Bucket; +use crate::Epoch; +use crate::Error; +use crate::Ownership; +use crate::Result; +use crate::Txid; +use crate::capture::Capture; +use crate::ltx::BucketClient; +use crate::ltx::SNAPSHOT_LEVEL; +use crate::ltx::TXID; +use crate::ltx::client::ReplicaClient; +use crate::ownership::Owned; + +/// Objects compacting at once, and deletes in flight for one compaction. +const TIER_CONCURRENCY: usize = 64; +/// Evictions a node uploads at once. +const EVICT_CONCURRENCY: usize = 32; +/// Evicted objects whose database file this node remembers as current. +const DORMANT_MAX: usize = 1 << 16; + +/// An owned object, its capture, and how far its commits are acknowledged. +struct Slot { + object: tokio::sync::Mutex, + /// Outside the lock: `SQLite` reports WAL writes synchronously, and may + /// while an upload holds the lock. + capture: RefCell, + /// Through where commits are in the bucket, by epoch, then txid. + acked: tokio::sync::watch::Sender<(Epoch, Txid)>, + /// Whether a commit is uploading on behalf of the others. + pushing: Cell, +} + +impl Slot { + fn opened(object: Object, capture: Capture) -> Rc { + let acked = (object.epoch, object.tiered); + Rc::new(Self { + object: tokio::sync::Mutex::new(object), + capture: RefCell::new(capture), + acked: tokio::sync::watch::Sender::new(acked), + pushing: Cell::new(false), + }) + } + + /// The bucket holds everything through `through`. + fn tiered(&self, object: &mut Object, through: Txid, file: Option) { + object.tiered = object.tiered.max(through); + object.files.extend(file); + self.capture.borrow_mut().forget(through); + } + + fn ack(&self, epoch: Epoch, through: Txid) { + self.acked.send_if_modified(|acked| { + let more = (epoch, through) > *acked; + *acked = (*acked).max((epoch, through)); + more + }); + } +} + +/// The uploading commit's turn. Ending it, even by cancellation, wakes the +/// waiting commits so one of them takes the next. +struct Pushing<'a>(&'a Slot); + +impl Drop for Pushing<'_> { + fn drop(&mut self) { + self.0.pushing.set(false); + self.0.acked.send_modify(|_| {}); + } +} + +/// An object this node owns. +#[derive(Clone)] +struct Object { + ns: String, + actor: String, + epoch: Epoch, + replica: Replica, + /// How far the epoch's chain in the bucket is contiguous. + tiered: Txid, + /// The files of this epoch's chain. Only this owner writes the epoch's + /// prefix, so compaction deletes from here rather than listing it. + files: Vec, +} + +pub struct Owners { + bucket: Bucket, + objects: RefCell>>, + /// New objects' first epochs, opened at the claim, until their databases + /// open. + opened: RefCell>, + /// Objects workerd closed, until their eviction. + retiring: RefCell>>, + /// Evictions in flight. Objects written together go idle together, and + /// a node's lease renewals must not wait behind all their uploads. + evictions: tokio::sync::Semaphore, + /// Objects evicted with the bucket holding their epoch's chain: this + /// node's database file is that chain's state. + dormant: RefCell>, +} + +fn object_key(ns: &str, actor: &str) -> String { + format!("{ns}/{actor}") +} + +impl Owners { + pub fn new(bucket: Bucket) -> Self { + Self { + bucket, + objects: RefCell::new(HashMap::new()), + opened: RefCell::new(HashMap::new()), + retiring: RefCell::new(HashMap::new()), + evictions: tokio::sync::Semaphore::new(EVICT_CONCURRENCY), + dormant: RefCell::new(HashMap::new()), + } + } + + fn client(&self, ns: &str, actor: &str, epoch: Epoch) -> BucketClient { + let path = format!("{}do/{ns}/{actor}/ltx/e{epoch}", self.bucket.prefix()); + BucketClient::new(self.bucket.at(&path)) + } + + /// Take the object over at `owned.epoch` and open the epoch, leaving its + /// database at `path` ready for workerd to open. + pub async fn take_over( + &self, + ownership: &Ownership, + ns: &str, + actor: &str, + owned: &Owned, + path: &Path, + ) -> Result<()> { + let key = object_key(ns, actor); + if owned.recover.is_none() && !self.opened.borrow().contains_key(&key) { + if self.objects.borrow().contains_key(&key) { + return Ok(()); + } + // Woken where it was evicted, own.json unchanged: continue the + // epoch's chain over the file, with no write to the bucket. A + // dormant entry forgotten since the claim fails this attempt; + // the retry recovers at the next epoch. + let dormant = self.dormant.borrow_mut().remove(&key); + let Some(object) = dormant.filter(|o| o.epoch == owned.epoch && path.exists()) else { + return Err(Error::Contended(format!( + "{key}: not open here, nor dormant" + ))); + }; + let capture = Capture::resume(path, object.tiered); + tracing::info!( + "cluster: {ns}/{actor} epoch {} resumes at {}", + object.epoch, + object.tiered + ); + self.objects + .borrow_mut() + .insert(key, Slot::opened(object, capture)); + return Ok(()); + } + // Woken here from the epoch this node evicted: its database file is + // what the bucket holds, so the restore is skipped. The entry stays + // until the epoch opens, so a concurrent claim here also finds the + // object dormant. + let dormant = self.dormant.borrow().get(&key).map(|o| o.epoch); + let recover = owned.recover.filter(|&from| from != 0); + let local = path.exists() && recover.is_some_and(|from| dormant == Some(from)); + if !local { + remove_database(path)?; + if let Some(recover) = recover { + let from = self.client(ns, actor, recover); + let end = end_of(&from).await?; + tracing::info!( + "cluster: {ns}/{actor} epoch {} recovers {recover}: bucket through {end}", + owned.epoch + ); + // Nothing in the bucket: an object that committed nothing. + if end > 0 { + crate::ltx::restore(&from, path, TXID(end)).await?; + } + } + } + // The state the epoch inherits is its snapshot at txid 1, encoded + // from the file on the blocking pool; a new object has none. + let cache = open_cache(path, owned.epoch)?; + let snapshot = File { + level: 0, + first: 1, + last: 1, + size: 0, + }; + let size = if self.opened.borrow().contains_key(&key) { + None + } else { + let (path, out) = (path.to_owned(), cache.join(snapshot.name())); + blocking(move || crate::capture::snapshot_file(&path, &out)).await? + }; + let snapshot = size.map(|size| File { size, ..snapshot }); + let capture = Capture::resume(path, snapshot.map_or(0, |f| f.last)); + let object = self + .open_epoch(ownership, ns, actor, owned, cache, snapshot) + .await?; + self.dormant.borrow_mut().remove(&key); + self.objects + .borrow_mut() + .insert(key, Slot::opened(object, capture)); + Ok(()) + } + + /// Open a new object's first epoch at its claim, before a request + /// reaches it: create own.json, open, in one conditional write. A + /// claimant that loses the write answers `false` and routes to the + /// winner. Nothing to snapshot: an epoch that committed nothing restores + /// as an empty database. + pub async fn open_fresh( + &self, + ownership: &Ownership, + ns: &str, + actor: &str, + owned: &Owned, + ) -> Result { + if !ownership.open(ns, actor, owned.epoch).await? { + return Ok(false); + } + tracing::info!("cluster: {ns}/{actor} epoch {} opens new", owned.epoch); + self.opened + .borrow_mut() + .insert(object_key(ns, actor), owned.epoch); + Ok(true) + } + + /// Open a claimed epoch over the recovered state. Its chain starts with + /// the state's snapshot at txid 1 (none for an empty database), which + /// the bucket holds before the epoch opens. + async fn open_epoch( + &self, + ownership: &Ownership, + ns: &str, + actor: &str, + owned: &Owned, + cache: PathBuf, + snapshot: Option, + ) -> Result { + let mut object = Object { + ns: ns.to_owned(), + actor: actor.to_owned(), + epoch: owned.epoch, + replica: Replica { + client: self.client(ns, actor, owned.epoch), + cache, + }, + tiered: 0, + files: Vec::new(), + }; + if self + .opened + .borrow_mut() + .remove(&object_key(ns, actor)) + .is_some() + { + // A new object, opened at its claim. + return Ok(object); + } + if let Some(snapshot) = snapshot { + object.files.push(upload(&object.replica, snapshot).await?); + object.tiered = 1; + } + if !ownership.open(ns, actor, owned.epoch).await? { + return Err(Error::Contended(format!( + "{ns}/{actor}: epoch {} was superseded before it opened", + owned.epoch + ))); + } + tracing::info!("cluster: {ns}/{actor} epoch {} open", owned.epoch); + Ok(object) + } + + /// Capture every commit since the last call and make it durable in the + /// bucket. + /// + /// Commits upload as a group. Each captures its own transactions, then + /// one at a time uploads everything captured so far on behalf of all, + /// so a hot object pays an upload per group, not per transaction. + pub async fn commit(&self, ownership: &Ownership, ns: &str, actor: &str) -> Result<()> { + let slot = self.objects.borrow().get(&object_key(ns, actor)).cloned(); + let slot = + slot.ok_or_else(|| Error::Contended(format!("{ns}/{actor} is not open here")))?; + let target = { + let guard = slot.object.lock().await; + let target = slot.capture.borrow().last()?; + (guard.epoch, target) + }; + let mut acked = slot.acked.subscribe(); + loop { + if *acked.borrow_and_update() >= target { + return Ok(()); + } + if slot.pushing.replace(true) { + let _ = acked.changed().await; + continue; + } + let _pushing = Pushing(&slot); + self.push(ownership, &slot).await?; + } + } + + /// Upload everything captured, and acknowledge it if own.json still + /// names the epoch after the upload. + async fn push(&self, ownership: &Ownership, slot: &Slot) -> Result<()> { + let (ns, actor, epoch, round, staged, replica) = { + let guard = slot.object.lock().await; + let round = slot.capture.borrow().last()?; + let staged = stage(|t| slot.capture.borrow().entry(t), guard.tiered, round)?; + let (ns, actor) = (guard.ns.clone(), guard.actor.clone()); + (ns, actor, guard.epoch, round, staged, guard.replica.clone()) + }; + let file = put(&replica, 0, staged).await?; + { + let mut guard = slot.object.lock().await; + if guard.epoch == epoch { + slot.tiered(&mut guard, round, file); + } + } + if ownership.current_epoch(&ns, &actor).await? != Some(epoch) { + return Err(Error::Contended(format!( + "{ns}/{actor}: epoch {epoch} was superseded" + ))); + } + slot.ack(epoch, round); + Ok(()) + } + + /// Compact every owned object's chain that grew long enough. Runs off + /// the commit path. + pub async fn tier(&self) { + let objects: Vec<_> = self.objects.borrow().values().cloned().collect(); + stream::iter(objects) + .for_each_concurrent(TIER_CONCURRENCY, |object| async move { + let due = snapshot_due(&object.object.lock().await.files); + if due { + self.compact(&object).await; + } + }) + .await; + } + + /// Upload what the bucket lacks for one object. + async fn tier_one(&self, object: &Slot) { + let guard = object.object.lock().await; + let Ok(through) = object.capture.borrow().last() else { + return; + }; + let staged = stage(|t| object.capture.borrow().entry(t), guard.tiered, through); + let Ok(Some(staged)) = staged else { + return; + }; + let replica = guard.replica.clone(); + drop(guard); + let Ok(file) = put(&replica, 0, Some(staged)).await else { + return; + }; + let mut guard = object.object.lock().await; + object.tiered(&mut guard, through, file); + } + + /// Write a snapshot of an epoch's chain at `SNAPSHOT_LEVEL` and delete + /// what it covers, so a takeover's restore reads one file, not one per + /// upload. The snapshot merges the chain's files, never the live + /// database, so the merge runs off the event loop. + async fn compact(&self, slot: &Slot) { + let (replica, epoch, chain) = { + let guard = slot.object.lock().await; + let Some(chain) = chain(&guard.files) else { + return; + }; + (guard.replica.clone(), guard.epoch, chain) + }; + let Ok(snapshot) = merged(&replica, &chain).await else { + return; + }; + let last = snapshot.last; + let Ok(snapshot) = upload(&replica, snapshot).await else { + return; + }; + let replaced: Vec = { + let mut guard = slot.object.lock().await; + if guard.epoch != epoch { + return; + } + slot.tiered(&mut guard, last, Some(snapshot)); + let (replaced, kept) = std::mem::take(&mut guard.files) + .into_iter() + .partition(|f| f.last <= last && *f != snapshot); + guard.files = kept; + replaced + }; + let replica = &replica; + stream::iter(replaced) + .for_each_concurrent(TIER_CONCURRENCY, |f| async move { + let _ = std::fs::remove_file(replica.cache.join(f.name())); + let _ = replica + .client + .delete_ltx_file(f.level, TXID(f.first), TXID(f.last)) + .await; + }) + .await; + } + + /// Whether an object is open here, or opened by its claim. + pub fn holds(&self, ns: &str, actor: &str) -> bool { + let key = object_key(ns, actor); + self.objects.borrow().contains_key(&key) || self.opened.borrow().contains_key(&key) + } + + /// Stop serving an object workerd closed, keeping it for its eviction. + pub fn retire(&self, ns: &str, actor: &str) { + let key = object_key(ns, actor); + self.opened.borrow_mut().remove(&key); + if let Some(slot) = self.objects.borrow_mut().remove(&key) { + self.retiring.borrow_mut().insert(key, slot); + } + } + + /// Keep an object workerd evicted for being idle: upload what the bucket + /// lacks, and remember that this node's file holds the epoch. The record + /// still names this node, so other nodes route here, and the next + /// request wakes it without a restore. + pub async fn evict(&self, ns: &str, actor: &str, epoch: Epoch) { + let _permit = self.evictions.acquire().await; + let key = object_key(ns, actor); + let Some(slot) = self.retiring.borrow_mut().remove(&key) else { + return; + }; + self.tier_one(&slot).await; + let guard = slot.object.lock().await; + let last = slot.capture.borrow().last(); + if guard.epoch == epoch && last.is_ok_and(|last| guard.tiered >= last) { + // Bounded: a forgotten entry costs its object a restore. + let mut map = self.dormant.borrow_mut(); + if map.len() >= DORMANT_MAX { + map.clear(); + } + map.insert(key, guard.clone()); + } + } + + /// The epoch this node evicted an object at, with the bucket holding it. + pub fn dormant(&self, ns: &str, actor: &str) -> Option { + let dormant = self.dormant.borrow(); + dormant.get(&object_key(ns, actor)).map(|o| o.epoch) + } + + /// Forget an object that left this node. + pub fn close(&self, ns: &str, actor: &str) { + let key = object_key(ns, actor); + self.objects.borrow_mut().remove(&key); + self.opened.borrow_mut().remove(&key); + } + + /// An owned object's database was deleted: its chain continues with an + /// empty database, which the next commit makes durable. + pub fn database_reset(&self, ns: &str, actor: &str) { + if let Some(slot) = self.objects.borrow().get(&object_key(ns, actor)) { + slot.capture.borrow_mut().reset(); + } + } + + /// A write `SQLite` made to an owned object's WAL. + pub fn wal_write(&self, ns: &str, actor: &str, offset: u64, data: &[u8]) { + if let Some(slot) = self.objects.borrow().get(&object_key(ns, actor)) { + slot.capture.borrow_mut().write(offset, data); + } + } +} + +/// Where an epoch's chain in the bucket is contiguous from its start +/// through. +async fn end_of(client: &BucketClient) -> Result { + let mut end = 0; + for info in crate::ltx::plan(client, TXID(0)).await? { + if info.min_txid.0 > end + 1 { + break; + } + end = end.max(info.max_txid.0); + } + Ok(end) +} + +/// One LTX file covering txids `first..=last`, ready to upload. +struct Staged { + first: Txid, + last: Txid, + bytes: Vec, +} + +/// The capture's entries after `from` through `through`, merged into one +/// LTX file, as litestream compacts. +fn stage( + entry: impl Fn(Txid) -> Result>, + from: Txid, + through: Txid, +) -> Result> { + if through <= from { + return Ok(None); + } + let files = (from + 1..=through) + .map(entry) + .collect::>>()?; + Ok(Some(Staged { + first: from + 1, + last: through, + bytes: crate::ltx::ltx::merge_files(&files)?, + })) +} + +/// The files a restore reads, in order, as `ltx::plan` picks them; `None` +/// unless they run unbroken from txid 1. +fn chain(files: &[File]) -> Option> { + let mut files = files.to_vec(); + files.sort_by_key(|f| (f.first, std::cmp::Reverse(f.last))); + let mut reached = 0; + let mut chain = Vec::new(); + for f in files { + if f.first > reached + 1 { + return None; + } + if f.last > reached { + reached = f.last; + chain.push(f); + } + } + (!chain.is_empty()).then_some(chain) +} + +/// `chain`'s files merged into this node's copy of a snapshot covering +/// txid 1 through the chain's end, a page at a time on the blocking pool. +/// A missing copy is fetched from the bucket first. +async fn merged(replica: &Replica, chain: &[File]) -> Result { + let paths: Vec<_> = chain.iter().map(|f| replica.cache.join(f.name())).collect(); + for (f, path) in chain.iter().zip(&paths) { + if !path.exists() { + let (first, last, tmp) = (TXID(f.first), TXID(f.last), tmp_path(path)); + replica + .client + .open_ltx_file(f.level, first, last, &tmp) + .await?; + std::fs::rename(&tmp, path).map_err(crate::ltx::Error::from)?; + } + } + let snapshot = File { + level: SNAPSHOT_LEVEL, + first: 1, + last: chain.last().map_or(0, |f| f.last), + size: 0, + }; + let out = replica.cache.join(snapshot.name()); + let size = blocking(move || { + let io = |e| crate::Error::from(crate::ltx::Error::from(e)); + let files = paths.iter().map(std::fs::File::open); + let readers: Vec<_> = files + .map(|f| f.map(std::io::BufReader::new)) + .collect::>() + .map_err(io)?; + let tmp = tmp_path(&out); + let mut writer = std::io::BufWriter::new(std::fs::File::create(&tmp).map_err(io)?); + crate::ltx::ltx::merge_into(readers, &mut writer)?; + std::io::Write::flush(&mut writer).map_err(io)?; + std::fs::rename(&tmp, &out).map_err(io)?; + Ok(std::fs::metadata(&out).map_err(io)?.len()) + }) + .await?; + Ok(File { size, ..snapshot }) +} + +/// Writes a copy whole and renames it into place, so a reader never sees +/// part of one. +fn keep(path: &Path, bytes: &[u8]) -> std::io::Result<()> { + let tmp = tmp_path(path); + std::fs::write(&tmp, bytes)?; + std::fs::rename(&tmp, path) +} + +/// A temporary name beside `path`, unique within the process: a push and a +/// tiering pass can stage the same file at once. +fn tmp_path(path: &Path) -> PathBuf { + static NEXT: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + let n = NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let mut tmp = path.as_os_str().to_owned(); + tmp.push(format!(".{n}.tmp")); + tmp.into() +} + +async fn blocking(f: impl FnOnce() -> Result + Send + 'static) -> Result { + tokio::task::spawn_blocking(f) + .await + .map_err(|e| Error::Replica(format!("blocking task: {e}")))? +} + +/// Where an epoch's chain is: the bucket, and this node's copy of it. +#[derive(Clone)] +struct Replica { + client: BucketClient, + /// `.ltx/e`. + cache: PathBuf, +} + +/// The copies' directory for `epoch` of the database at `path`, created +/// empty. Other epochs' copies are removed: a file name repeats in every +/// epoch's chain. +fn open_cache(path: &Path, epoch: Epoch) -> Result { + let root = cache_root(path); + remove_dir(&root)?; + let cache = root.join(format!("e{epoch}")); + std::fs::create_dir_all(&cache).map_err(crate::ltx::Error::from)?; + Ok(cache) +} + +fn cache_root(path: &Path) -> PathBuf { + let mut root = path.as_os_str().to_owned(); + root.push(".ltx"); + root.into() +} + +fn remove_dir(path: &Path) -> Result<()> { + match std::fs::remove_dir_all(path) { + Err(e) if e.kind() != std::io::ErrorKind::NotFound => { + Err(crate::ltx::Error::from(e).into()) + } + _ => Ok(()), + } +} + +/// One LTX file in an epoch's chain. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +struct File { + level: i32, + first: Txid, + last: Txid, + size: u64, +} + +impl File { + fn name(&self) -> String { + format!( + "{:04x}-{:016x}-{:016x}.ltx", + self.level, self.first, self.last + ) + } +} + +/// L0 files a snapshot waits for, so a small object does not snapshot on +/// nearly every commit. +const SNAPSHOT_MIN_FILES: usize = 32; + +/// Whether `files` call for a snapshot: the L0 bytes since the last one +/// reach its size, so bucket writes stay within about twice +/// the log whatever the database's size. +fn snapshot_due(files: &[File]) -> bool { + let image: u64 = files + .iter() + .filter(|f| f.level == SNAPSHOT_LEVEL) + .map(|f| f.size) + .max() + .unwrap_or(0); + let l0: Vec<_> = files.iter().filter(|f| f.level == 0).collect(); + l0.len() >= SNAPSHOT_MIN_FILES && l0.iter().map(|f| f.size).sum::() >= image +} + +/// Uploads this node's copy of `file`. +async fn upload(replica: &Replica, file: File) -> Result { + let (first, last) = (TXID(file.first), TXID(file.last)); + let path = replica.cache.join(file.name()); + replica + .client + .write_ltx_file(file.level, first, last, &path) + .await?; + Ok(file) +} + +/// Keeps `staged`, if any, as this node's copy at `level`, then uploads it. +async fn put(replica: &Replica, level: i32, staged: Option) -> Result> { + let Some(Staged { first, last, bytes }) = staged else { + return Ok(None); + }; + let file = File { + level, + first, + last, + size: bytes.len() as u64, + }; + let path = replica.cache.join(file.name()); + blocking(move || Ok(keep(&path, &bytes).map_err(crate::ltx::Error::from)?)).await?; + upload(replica, file).await.map(Some) +} + +fn remove_file(path: &Path) -> Result<()> { + match std::fs::remove_file(path) { + Err(e) if e.kind() != std::io::ErrorKind::NotFound => { + Err(crate::ltx::Error::from(e).into()) + } + _ => Ok(()), + } +} + +/// A local copy is never authoritative at takeover: every acknowledged +/// commit is in the bucket. +fn remove_database(path: &Path) -> Result<()> { + for suffix in ["", "-wal", "-shm"] { + let mut file = path.as_os_str().to_owned(); + file.push(suffix); + remove_file(Path::new(&file))?; + } + remove_dir(&cache_root(path)) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn file(level: i32, size: u64) -> File { + File { + level, + first: 1, + last: 1, + size, + } + } + + fn span(level: i32, first: Txid, last: Txid) -> File { + File { + level, + first, + last, + size: 0, + } + } + + #[test] + fn a_chain_is_what_a_restore_reads() { + let snapshot = span(SNAPSHOT_LEVEL, 1, 4); + let (narrow, wide) = (span(0, 5, 6), span(0, 5, 8)); + let chain = chain(&[span(0, 9, 9), narrow, snapshot, wide]); + assert_eq!(chain, Some(vec![snapshot, wide, span(0, 9, 9)])); + assert_eq!(super::chain(&[snapshot, span(0, 6, 6)]), None, "a gap"); + assert_eq!(super::chain(&[]), None); + } + + #[test] + fn a_compaction_merges_the_chain() { + let ltx = |first: Txid, last: Txid, commit: u32, pages: &[(u32, u8)]| { + let header = crate::ltx::ltx::Header { + version: crate::ltx::ltx::VERSION, + flags: crate::ltx::ltx::HEADER_FLAG_NO_CHECKSUM, + page_size: 512, + commit, + min_txid: TXID(first), + max_txid: TXID(last), + ..Default::default() + }; + let pages: Vec<_> = pages.iter().map(|&(p, b)| (p, vec![b; 512])).collect(); + crate::ltx::ltx::encode_file(&header, &pages, 0).unwrap() + }; + let dir = std::env::temp_dir().join(format!("owner-test-{}", std::process::id())); + let replica = Replica { + client: BucketClient::new(Bucket::memory("e1")), + cache: open_cache(&dir.join("a.sqlite"), 1).unwrap(), + }; + let files = [ + (span(0, 1, 1), ltx(1, 1, 3, &[(1, 1), (2, 1), (3, 1)])), + (span(0, 2, 3), ltx(2, 3, 3, &[(2, 2)])), + (span(0, 4, 4), ltx(4, 4, 2, &[(1, 4)])), + ]; + let runtime = tokio::runtime::Builder::new_current_thread() + .build() + .unwrap(); + let snapshot = runtime.block_on(async { + // The first two only here, the last only in the bucket. + for (f, bytes) in &files[..2] { + std::fs::write(replica.cache.join(f.name()), bytes).unwrap(); + } + let (f, bytes) = &files[2]; + let (first, last, from) = (TXID(f.first), TXID(f.last), dir.join("up")); + std::fs::write(&from, bytes).unwrap(); + replica + .client + .write_ltx_file(f.level, first, last, &from) + .await + .unwrap(); + let chain: Vec<_> = files.iter().map(|(f, _)| *f).collect(); + merged(&replica, &chain).await.unwrap() + }); + let bytes = std::fs::read(replica.cache.join(snapshot.name())).unwrap(); + std::fs::remove_dir_all(&dir).unwrap(); + assert_eq!((snapshot.first, snapshot.last), (1, 4)); + let mut merge = crate::ltx::ltx::Merge::new(vec![bytes.as_slice()]).unwrap(); + assert_eq!(merge.header.commit, 2, "truncated by the last commit"); + let mut firsts = Vec::new(); + while let Some((pgno, page)) = merge.next().unwrap() { + firsts.push((pgno, page[0])); + } + assert_eq!(firsts, [(1, 4), (2, 2)]); + } + + #[test] + fn a_snapshot_is_due_when_the_log_reaches_its_size() { + let mut files = vec![file(SNAPSHOT_LEVEL, 1000)]; + files.extend((0..SNAPSHOT_MIN_FILES).map(|_| file(0, 30))); + assert!( + !snapshot_due(&files), + "960 bytes of log under a 1000-byte image" + ); + files.push(file(0, 40)); + assert!(snapshot_due(&files)); + let few: Vec<_> = (0..SNAPSHOT_MIN_FILES - 1) + .map(|_| file(0, 1 << 20)) + .collect(); + assert!(!snapshot_due(&few), "fewer files than the floor"); + } +} diff --git a/src/workerd/server/cluster/ownership.rs b/src/workerd/server/cluster/ownership.rs new file mode 100644 index 00000000000..c1b7cc41161 --- /dev/null +++ b/src/workerd/server/cluster/ownership.rs @@ -0,0 +1,642 @@ +// Copyright (c) 2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +use std::cell::Cell; +use std::cell::RefCell; +use std::collections::HashMap; +use std::rc::Rc; + +use serde::Deserialize; +use serde::Serialize; + +use crate::Bucket; +use crate::Error; +use crate::Guard; +use crate::Registry; +use crate::Result; +use crate::registry::NodeRecord; + +/// One Durable Object's record in `do///own.json`, the only +/// authority for who owns it and whose log holds its latest commits. +/// +/// `from` is the last epoch that opened. A claim writes the next `epoch` +/// and copies `from` from the record it replaced; the opening write sets +/// `from` to `epoch`. So an epoch is recovering while `from != epoch`, and +/// recovers `from`'s log (none when 0). +/// +/// `facets` is the object's facet tree index, `(parent, name)` by id from 1: +/// each facet's database is a log of its own under the object's epoch. +/// Every epoch carries the index forward, and an owner adds to it before a +/// new facet's log opens. +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +struct OwnerRecord { + node: String, + epoch: u64, + from: u64, + facets: Vec<(u32, String)>, +} + +impl OwnerRecord { + /// `node`'s claim on the next epoch. A claimant that died mid-recovery + /// leaves `from` unchanged. + fn successor(&self, node: &str) -> Self { + Self { + node: node.to_owned(), + epoch: self.epoch + 1, + from: self.from, + facets: self.facets.clone(), + } + } + + fn owned(self) -> Owned { + Owned { + recover: (self.from != self.epoch).then_some(self.from), + epoch: self.epoch, + facets: self.facets, + } + } +} + +/// This instance's claim on an object. +#[derive(Debug, PartialEq, Eq)] +pub struct Owned { + pub epoch: u64, + /// The epoch whose log a new owner recovers (0: none). `None` when this + /// instance already opened the epoch. + pub recover: Option, + /// The facet tree index, as own.json holds it. + pub facets: Vec<(u32, String)>, +} + +/// Who serves an object. +#[derive(Debug, PartialEq, Eq)] +pub enum Claim { + Owned(Owned), + /// A live peer owns it; send the request there. A `stale` route is + /// still used, and the caller refreshes it in the background. + Route { + node: String, + address: String, + stale: bool, + }, +} + +/// Claims, routes and releases Durable Objects through their owner records. +pub struct Ownership { + bucket: Bucket, + registry: Rc, + /// Whether this instance may claim objects: off once it drains. + pub may_claim: Cell, + /// Where each routed object's owner was, and when that was read. + routes: RefCell>, + /// The record each claim wrote, and the guard that it is still the + /// current one, so `open` changes it without reading it back. A claim + /// of an object with no record writes nothing: `open` creates the + /// record, open, in one conditional write, as celld claims a new cell. + claimed: RefCell>, +} + +struct Route { + node: String, + address: String, + read_ms: u64, +} + +/// How old a route gets before a use of it also rereads the owner record. +/// A route is used for as long as its node is in the registry listing, as +/// celld does: rereading on the request path put a bucket read on one +/// request in every interval, and dropping the route at a time limit does +/// nothing a dead owner's lease does not already do. A route to a live +/// former owner costs a hop, which that owner routes on. +const ROUTE_REFRESH_MS: u64 = 5_000; + +/// How many times a claim retries when another instance keeps winning. +const ATTEMPTS: usize = 8; + +fn owner_key(namespace: &str, actor: &str) -> String { + format!("do/{namespace}/{actor}/own.json") +} + +impl Ownership { + pub fn new(bucket: Bucket, registry: Rc) -> Self { + Self { + bucket, + registry, + may_claim: Cell::new(true), + routes: RefCell::new(HashMap::new()), + claimed: RefCell::new(HashMap::new()), + } + } + + /// Own the object, or learn which live peer does. A record with no owner, + /// or one naming an instance whose lease expired, is taken over with the + /// next epoch; the conditional write makes exactly one claimant win. + pub async fn acquire_or_route( + &self, + namespace: &str, + actor: &str, + now_ms: u64, + ) -> Result { + let key = owner_key(namespace, actor); + let me = self.registry.node(); + // A route whose node a stale listing no longer shows: its own record + // decides, not the object's. + let unlisted = self.routes.borrow().get(&key).map(|r| r.node.clone()); + if let Some(node) = unlisted + && !self.registry.listed_live(&node, now_ms) + { + self.registry.live(&node, now_ms).await?; + } + if let Some(route) = self.routes.borrow_mut().get_mut(&key) + && self.registry.listed_live(&route.node, now_ms) + { + // One refresh per interval: the first use past it takes it. + let stale = now_ms >= route.read_ms + ROUTE_REFRESH_MS; + if stale { + route.read_ms = now_ms; + } + return Ok(Claim::Route { + node: route.node.clone(), + address: route.address.clone(), + stale, + }); + } + for _ in 0..ATTEMPTS { + let (previous, guard) = match self.bucket.get(&key).await? { + None => (OwnerRecord::default(), Guard::Absent), + Some((body, version)) => { + let record: OwnerRecord = serde_json::from_slice(&body)?; + if record.node == me { + return Ok(Claim::Owned(record.owned())); + } + if !record.node.is_empty() + && let Some(peer) = self.registry.live(&record.node, now_ms).await? + { + return Ok(self.route(key, peer, now_ms)); + } + (record, Guard::Match(version)) + } + }; + if !self.may_claim.get() { + return Err(Error::Replica(format!("{key}: this node is draining"))); + } + let record = previous.successor(me); + // A new object opens at its claim, where a loser still routes + // (`open_fresh`). + let fresh = matches!(guard, Guard::Absent); + let written = if fresh { + Some(Guard::Absent) + } else { + let body = serde_json::to_vec(&record)?; + let version = self.bucket.put_if(&key, body, guard).await?; + version.map(Guard::Match) + }; + if let Some(guard) = written { + self.claimed + .borrow_mut() + .insert(key, (record.clone(), guard)); + return Ok(Claim::Owned(record.owned())); + } + } + Err(Error::Contended(key)) + } + + fn route(&self, key: String, peer: NodeRecord, now_ms: u64) -> Claim { + let claim = Claim::Route { + node: peer.node.clone(), + address: peer.address.clone(), + stale: false, + }; + let route = Route { + node: peer.node, + address: peer.address, + read_ms: now_ms, + }; + self.routes.borrow_mut().insert(key, route); + claim + } + + /// Reread a stale route's owner record. The route goes if no live peer + /// owns the object any more; the next request then claims or routes. + pub async fn refresh_route(&self, namespace: &str, actor: &str, now_ms: u64) -> Result<()> { + let key = owner_key(namespace, actor); + let node = match self.bucket.get(&key).await? { + Some((body, _)) => serde_json::from_slice::(&body)?.node, + None => String::new(), + }; + let peer = if node.is_empty() || node == self.registry.node() { + None + } else { + self.registry.live(&node, now_ms).await? + }; + match peer { + Some(peer) => drop(self.route(key, peer, now_ms)), + None => drop(self.routes.borrow_mut().remove(&key)), + } + Ok(()) + } + + /// Open the epoch this instance claimed and recovered. Answers whether + /// the record still named this claim; if not, the epoch never opens. + pub async fn open(&self, namespace: &str, actor: &str, epoch: u64) -> Result { + let key = owner_key(namespace, actor); + let claimed = self.claimed.borrow_mut().remove(&key); + let (mut record, guard) = if let Some(claimed) = claimed { + claimed + } else { + let Some((body, version)) = self.bucket.get(&key).await? else { + return Ok(false); + }; + (serde_json::from_slice(&body)?, Guard::Match(version)) + }; + if record.node != self.registry.node() || record.epoch != epoch || record.from == epoch { + return Ok(false); + } + record.from = epoch; + let body = serde_json::to_vec(&record)?; + Ok(self.bucket.put_if(&key, body, guard).await?.is_some()) + } + + /// Move this instance's open epoch to the next, which recovers the one + /// it leaves like any other. `None` when the record no longer names this + /// open epoch. + pub async fn advance(&self, namespace: &str, actor: &str, epoch: u64) -> Result> { + let key = owner_key(namespace, actor); + let Some((body, version)) = self.bucket.get(&key).await? else { + return Ok(None); + }; + let previous: OwnerRecord = serde_json::from_slice(&body)?; + if previous.node != self.registry.node() + || previous.epoch != epoch + || previous.from != epoch + { + return Ok(None); + } + let record = previous.successor(&previous.node); + let body = serde_json::to_vec(&record)?; + let Some(version) = self + .bucket + .put_if(&key, body, Guard::Match(version)) + .await? + else { + return Ok(None); + }; + self.claimed + .borrow_mut() + .insert(key, (record.clone(), Guard::Match(version))); + Ok(Some(record.owned())) + } + + /// Record a longer facet index for this instance's open epoch, before a + /// new facet's log opens: a takeover restores the facets own.json names. + /// `false` when the record no longer names this open epoch. + pub async fn set_facets( + &self, + namespace: &str, + actor: &str, + epoch: u64, + facets: &[(u32, String)], + ) -> Result { + let key = owner_key(namespace, actor); + for _ in 0..ATTEMPTS { + let Some((body, version)) = self.bucket.get(&key).await? else { + return Ok(false); + }; + let mut record: OwnerRecord = serde_json::from_slice(&body)?; + if record.node != self.registry.node() || record.epoch != epoch || record.from != epoch + { + return Ok(false); + } + if record.facets.len() >= facets.len() { + return Ok(true); + } + record.facets = facets.to_vec(); + let body = serde_json::to_vec(&record)?; + if self + .bucket + .put_if(&key, body, Guard::Match(version)) + .await? + .is_some() + { + return Ok(true); + } + } + Err(Error::Contended(key)) + } + + /// The epoch own.json names now: the re-read after a bucket upload that + /// makes the upload an acknowledgement (spec: `TierAck`). + pub async fn current_epoch(&self, namespace: &str, actor: &str) -> Result> { + Ok(match self.bucket.get(&owner_key(namespace, actor)).await? { + None => None, + Some((body, _)) => Some(serde_json::from_slice::(&body)?.epoch), + }) + } +} + +#[cfg(test)] +mod tests { + #![allow(clippy::unwrap_used)] + + use std::rc::Rc; + + use futures::executor::block_on; + + use crate::Bucket; + use crate::Claim; + use crate::Error; + use crate::Ownership; + use crate::Registry; + use crate::ownership::OwnerRecord; + use crate::ownership::owner_key; + + const TTL: u64 = 10_000; + + /// Matches any `Owned` claim at `epoch`. + fn owned(epoch: u64) -> EpochOf { + EpochOf(epoch) + } + + #[derive(Debug)] + struct EpochOf(u64); + + impl PartialEq for Claim { + fn eq(&self, other: &EpochOf) -> bool { + matches!(self, Self::Owned(o) if o.epoch == other.0) + } + } + + fn node(store: &Bucket, name: &str) -> (Rc, Ownership) { + let bucket = store.at("cluster"); + let registry = Rc::new(Registry::new( + bucket.clone(), + name, + &format!("{name}:1"), + TTL, + )); + (registry.clone(), Ownership::new(bucket, registry)) + } + + /// `own` claims and opens a new object `x` at epoch 1: a new object's + /// record appears when its epoch opens. + async fn open_x(own: &Ownership) { + let claim = own.acquire_or_route("ns", "x", 0).await; + assert_eq!(claim.unwrap(), owned(1)); + assert!(own.open("ns", "x", 1).await.unwrap()); + } + + #[test] + fn peers_are_the_live_registrations() { + block_on(async { + let store = Bucket::memory(""); + let (a, _) = node(&store, "a"); + let (b, _) = node(&store, "b"); + a.renew(0).await.unwrap(); + b.renew(5_000).await.unwrap(); + let names = |peers: Vec| { + peers.into_iter().map(|peer| peer.node).collect::>() + }; + assert_eq!(names(a.peers(1_000).await.unwrap()), ["a", "b"]); + // a's lease ran out at 10s; b's holds until 15s. + assert_eq!(names(a.peers(12_000).await.unwrap()), ["b"]); + b.deregister().await.unwrap(); + assert!(a.peers(12_000).await.unwrap().is_empty()); + }); + } + + #[test] + fn a_rewritten_node_record_fences_its_instance() { + block_on(async { + let store = Bucket::memory(""); + let (a, _) = node(&store, "a"); + let (impostor, _) = node(&store, "a"); + a.renew(0).await.unwrap(); + // A second process with the same identity cannot publish over it. + assert!(matches!(impostor.renew(0).await, Err(Error::Fenced(_)))); + a.renew(1_000).await.unwrap(); + }); + } + + #[test] + fn a_claim_owns_an_unowned_object_and_routes_to_a_live_owner() { + block_on(async { + let store = Bucket::memory(""); + let (a, a_own) = node(&store, "a"); + let (b, b_own) = node(&store, "b"); + a.renew(0).await.unwrap(); + b.renew(0).await.unwrap(); + open_x(&a_own).await; + // Asking again keeps it, at the same epoch. + assert_eq!( + a_own.acquire_or_route("ns", "x", 1).await.unwrap(), + owned(1) + ); + assert_eq!( + b_own.acquire_or_route("ns", "x", 1).await.unwrap(), + Claim::Route { + node: "a".into(), + address: "a:1".into(), + stale: false, + } + ); + }); + } + + #[test] + fn a_stale_listing_keeps_a_route_to_a_live_owner() { + block_on(async { + let store = Bucket::memory(""); + let (a, a_own) = node(&store, "a"); + let (b, b_own) = node(&store, "b"); + a.renew(0).await.unwrap(); + b.renew(0).await.unwrap(); + open_x(&a_own).await; + let route = b_own.acquire_or_route("ns", "x", 1).await; + assert!(matches!(route.unwrap(), Claim::Route { .. })); + // a renews, but b lists nothing more: what b knows of a expires. + a.renew(8_000).await.unwrap(); + // With the record gone a reread would claim x, so a route shows + // that only a's own record was read. + store.delete("cluster/do/ns/x/own.json").await.unwrap(); + assert_eq!( + b_own.acquire_or_route("ns", "x", 12_000).await.unwrap(), + Claim::Route { + node: "a".into(), + address: "a:1".into(), + stale: true, + } + ); + }); + } + + #[test] + fn a_dead_owners_object_is_taken_over_at_the_next_epoch() { + block_on(async { + let store = Bucket::memory(""); + let (a, a_own) = node(&store, "a"); + let (b, b_own) = node(&store, "b"); + a.renew(0).await.unwrap(); + b.renew(0).await.unwrap(); + open_x(&a_own).await; + // a stops renewing; b renews past a's expiry and takes x over. + b.renew(9_000).await.unwrap(); + assert_eq!( + b_own.acquire_or_route("ns", "x", 11_000).await.unwrap(), + owned(2) + ); + assert_eq!( + a_own.acquire_or_route("ns", "x", 11_000).await.unwrap(), + Claim::Route { + node: "b".into(), + address: "b:1".into(), + stale: false, + } + ); + }); + } + + #[test] + fn an_owner_advances_only_its_open_epoch() { + block_on(async { + let store = Bucket::memory(""); + let (a, a_own) = node(&store, "a"); + a.renew(0).await.unwrap(); + open_x(&a_own).await; + // The next epoch recovers the one it left, and does not advance + // until it opens. + let next = a_own.advance("ns", "x", 1).await.unwrap().unwrap(); + assert_eq!((next.epoch, next.recover), (2, Some(1))); + assert!(a_own.advance("ns", "x", 2).await.unwrap().is_none()); + assert!(a_own.open("ns", "x", 2).await.unwrap()); + // A record changed since is not advanced over. + vacate(&store).await; + assert!(a_own.advance("ns", "x", 2).await.unwrap().is_none()); + }); + } + + /// Clear the owner of `x`, as another writer might, under a cached read. + async fn vacate(store: &Bucket) { + let bucket = store.at("cluster"); + let key = owner_key("ns", "x"); + let (body, _) = bucket.get(&key).await.unwrap().unwrap(); + let mut record: OwnerRecord = serde_json::from_slice(&body).unwrap(); + record.node = String::new(); + bucket + .put(&key, serde_json::to_vec(&record).unwrap(), &[]) + .await + .unwrap(); + } + + #[test] + fn concurrent_claims_open_exactly_one_epoch() { + block_on(async { + let store = Bucket::memory(""); + let nodes: Vec<_> = (0..8).map(|i| node(&store, &format!("n{i}"))).collect(); + for (registry, _) in &nodes { + registry.renew(0).await.unwrap(); + } + let claims = futures::future::join_all( + nodes + .iter() + .map(|(_, own)| own.acquire_or_route("ns", "x", 0)), + ) + .await; + // A new object's claims all succeed; the conditional write that + // opens the epoch has exactly one winner. + assert!(claims.iter().all(|c| matches!(c, Ok(Claim::Owned(_))))); + let opens = + futures::future::join_all(nodes.iter().map(|(_, own)| own.open("ns", "x", 1))) + .await; + let owners = opens.iter().filter(|o| matches!(o, Ok(true))).count(); + assert_eq!(owners, 1); + }); + } + + #[test] + fn a_takeover_recovers_the_last_epoch_that_opened() { + block_on(async { + let store = Bucket::memory(""); + let (a, a_own) = node(&store, "a"); + let (b, b_own) = node(&store, "b"); + let (c, c_own) = node(&store, "c"); + for r in [&a, &b, &c] { + r.renew(0).await.unwrap(); + } + let Claim::Owned(first) = a_own.acquire_or_route("ns", "x", 0).await.unwrap() else { + panic!("a should own x") + }; + assert_eq!(first.recover, Some(0)); + assert!(a_own.open("ns", "x", 1).await.unwrap()); + // Only the claimant, at its epoch, opens. + assert!(!b_own.open("ns", "x", 1).await.unwrap()); + + // a's lease runs out; b claims epoch 2 and dies before opening. + b.renew(9_000).await.unwrap(); + c.renew(9_000).await.unwrap(); + let Claim::Owned(second) = b_own.acquire_or_route("ns", "x", 11_000).await.unwrap() + else { + panic!("b should own x") + }; + assert_eq!(second.recover, Some(1)); + + // c takes over from b's unopened epoch: it still recovers epoch 1. + c.renew(20_000).await.unwrap(); + let Claim::Owned(third) = c_own.acquire_or_route("ns", "x", 21_000).await.unwrap() + else { + panic!("c should own x") + }; + assert_eq!(third.epoch, 3); + assert_eq!(third.recover, Some(1)); + }); + } + + #[test] + fn a_listed_owners_route_is_used_and_refreshed_in_the_background() { + block_on(async { + let store = Bucket::memory(""); + let (a, a_own) = node(&store, "a"); + let (b, b_own) = node(&store, "b"); + a.renew(0).await.unwrap(); + b.renew(0).await.unwrap(); + b.peers(0).await.unwrap(); + open_x(&a_own).await; + let to_a = |stale| Claim::Route { + node: "a".into(), + address: "a:1".into(), + stale, + }; + let route = async |now| b_own.acquire_or_route("ns", "x", now).await.unwrap(); + assert_eq!(route(1_000).await, to_a(false)); + // x loses its owner; b still routes to a until it refreshes. + vacate(&store).await; + assert_eq!(route(5_000).await, to_a(false)); + // The first use past the interval is stale, and only the first. + assert_eq!(route(6_000).await, to_a(true)); + assert_eq!(route(6_001).await, to_a(false)); + // The refresh finds no owner, so b claims x. + b_own.refresh_route("ns", "x", 6_002).await.unwrap(); + assert_eq!(route(6_003).await, owned(2)); + }); + } + + #[test] + fn a_route_ends_with_its_owners_listed_lease() { + block_on(async { + let store = Bucket::memory(""); + let (a, a_own) = node(&store, "a"); + let (b, b_own) = node(&store, "b"); + a.renew(0).await.unwrap(); + b.renew(0).await.unwrap(); + b.peers(0).await.unwrap(); + open_x(&a_own).await; + b_own.acquire_or_route("ns", "x", 1).await.unwrap(); + // a dies; b's route goes when a's listed lease ends. + b.renew(9_000).await.unwrap(); + assert_eq!( + b_own.acquire_or_route("ns", "x", 11_000).await.unwrap(), + owned(2) + ); + }); + } +} diff --git a/src/workerd/server/cluster/registry.rs b/src/workerd/server/cluster/registry.rs new file mode 100644 index 00000000000..dacdca3e20b --- /dev/null +++ b/src/workerd/server/cluster/registry.rs @@ -0,0 +1,316 @@ +// Copyright (c) 2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +use std::cell::RefCell; +use std::collections::HashMap; + +use serde::Deserialize; +use serde::Serialize; + +use crate::Bucket; +use crate::Error; +use crate::Guard; +use crate::Result; +use crate::Version; + +/// One instance's record in `nodes/.json`. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct NodeRecord { + /// The instance's identity: its cluster public key, hex. + pub node: String, + /// Where peers reach it: its sockaddr's bytes, hex. + pub address: String, + /// The instance is live until this time and must renew before it. + pub expires_ms: u64, + /// The process that wrote it. + #[serde(default)] + pub instance: u64, +} + +/// This instance's membership: it publishes its own record and reads its +/// peers'. A peer whose lease expired is dead; nothing else is. +pub struct Registry { + bucket: Bucket, + node: String, + address: String, + ttl_ms: u64, + /// The version of our own record we last wrote, so a renewal replaces + /// exactly that and notices anyone else's write. + own: RefCell>, + /// Random per process: a renewal whose answer was lost may still have + /// landed, and this tells that write from another writer's. + instance: u64, + /// Live peers' records, as of the last listing. + listed: RefCell>, +} + +fn node_key(node: &str) -> String { + format!("nodes/{node}.json") +} + +impl Registry { + pub fn new(bucket: Bucket, node: &str, address: &str, ttl_ms: u64) -> Self { + Self { + bucket, + node: node.to_owned(), + address: address.to_owned(), + ttl_ms, + own: RefCell::new(None), + instance: std::hash::BuildHasher::hash_one( + &std::collections::hash_map::RandomState::new(), + std::time::SystemTime::now(), + ), + listed: RefCell::new(HashMap::new()), + } + } + + pub fn node(&self) -> &str { + &self.node + } + + /// Where peer `node` listens, as of the last listing. + pub fn address(&self, node: &str) -> Option { + self.listed.borrow().get(node).map(|r| r.address.clone()) + } + + /// Publish or renew this instance's record; answers the new expiry. The + /// first write requires the key to be absent (the node id is fresh per + /// process), and every later one requires our last version, so a record + /// someone else wrote fences this process instead of being overwritten. + pub async fn renew(&self, now_ms: u64) -> Result { + let record = NodeRecord { + node: self.node.clone(), + address: self.address.clone(), + expires_ms: now_ms + self.ttl_ms, + instance: self.instance, + }; + let key = node_key(&self.node); + let body = serde_json::to_vec(&record)?; + let mut guard = self + .own + .borrow() + .clone() + .map_or(Guard::Absent, Guard::Match); + loop { + if let Some(version) = self.bucket.put_if(&key, body.clone(), guard).await? { + self.own.replace(Some(version)); + return Ok(record.expires_ms); + } + // Not the version we know. If this process wrote it, earlier + // renewals landed and only their answers were lost (timed-out + // rounds on the fleet), and one may land between our read and + // write: retry. Anything else is another writer. + let Some((current, version)) = self.bucket.get(&key).await? else { + return Err(Error::Fenced(self.node.clone())); + }; + let current: NodeRecord = serde_json::from_slice(¤t)?; + if current.instance != self.instance { + return Err(Error::Fenced(self.node.clone())); + } + guard = Guard::Match(version); + } + } + + /// Remove this instance's record on a clean shutdown, so peers stop + /// routing to it before its lease would have run out. + pub async fn deregister(&self) -> Result<()> { + self.own.take(); + self.bucket.delete(&node_key(&self.node)).await + } + + /// Whether `node` held a lease past `now_ms` at the last listing. No + /// I/O: a route checks it on every use. + pub fn listed_live(&self, node: &str, now_ms: u64) -> bool { + self.listed + .borrow() + .get(node) + .is_some_and(|record| record.expires_ms > now_ms) + } + + /// A peer's record while its lease holds, from the last listing when it + /// was there: routing asks on every request, and a GET per request was + /// most of a routed request's time. + pub async fn live(&self, node: &str, now_ms: u64) -> Result> { + if let Some(record) = self.listed.borrow().get(node) + && record.expires_ms > now_ms + { + return Ok(Some(record.clone())); + } + self.fetch(node, now_ms).await + } + + async fn fetch(&self, node: &str, now_ms: u64) -> Result> { + let Some((body, _)) = self.bucket.get(&node_key(node)).await? else { + return Ok(None); + }; + let record: NodeRecord = serde_json::from_slice(&body)?; + if record.expires_ms + DEAD_RECORD_TTLS * self.ttl_ms < now_ms { + // A fenced or crashed instance leaves its record, every restart + // adds one, and every listing read them all: 40 after an + // afternoon of fleet tests. One this long expired belongs to a + // process that fenced long ago; were it still running, its next + // renewal's conditional write would fail and fence it. + self.bucket.delete(&node_key(node)).await?; + } + if record.expires_ms <= now_ms { + return Ok(None); + } + // Until the next listing: one read serves every request that finds + // the listing stale, and a stale listing is when the bucket is busy. + self.listed + .borrow_mut() + .insert(node.to_owned(), record.clone()); + Ok(Some(record)) + } + + /// Every live instance, this one included when its lease holds. + pub async fn peers(&self, now_ms: u64) -> Result> { + // The records are read concurrently: a lease round is bounded, and + // reading them one after another made it time out on R2's spikes. + let listed = self.bucket.list("nodes/").await?; + let nodes = listed.iter().filter_map(|entry| { + entry + .key + .strip_prefix("nodes/") + .and_then(|key| key.strip_suffix(".json")) + }); + let fetched = futures::future::join_all(nodes.map(|node| self.fetch(node, now_ms))).await; + let mut peers = Vec::new(); + for record in fetched { + peers.extend(record?); + } + *self.listed.borrow_mut() = peers.iter().map(|p| (p.node.clone(), p.clone())).collect(); + Ok(peers) + } +} + +/// How many lease lifetimes past its expiry a node record is deleted. +const DEAD_RECORD_TTLS: u64 = 10; + +#[cfg(test)] +mod tests { + #![allow(clippy::unwrap_used)] + + use super::*; + + fn block_on(f: F) -> F::Output { + tokio::runtime::Builder::new_current_thread() + .build() + .unwrap() + .block_on(f) + } + + #[test] + fn a_renewal_whose_answer_was_lost_is_not_a_fence() { + block_on(async { + let bucket = Bucket::memory(""); + let a = Registry::new(bucket.clone(), "a", "a:1", 10_000); + a.renew(0).await.unwrap(); + // The renewal at 1_000 lands, but its answer never arrives. + let landed = NodeRecord { + node: "a".into(), + address: "a:1".into(), + expires_ms: 11_000, + instance: a.instance, + }; + let body = serde_json::to_vec(&landed).unwrap(); + bucket.put(&node_key("a"), body, &[]).await.unwrap(); + assert_eq!(a.renew(2_000).await.unwrap(), 12_000); + // Someone else's write is still a fence. + let other = NodeRecord { + instance: a.instance + 1, + ..landed + }; + let body = serde_json::to_vec(&other).unwrap(); + bucket.put(&node_key("a"), body, &[]).await.unwrap(); + assert!(matches!(a.renew(3_000).await, Err(Error::Fenced(_)))); + }); + } + + /// A write's key and body. + type Write = (String, Vec); + + /// Lands `late`, a write of ours a timed-out round left in flight, just + /// after serving the next GET. + #[derive(Default)] + struct Late { + inner: crate::bucket::memory::Memory, + late: std::rc::Rc>>, + } + + #[async_trait::async_trait(?Send)] + impl crate::bucket::Store for Late { + async fn get( + &self, + key: &str, + to: Option<&std::path::Path>, + ) -> Result, Version)>> { + let got = self.inner.get(key, to).await; + if let Some((key, body)) = self.late.take() { + let body = crate::bucket::Payload::Bytes(body.into()); + self.inner.put(&key, body, None, &[]).await?; + } + got + } + async fn put( + &self, + key: &str, + body: crate::bucket::Payload, + guard: Option, + metadata: &[(&str, String)], + ) -> Result> { + self.inner.put(key, body, guard, metadata).await + } + async fn list(&self, prefix: &str) -> Result> { + self.inner.list(prefix).await + } + async fn delete(&self, key: &str) -> Result<()> { + self.inner.delete(key).await + } + } + + #[test] + fn two_late_renewals_of_ours_are_not_a_fence() { + block_on(async { + let store = Late::default(); + let late = store.late.clone(); + let bucket = Bucket::new(store, ""); + let a = Registry::new(bucket.clone(), "a", "a:1", 10_000); + a.renew(0).await.unwrap(); + let ours = |expires_ms| { + let record = NodeRecord { + node: "a".into(), + address: "a:1".into(), + expires_ms, + instance: a.instance, + }; + serde_json::to_vec(&record).unwrap() + }; + // Two timed-out rounds landed: one before this renewal's first + // read, one just after it. + bucket.put(&node_key("a"), ours(11_000), &[]).await.unwrap(); + late.replace(Some((node_key("a"), ours(12_000)))); + assert_eq!(a.renew(3_000).await.unwrap(), 13_000); + }); + } + + #[test] + fn a_listing_deletes_records_long_expired() { + block_on(async { + let bucket = Bucket::memory(""); + let a = Registry::new(bucket.clone(), "a", "a:1", 10_000); + let b = Registry::new(bucket.clone(), "b", "b:1", 10_000); + let c = Registry::new(bucket.clone(), "c", "c:1", 10_000); + a.renew(0).await.unwrap(); + b.renew(90_000).await.unwrap(); + c.renew(115_000).await.unwrap(); + // a expired at 10s, b at 100s: at 115s only a is ten lives dead. + let live = c.peers(115_000).await.unwrap(); + assert_eq!(live.iter().map(|p| &*p.node).collect::>(), ["c"]); + let keys: Vec<_> = bucket.list("nodes/").await.unwrap(); + let keys: Vec<_> = keys.iter().map(|e| &*e.key).collect(); + assert_eq!(keys, ["nodes/b.json", "nodes/c.json"]); + }); + } +} diff --git a/src/workerd/server/cluster/s3.rs b/src/workerd/server/cluster/s3.rs new file mode 100644 index 00000000000..f062979e6ff --- /dev/null +++ b/src/workerd/server/cluster/s3.rs @@ -0,0 +1,552 @@ +// Copyright (c) 2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +//! The four S3 calls the cluster makes, signed with `SigV4`. Requests use +//! path-style URLs, which S3, R2 and `MinIO` all accept. + +use std::cell::Cell; +use std::cell::RefCell; +use std::fmt::Write; +use std::path::Path; +use std::path::PathBuf; +use std::time::Duration; +use std::time::SystemTime; +use std::time::UNIX_EPOCH; + +use crate::Error; +use crate::Result; +use crate::bucket::Entry; +use crate::bucket::Guard; +use crate::bucket::Store; +use crate::bucket::Version; +use crate::hex; +use crate::hmac_sha256; +use crate::http::Http; +use crate::http::HttpResponse; +use crate::http::Payload; +use crate::sha256; + +pub struct S3 { + http: Http, + endpoint: String, + bucket: String, + region: String, + credentials: RefCell, + /// `aws_credentials_file`, reread when its modification time changes. + file: Option, + /// This node's wall clock less the bucket's, at the last response. + skew_ms: Cell>, +} + +#[derive(Clone, Default)] +struct Credentials { + key_id: String, + secret: String, + token: Option, + modified: Option, +} + +impl Credentials { + /// `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` and `AWS_SESSION_TOKEN` + /// lines, as an env file writes them. + fn parse(text: &str, modified: Option) -> Option { + let mut out = Self { + modified, + ..Self::default() + }; + for line in text.lines() { + let line = line.trim().trim_start_matches("export "); + let Some((name, value)) = line.split_once('=') else { + continue; + }; + let value = value.trim().trim_matches(['\'', '"']).to_owned(); + match name.trim() { + "AWS_ACCESS_KEY_ID" => out.key_id = value, + "AWS_SECRET_ACCESS_KEY" => out.secret = value, + "AWS_SESSION_TOKEN" => out.token = Some(value), + _ => {} + } + } + (!out.key_id.is_empty() && !out.secret.is_empty()).then_some(out) + } +} + +impl S3 { + /// `url` is `s3:///`; answers the client and the prefix. + /// Options are `aws_endpoint`, `aws_region`, `aws_access_key_id`, + /// `aws_secret_access_key` and `aws_token`, or in place of the last three + /// `aws_credentials_file`, so that rotating short-lived credentials + /// needs no restart. Each defaults to its AWS environment variable + /// (`AWS_ENDPOINT_URL`, `AWS_SESSION_TOKEN`, else upper-cased). + pub fn new(url: &str, options: &[(String, String)]) -> Result<(Self, String)> { + let rest = url + .strip_prefix("s3://") + .ok_or_else(|| Error::Config(format!("{url}: expected s3:///")))?; + let (bucket, prefix) = rest.split_once('/').unwrap_or((rest, "")); + // An option missing from the config falls back to the environment + // variable the AWS tools read. + let option = |name: &str| { + let var = match name { + "aws_endpoint" => "AWS_ENDPOINT_URL".to_owned(), + "aws_token" => "AWS_SESSION_TOKEN".to_owned(), + _ => name.to_uppercase(), + }; + options + .iter() + .find(|(key, _)| key == name) + .map(|(_, value)| value.clone()) + .or_else(|| std::env::var(var).ok()) + }; + let required = |name: &str| option(name).ok_or_else(|| Error::Config(name.to_owned())); + let region = option("aws_region").unwrap_or_else(|| "us-east-1".into()); + let endpoint = option("aws_endpoint") + .unwrap_or_else(|| format!("https://s3.{region}.amazonaws.com")) + .trim_end_matches('/') + .to_owned(); + let file = option("aws_credentials_file").map(PathBuf::from); + let credentials = match &file { + Some(path) => read_credentials(path) + .ok_or_else(|| Error::Config(format!("{}: no credentials", path.display())))?, + None => Credentials { + key_id: required("aws_access_key_id")?, + secret: required("aws_secret_access_key")?, + token: option("aws_token"), + modified: None, + }, + }; + Ok(( + Self { + http: Http::new().map_err(|e| Error::Config(e.to_string()))?, + endpoint, + bucket: bucket.to_owned(), + region, + credentials: RefCell::new(credentials), + file, + skew_ms: Cell::new(None), + }, + prefix.to_owned(), + )) + } +} + +#[async_trait::async_trait(?Send)] +impl Store for S3 { + async fn get(&self, key: &str, to: Option<&Path>) -> Result, Version)>> { + let response = self.send("GET", key, &[], Vec::new(), none(), to).await?; + match response.status { + 200 => Ok(Some((response.body, Version(response.etag)))), + 404 => Ok(None), + _ => Err(failure("GET", key, &response)), + } + } + + /// `Ok(None)` when the guard failed and nothing was written. + async fn put( + &self, + key: &str, + body: Payload, + guard: Option, + metadata: &[(&str, String)], + ) -> Result> { + let mut headers: Vec<(String, String)> = metadata + .iter() + .map(|(name, value)| (format!("x-amz-meta-{name}"), value.clone())) + .collect(); + match guard { + None => {} + Some(Guard::Absent) => headers.push(("if-none-match".into(), "*".into())), + Some(Guard::Match(Version(etag))) => headers.push(("if-match".into(), etag)), + } + let response = self.send("PUT", key, &[], headers, body, None).await?; + match response.status { + 200 => Ok(Some(Version(response.etag))), + // 412: the guard failed. 409: a concurrent conditional write to + // the same key won the race, which is the same answer. + 409 | 412 => Ok(None), + _ => Err(failure("PUT", key, &response)), + } + } + + async fn delete(&self, key: &str) -> Result<()> { + let response = self + .send("DELETE", key, &[], Vec::new(), none(), None) + .await?; + match response.status { + 200 | 204 | 404 => Ok(()), + _ => Err(failure("DELETE", key, &response)), + } + } + + async fn list(&self, prefix: &str) -> Result> { + let mut entries = Vec::new(); + let mut token: Option = None; + loop { + let mut query = vec![("list-type", "2".to_owned()), ("prefix", prefix.to_owned())]; + if let Some(token) = token.take() { + query.push(("continuation-token", token)); + } + let response = self + .send("GET", "", &query, Vec::new(), none(), None) + .await?; + if response.status != 200 { + return Err(failure("LIST", prefix, &response)); + } + let xml = String::from_utf8_lossy(&response.body); + for contents in elements(&xml, "Contents") { + let key = elements(contents, "Key").next().map(unescape); + entries.extend(key.map(|key| Entry { key })); + } + match elements(&xml, "NextContinuationToken").next() { + Some(next) if elements(&xml, "IsTruncated").next() == Some("true") => { + token = Some(unescape(next)); + } + _ => return Ok(entries), + } + } + } + + /// How far this node's wall clock is outside the bucket's, from the last + /// response's `Date` header: 0 when the header fits the request's window. + fn clock_skew_ms(&self) -> Option { + self.skew_ms.get() + } +} + +impl S3 { + /// Sign and send one request. `key` is the object key, or empty for a + /// bucket-level call. + /// R2 answers 429 to more than about one write a second to one key, and + /// an owner record is written twice in a row when an object reclaims: + /// under a 10,000-object kill, four reclaims failed that way and dropped + /// their objects. A refused request did nothing, conditional or not, so + /// it is retried after a pause. + async fn send( + &self, + method: &str, + key: &str, + query: &[(&str, String)], + headers: Vec<(String, String)>, + body: Payload, + to: Option<&Path>, + ) -> Result { + let mut pause = Duration::from_millis(250); + loop { + let response = self + .send_once(method, key, query, headers.clone(), body.clone(), to) + .await?; + if !matches!(response.status, 429 | 503) || pause > Duration::from_secs(2) { + return Ok(response); + } + tokio::time::sleep(pause).await; + pause *= 2; + } + } + + async fn send_once( + &self, + method: &str, + key: &str, + query: &[(&str, String)], + mut headers: Vec<(String, String)>, + body: Payload, + to: Option<&Path>, + ) -> Result { + let path = format!("/{}/{}", encode(&self.bucket, false), encode(key, false)); + let mut query: Vec = query + .iter() + .map(|(name, value)| format!("{}={}", encode(name, true), encode(value, true))) + .collect(); + query.sort(); + let query = query.join("&"); + + let host = self + .endpoint + .split_once("://") + .map_or(self.endpoint.as_str(), |(_, host)| host); + self.refresh(); + let credentials = self.credentials.borrow().clone(); + let (date, time) = amz_date(SystemTime::now()); + // The payload is not hashed: TLS protects it, and hashing would mean + // a second pass over every LTX file. + let mut signed = vec![ + ("host".to_owned(), host.to_owned()), + ( + "x-amz-content-sha256".to_owned(), + "UNSIGNED-PAYLOAD".to_owned(), + ), + ("x-amz-date".to_owned(), time.clone()), + ]; + if let Some(token) = &credentials.token { + signed.push(("x-amz-security-token".to_owned(), token.clone())); + } + signed.sort(); + let names = signed + .iter() + .map(|(name, _)| name.as_str()) + .collect::>() + .join(";"); + let canonical = format!( + "{method}\n{path}\n{query}\n{}\n{names}\nUNSIGNED-PAYLOAD", + signed.iter().fold(String::new(), |out, (name, value)| out + + name + + ":" + + value + + "\n"), + ); + let scope = format!("{date}/{}/s3/aws4_request", self.region); + let to_sign = format!( + "AWS4-HMAC-SHA256\n{time}\n{scope}\n{}", + hex(&sha256(canonical.as_bytes())) + ); + let mut signing = format!("AWS4{}", credentials.secret).into_bytes(); + for part in [date.as_str(), &self.region, "s3", "aws4_request"] { + signing = hmac_sha256(&signing, part.as_bytes()); + } + let signature = hex(&hmac_sha256(&signing, to_sign.as_bytes())); + headers.extend(signed); + headers.push(( + "authorization".into(), + format!( + "AWS4-HMAC-SHA256 Credential={}/{scope}, SignedHeaders={names}, Signature={signature}", + credentials.key_id + ), + )); + + let url = if query.is_empty() { + format!("{}{path}", self.endpoint) + } else { + format!("{}{path}?{query}", self.endpoint) + }; + let headers = headers.into_iter().flat_map(<[String; 2]>::from).collect(); + // A hung request would stall whatever waits on it: a tiering pass + // waits for every object, so one could stop a node's tiering. A + // large body gets a second per MiB on top, at least. + let size = match &body { + Payload::Bytes(bytes) => bytes.len() as u64, + Payload::File(path) => std::fs::metadata(path).map_or(0, |m| m.len()), + }; + let head = REQUEST_TIMEOUT + Duration::from_secs(size >> 20); + let sent = SystemTime::now(); + let response = self + .http + .send( + method.into(), + url, + headers, + body, + to, + (head, REQUEST_TIMEOUT), + ) + .await + .map_err(|error| Error::Http(format!("{method} {key}: {error}")))?; + if let Some(date) = http_date_ms(&response.date) { + // The bucket stamped the header, in whole seconds, between the + // send and the answer: this clock is off only by what lies + // outside that window. Measured from the send alone, a write + // R2 held 8 s at its 5-minute mark read as a clock 8 s slow. + let (sent, received) = (unix_ms(sent), unix_ms(SystemTime::now())); + let skew = if received < date { + received - date + } else if sent > date + 1000 { + sent - date - 1000 + } else { + 0 + }; + self.skew_ms.set(Some(skew)); + } + Ok(response) + } + + /// Rereads the credentials file if it changed. A file caught mid-write + /// or emptied keeps the credentials in use. + fn refresh(&self) { + let Some(path) = &self.file else { + return; + }; + let modified = std::fs::metadata(path).and_then(|m| m.modified()).ok(); + if modified.is_none() || modified == self.credentials.borrow().modified { + return; + } + if let Some(credentials) = read_credentials(path) { + *self.credentials.borrow_mut() = credentials; + } + } +} + +fn read_credentials(path: &std::path::Path) -> Option { + let modified = std::fs::metadata(path).and_then(|m| m.modified()).ok(); + Credentials::parse(&std::fs::read_to_string(path).ok()?, modified) +} + +fn none() -> Payload { + Payload::Bytes(bytes::Bytes::new()) +} + +/// How long one bucket request may take. R2 answers in 0.1-1 s from the +/// lab, with spikes of a few seconds. +const REQUEST_TIMEOUT: Duration = Duration::from_secs(20); + +fn failure(call: &str, key: &str, response: &HttpResponse) -> Error { + let body = String::from_utf8_lossy(&response.body); + let code = elements(&body, "Code").next().unwrap_or(""); + Error::Http(format!("{call} {key}: {} {code}", response.status)) +} + +/// `SigV4` URI encoding: everything but unreserved characters, and `/` too +/// unless the value is a path. +fn encode(value: &str, slash: bool) -> String { + value.bytes().fold(String::new(), |mut out, byte| { + if byte.is_ascii_alphanumeric() || b"-_.~".contains(&byte) || (byte == b'/' && !slash) { + out.push(char::from(byte)); + } else { + let _ = write!(out, "%{byte:02X}"); + } + out + }) +} + +/// The text of each `…` element, in order. S3's listing XML has +/// no attributes on the elements read here and no nesting of equal names. +fn elements<'a>(xml: &'a str, name: &str) -> impl Iterator { + let open = format!("<{name}>"); + let close = format!(""); + let mut rest = xml; + std::iter::from_fn(move || { + let start = rest.find(&open)? + open.len(); + let end = start + rest[start..].find(&close)?; + let text = &rest[start..end]; + rest = &rest[end + close.len()..]; + Some(text) + }) +} + +fn unescape(text: &str) -> String { + text.replace("<", "<") + .replace(">", ">") + .replace(""", "\"") + .replace("'", "'") + .replace("&", "&") +} + +/// Days since 1970-01-01 to a civil date (Howard Hinnant's algorithm). +fn civil(days: i64) -> (i64, u32, u32) { + let z = days + 719_468; + let era = z.div_euclid(146_097); + let doe = z.rem_euclid(146_097); + let yoe = (doe - doe / 1460 + doe / 36_524 - doe / 146_096) / 365; + let doy = doe - (365 * yoe + yoe / 4 - yoe / 100); + let mp = (5 * doy + 2) / 153; + let day = u32::try_from(doy - (153 * mp + 2) / 5 + 1).unwrap_or(1); + let month = u32::try_from(if mp < 10 { mp + 3 } else { mp - 9 }).unwrap_or(1); + (yoe + era * 400 + i64::from(month <= 2), month, day) +} + +fn unix_ms(t: SystemTime) -> i64 { + i64::try_from(t.duration_since(UNIX_EPOCH).unwrap_or_default().as_millis()).unwrap_or(0) +} + +/// An RFC 7231 date, `Mon, 12 Oct 2026 23:10:55 GMT`, in ms since the epoch. +fn http_date_ms(date: &str) -> Option { + httpdate::parse_http_date(date).ok().map(unix_ms) +} + +/// `(YYYYMMDD, YYYYMMDDTHHMMSSZ)`. +fn amz_date(now: SystemTime) -> (String, String) { + let secs = + i64::try_from(now.duration_since(UNIX_EPOCH).unwrap_or_default().as_secs()).unwrap_or(0); + let (year, month, day) = civil(secs.div_euclid(86_400)); + let s = secs.rem_euclid(86_400); + let date = format!("{year:04}{month:02}{day:02}"); + let time = format!("{date}T{:02}{:02}{:02}Z", s / 3600, s % 3600 / 60, s % 60); + (date, time) +} + +/// Unix milliseconds as Go's `time.RFC3339Nano` prints them: trailing zeros +/// of the fraction dropped, and no fraction at all when it is zero. +pub fn format_rfc3339_nano(unix_millis: i64) -> String { + let millis = unix_millis.max(0); + let secs = millis / 1000; + let (year, month, day) = civil(secs / 86_400); + let s = secs % 86_400; + let mut out = format!( + "{year:04}-{month:02}-{day:02}T{:02}:{:02}:{:02}", + s / 3600, + s % 3600 / 60, + s % 60 + ); + let fraction = millis % 1000; + if fraction != 0 { + let digits = format!("{fraction:03}"); + out = out + "." + digits.trim_end_matches('0'); + } + out + "Z" +} + +#[cfg(test)] +mod tests { + #![allow(clippy::unwrap_used)] + + use super::*; + + #[test] + fn dates_round_trip() { + let t = UNIX_EPOCH + Duration::from_millis(1_790_348_414_500); + assert_eq!( + format_rfc3339_nano(1_790_348_414_500), + "2026-09-25T15:00:14.5Z" + ); + assert_eq!( + format_rfc3339_nano(1_790_348_414_000), + "2026-09-25T15:00:14Z" + ); + assert_eq!(amz_date(t).1, "20260925T150014Z"); + assert_eq!(civil(11_016), (2000, 2, 29)); + } + + #[test] + fn http_dates() { + assert_eq!( + http_date_ms("Mon, 12 Oct 2026 23:10:55 GMT"), + Some(1_791_846_655_000) + ); + assert_eq!( + http_date_ms("Tue, 29 Feb 2000 00:00:00 GMT"), + Some(951_782_400_000) + ); + assert_eq!(http_date_ms("nonsense"), None); + } + + #[test] + fn credentials_from_an_env_file() { + let c = Credentials::parse( + "# r2\nexport AWS_ACCESS_KEY_ID='k'\nAWS_SECRET_ACCESS_KEY=\"s\"\n\ + AWS_SESSION_TOKEN=t=\nS3_ENDPOINT=https://x\n", + None, + ) + .unwrap(); + assert_eq!( + (&*c.key_id, &*c.secret, c.token.as_deref()), + ("k", "s", Some("t=")) + ); + assert!(Credentials::parse("AWS_ACCESS_KEY_ID=k\n", None).is_none()); + } + + #[test] + fn listing_xml() { + let xml = "true\ + a&b3\ + t"; + let keys: Vec<_> = elements(xml, "Contents") + .map(|c| unescape(elements(c, "Key").next().unwrap())) + .collect(); + assert_eq!(keys, ["a&b"]); + assert_eq!(elements(xml, "NextContinuationToken").next(), Some("t")); + } + + #[test] + fn sigv4_encoding() { + assert_eq!(encode("ns/a b+c", false), "ns/a%20b%2Bc"); + assert_eq!(encode("ns/a", true), "ns%2Fa"); + } +} diff --git a/src/workerd/server/factory/worker-factory-actor.c++ b/src/workerd/server/factory/worker-factory-actor.c++ index f55c0785511..34b3dae33b4 100644 --- a/src/workerd/server/factory/worker-factory-actor.c++ +++ b/src/workerd/server/factory/worker-factory-actor.c++ @@ -18,26 +18,43 @@ namespace workerd::server { ActorStorage::Impl::Impl(const WorkerFactory& factory, kj::Maybe> directoryParam, kj::String uniqueKey, - ::rust::Box actors) - : directory(kj::mv(directoryParam)), + ::rust::Box actorsParam) + : actors(kj::mv(actorsParam)), + directory(kj::mv(directoryParam)), vfsDirectory(directory.map([](kj::Own& d) { return d->clone(); }) .orDefault([&]() { return kj::newInMemoryDirectory(kj::systemPreciseCalendarClock()); })), - vfs(*vfsDirectory), + vfs(*vfsDirectory, vfsOptions()), + schedulerVfs(*vfsDirectory), uniqueKey(kj::mv(uniqueKey)), alarmScheduler(kj::heap(kj::systemPreciseCalendarClock(), factory.getImpl().timer, - vfs, + schedulerVfs, kj::Path({"metadata.sqlite"}), - [uniqueKey = this->uniqueKey.asPtr(), actors = kj::mv(actors)]( - const ActorKey& actor) mutable -> kj::Own { + [this](const ActorKey& actor) -> kj::Own { // The id carries the persisted name so that `ctx.id.name` is set when the alarm runs. - auto id = ActorIdFactoryImpl(uniqueKey).idFromStringNamed( - kj::str(actor.actorId), actor.name.map([](kj::StringPtr n) { return kj::str(n); })); + auto id = ActorIdFactoryImpl(this->uniqueKey) + .idFromStringNamed(kj::str(actor.actorId), + actor.name.map([](kj::StringPtr n) { return kj::str(n); })); return actors->actor_for_alarm(kj::heap(kj::mv(id))); })) {} +SqliteDatabase::VfsOptions ActorStorage::Impl::vfsOptions() { + SqliteDatabase::VfsOptions options; + if (actors->replicated()) { + // A clustered namespace's files open only while this instance's lease holds, and their WAL + // writes are how the cluster captures commits. + options.afterOpen = [this]() { actors->check_open(); }; + options.walWrite = [this]( + kj::StringPtr file, uint64_t offset, kj::ArrayPtr data) { + actors->wal_write( + toRust(file), offset, ::rust::Slice(data.begin(), data.size())); + }; + } + return options; +} + kj::Maybe> ActorStorage::Impl::getFacetTreeIndexIfNotEmpty( const kj::Directory& dir, kj::StringPtr rootKey) { return dir.tryOpenFile(kj::Path({kj::str(rootKey, ".facets")}), kj::WriteMode::MODIFY) @@ -234,6 +251,9 @@ class ActorSqliteHooks final: public ActorSqlite::Hooks { kj::Promise scheduleRun( kj::Maybe newAlarmTime, kj::Promise priorTask) override { KJ_IF_SOME(scheduledTime, newAlarmTime) { + // TODO(cluster): an alarm must follow its object between instances. + JSG_REQUIRE( + !storage->actors->replicated(), Error, "Alarms are not yet supported in cluster mode."); storage->alarmScheduler->setAlarm(actor, scheduledTime); } else { storage->alarmScheduler->deleteAlarm(actor); @@ -261,10 +281,11 @@ class FacetAlarmHooks final: public ActorSqlite::Hooks { Worker::Actor::MakeActorCacheFunc makeActorCacheFunc(kj::Own storage, const ActorStorageSpec& spec, kj::String rootKey, - kj::Maybe actorName) { + kj::Maybe actorName, + kj::Maybe<::rust::Box> committer) { return [storage = kj::mv(storage), durable = spec.durable, facetId = spec.facet_id, - rootKey = kj::mv(rootKey), - actorName = kj::mv(actorName)](const ActorCache::SharedLru& sharedLru, + rootKey = kj::mv(rootKey), actorName = kj::mv(actorName), + committer = kj::mv(committer)](const ActorCache::SharedLru& sharedLru, OutputGate& outputGate, ActorCache::Hooks& hooks, SqliteObserver& sqliteObserver) mutable -> kj::Maybe> { if (!durable) return kj::none; @@ -290,12 +311,20 @@ Worker::Actor::MakeActorCacheFunc makeActorCacheFunc(kj::Own db->run("PRAGMA journal_mode=WAL;"); db->afterReset([storage = kj::addRef(*storage), rootKey = kj::str(rootKey), selfId]( SqliteDatabase& db) mutable { + storage->actors->database_reset(toRust(rootKey)); db.run("PRAGMA journal_mode=WAL;"); storage->deleteDescendantStorage(*KJ_ASSERT_NONNULL(storage->directory), rootKey, selfId); }); - return kj::heap(kj::mv(db), outputGate, - [](SpanParent) -> kj::Promise { return kj::READY_NOW; }, *sqliteHooks) - .attach(kj::mv(sqliteHooks)); + kj::Function(SpanParent)> commit = [](SpanParent) -> kj::Promise { + return kj::READY_NOW; + }; + KJ_IF_SOME(c, committer) { + commit = [c = kj::mv(c)]( + SpanParent) -> kj::Promise { return result_task_run(c->commit()); }; + committer = kj::none; + } + auto sqlite = kj::heap(kj::mv(db), outputGate, kj::mv(commit), *sqliteHooks); + return sqlite.attach(kj::mv(sqliteHooks)); }; } @@ -390,8 +419,10 @@ kj::Own worker_new_actor(const CompiledWorker& worker, KJ_IF_SOME(doId, actorId.tryGet>()) { actorName = doId->getName().map([](kj::StringPtr n) { return kj::str(n); }); } - auto makeActorCache = makeActorCacheFunc( - kj::addRef(storage.getImpl()), spec, kj::str(spec.root_key), kj::mv(actorName)); + kj::Maybe<::rust::Box> committer; + if (spec.replicated) committer = hooks->committer(); + auto makeActorCache = makeActorCacheFunc(kj::addRef(storage.getImpl()), spec, + kj::str(spec.root_key), kj::mv(actorName), kj::mv(committer)); auto makeStorage = [enableSql = spec.enable_sql](jsg::Lock& js, const Worker::Api& api, ActorCacheInterface& actorCache) -> jsg::Ref { @@ -475,6 +506,14 @@ void actor_reset_storage(const ActorHandle& actor) { } } +void actor_close_storage(const ActorHandle& actor) { + KJ_IF_SOME(cache, actor.getActor().getPersistent()) { + KJ_IF_SOME(db, cache.getSqliteDatabase()) { + db.close(); + } + } +} + kj::Maybe> actor_hibernation_manager(const ActorHandle& actor) { return actor.getActor().getHibernationManager().map( [](HibernationManager& m) { return kj::Rc(m.addRef()); }); diff --git a/src/workerd/server/factory/worker-factory-impl.h b/src/workerd/server/factory/worker-factory-impl.h index 00b4a28ed01..c3c7cedaf29 100644 --- a/src/workerd/server/factory/worker-factory-impl.h +++ b/src/workerd/server/factory/worker-factory-impl.h @@ -215,9 +215,13 @@ class ActorStorage::Impl final: public kj::Refcounted { kj::String uniqueKey, ::rust::Box actors); + // Declared first: the VFS's hooks and the alarm scheduler call into it. + ::rust::Box actors; kj::Maybe> directory; kj::Own vfsDirectory; // the in-memory directory when there is none SqliteDatabase::Vfs vfs; + // The scheduler's database is no actor's, so it opens without the cluster's VFS hooks. + SqliteDatabase::Vfs schedulerVfs; kj::String uniqueKey; kj::Own alarmScheduler; @@ -227,6 +231,7 @@ class ActorStorage::Impl final: public kj::Refcounted { const kj::Directory& dir, kj::StringPtr rootKey); kj::Path getSqlitePath(kj::StringPtr rootKey, uint facetId, kj::StringPtr suffix = ""_kj); + SqliteDatabase::VfsOptions vfsOptions(); void deleteFacet( const kj::Directory& dir, kj::StringPtr rootKey, FacetTreeIndex& index, uint facetId); void deleteDescendantStorage(const kj::Directory& dir, kj::StringPtr rootKey, uint parentId); @@ -239,14 +244,24 @@ class ActorStorage::Impl final: public kj::Refcounted { class RpcClient::Impl final { public: - Impl(const WorkerFactory& factory, kj::Own connection) + // A connection and its RPC system, shared by the capabilities obtained over it. + struct Session: public kj::Refcounted { + explicit Session(kj::Own connection) + : connection(kj::mv(connection)), + rpcSystem(*this->connection) {} + kj::Own connection; + capnp::TwoPartyClient rpcSystem; + }; + + Impl(const WorkerFactory& factory, kj::Own session, rpc::WorkerdBootstrap::Client target) : factory(factory), - connection(kj::mv(connection)), - rpcSystem(*this->connection) {} + session(kj::mv(session)), + target(kj::mv(target)) {} const WorkerFactory& factory; - kj::Own connection; - capnp::TwoPartyClient rpcSystem; + kj::Own session; + // Where events go: the session's bootstrap, or an actor obtained over it. + rpc::WorkerdBootstrap::Client target; }; // The `WorkerdBootstrap` a peer sees over capnp: every event it starts goes to `service`. diff --git a/src/workerd/server/factory/worker-factory-rpc.c++ b/src/workerd/server/factory/worker-factory-rpc.c++ index feeeec6b7c1..5a989c0cbf7 100644 --- a/src/workerd/server/factory/worker-factory-rpc.c++ +++ b/src/workerd/server/factory/worker-factory-rpc.c++ @@ -178,15 +178,28 @@ RpcClient::Impl& RpcClient::getImpl() const { } kj::Own new_rpc_client(const WorkerFactory& factory, kj::Own stream) { - return kj::heap(kj::heap(factory, kj::mv(stream))); + auto session = kj::refcounted(kj::mv(stream)); + auto target = session->rpcSystem.bootstrap().castAs(); + return kj::heap(kj::heap(factory, kj::mv(session), kj::mv(target))); +} + +kj::Own rpc_client_debug_actor( + const RpcClient& session, ::rust::Str service, ::rust::Str className, ::rust::Str actorId) { + auto& impl = session.getImpl(); + auto peer = impl.session->rpcSystem.bootstrap().castAs(); + auto request = peer.getActorRequest(); + request.setService(kj::str(service)); + request.setEntrypoint(kj::str(className)); + request.setActorId(kj::str(actorId)); + return kj::heap(kj::heap( + impl.factory, kj::addRef(*impl.session), request.send().getActor())); } kj::Promise rpc_client_custom_event(const RpcClient& client, kj::Own event, kj::Maybe<::rust::Str> cfBlobJson) { auto& impl = client.getImpl(); - auto bootstrap = impl.rpcSystem.bootstrap().castAs(); - auto request = bootstrap.startEventRequest(capnp::MessageSize{4, 0}); + auto request = impl.target.startEventRequest(capnp::MessageSize{4, 0}); KJ_IF_SOME(cf, cfBlobJson) request.setCfBlobJson(kj::str(cf)); auto dispatcher = request.send().getDispatcher(); // Workerd-to-workerd RPC does not support `restore()`, so Frankenvalues need no handler. @@ -196,8 +209,20 @@ kj::Promise rpc_client_custom_event(const RpcCl co_return rust::worker::CustomEventResult{.outcome = rust::worker::toRustOutcome(result.outcome)}; } +kj::Own rpc_client_start_request( + const RpcClient& client, kj::Own metadata) { + auto& impl = client.getImpl(); + auto request = impl.target.startEventRequest(capnp::MessageSize{4, 0}); + KJ_IF_SOME(cf, metadata->cfBlobJson) request.setCfBlobJson(cf); + request.setFromPersistentStub(metadata->fromPersistentStub == Persistent::YES); + auto dispatcher = request.send().getDispatcher(); + auto& factory = impl.factory.getImpl(); + return kj::heap(factory.httpOverCapnpFactory, factory.byteStreamFactory, + getUnsupportedFrankenvalueHandler(), kj::mv(dispatcher)); +} + kj::Promise rpc_client_on_disconnect(const RpcClient& client) { - return client.getImpl().rpcSystem.onDisconnect(); + return client.getImpl().session->rpcSystem.onDisconnect(); } // ======================================================================================= diff --git a/src/workerd/server/factory/worker-factory.c++ b/src/workerd/server/factory/worker-factory.c++ index aab6ec55ff9..e03e2333645 100644 --- a/src/workerd/server/factory/worker-factory.c++ +++ b/src/workerd/server/factory/worker-factory.c++ @@ -85,7 +85,9 @@ WorkerFactory::Impl::Impl(WorkerFactory& factory, timer, entropySource, headerTableBuilder, httpOverCapnpFactory, byteStreamFactory), headerTable(buildHeaderTable(headerTableBuilder, this->config)), memoryCacheProvider(kj::heap()), - channelTokenHandler(factory), + channelTokenHandler(factory, + this->config.hasCluster() ? kj::Maybe(this->config.getCluster().getKey()) + : kj::none), tasks(*this) { // The config's logging settings go on top of the caller's. auto& logging = this->options->loggingOptions; @@ -1403,10 +1405,13 @@ MainModuleIsPython isPythonMainModule(config::Worker::Reader conf) { // Compiles a config worker's compatibility flags into `arena`, as the worker's errors. CompatibilityFlags::Reader compileFlags(const WorkerFactory::Options& options, + kj::StringPtr name, config::Worker::Reader conf, capnp::MallocMessageBuilder& arena, ErrorReporter& errorReporter) { auto flags = arena.initRoot(); + // Built-in services' Workers need `experimental`, as under Miniflare (`sql.prepare()`). + bool experimental = options.experimental || name.startsWith("workerd:"); KJ_IF_SOME(overrideDate, options.testCompatibilityDateOverride) { if (conf.hasCompatibilityDate()) { errorReporter.addError( @@ -1416,11 +1421,11 @@ CompatibilityFlags::Reader compileFlags(const WorkerFactory::Options& options, } // FUTURE_FOR_TEST admits any valid date, such as 2999-12-31. compileCompatibilityFlags(overrideDate, conf.getCompatibilityFlags(), flags, errorReporter, - options.experimental, CompatibilityDateValidation::FUTURE_FOR_TEST, nullptr, + experimental, CompatibilityDateValidation::FUTURE_FOR_TEST, nullptr, isPythonMainModule(conf)); } else if (conf.hasCompatibilityDate()) { compileCompatibilityFlags(conf.getCompatibilityDate(), conf.getCompatibilityFlags(), flags, - errorReporter, options.experimental, CompatibilityDateValidation::CODE_VERSION, nullptr, + errorReporter, experimental, CompatibilityDateValidation::CODE_VERSION, nullptr, isPythonMainModule(conf)); } else KJ_IF_SOME(defaultDate, options.defaultCompatibilityDate) { compileCompatibilityFlags(defaultDate, conf.getCompatibilityFlags(), flags, errorReporter, @@ -1533,7 +1538,7 @@ kj::Promise> factory_new_worker(const WorkerFactory& con auto conf = factory.config.getServices()[index].getWorker(); extensions = factory.config.getExtensions(); impl->flagsArena = kj::heap(); - featureFlags = compileFlags(options, conf, *impl->flagsArena, errorReporter); + featureFlags = compileFlags(options, impl->name, conf, *impl->flagsArena, errorReporter); source = WorkerdApi::extractSource(impl->name, conf, featureFlags, errorReporter); if (conf.hasModuleFallback()) moduleFallback = conf.getModuleFallback(); compileBindings = [globals = kj::heapArray(kj::from(spec.globals))]( diff --git a/src/workerd/server/factory/worker-factory.h b/src/workerd/server/factory/worker-factory.h index e8b9574e867..7f7cb08396d 100644 --- a/src/workerd/server/factory/worker-factory.h +++ b/src/workerd/server/factory/worker-factory.h @@ -54,6 +54,8 @@ struct SpawnedTask; struct ActorClassChannel; struct ActorNamespaceHandle; struct ActorHooks; +struct Committer; +struct ResultTask; struct AbortReason; struct KeepAlive; struct WorkerInterfaceList; @@ -384,11 +386,17 @@ class RpcClient final { mutable kj::Own impl; }; +void actor_close_storage(const ActorHandle& actor); + kj::Own new_rpc_client(const WorkerFactory& factory, kj::Own stream); +kj::Own rpc_client_debug_actor( + const RpcClient& session, ::rust::Str service, ::rust::Str className, ::rust::Str actorId); kj::Promise rpc_client_custom_event(const RpcClient& client, kj::Own event, kj::Maybe<::rust::Str> cfBlobJson); kj::Promise rpc_client_on_disconnect(const RpcClient& client); +kj::Own rpc_client_start_request( + const RpcClient& client, kj::Own metadata); uint16_t factory_start_inspector(const WorkerFactory& factory, ::rust::Str address); diff --git a/src/workerd/server/server/BUILD.bazel b/src/workerd/server/server/BUILD.bazel index 7de9a202fb2..6eaf5929d79 100644 --- a/src/workerd/server/server/BUILD.bazel +++ b/src/workerd/server/server/BUILD.bazel @@ -35,6 +35,8 @@ wd_rust_crate( "//src/workerd/io:bench_capnp_rust", "//src/workerd/server:compiled-bindings_capnp_rust", "//src/workerd/server:workerd_capnp_rust", + "//src/workerd/server/cluster", + "//src/workerd/server/cluster/logic:cluster-logic", "@crates_vendor//:base64", "@crates_vendor//:capnp", "@crates_vendor//:data-encoding", @@ -46,6 +48,7 @@ wd_rust_crate( "@crates_vendor//:ipnet", "@crates_vendor//:percent-encoding", "@crates_vendor//:rand", + "@crates_vendor//:ring", "@crates_vendor//:rustls", "@crates_vendor//:serde", "@crates_vendor//:serde_json", diff --git a/src/workerd/server/server/actor.rs b/src/workerd/server/server/actor.rs index afeb9774aa1..b174149baf3 100644 --- a/src/workerd/server/server/actor.rs +++ b/src/workerd/server/server/actor.rs @@ -21,6 +21,7 @@ use std::cell::Cell; use std::cell::OnceCell; use std::cell::RefCell; +use std::collections::HashMap; use std::rc::Rc; use std::rc::Weak; use std::time::Duration; @@ -55,6 +56,11 @@ use crate::channels::SubrequestChannel; use crate::channels::TokenUsage; use crate::channels::WorkerInterface; use crate::channels::attach; +use crate::cluster::ClusterNode; +use crate::cluster::Committer; +use crate::cluster::Owned; +use crate::cluster::Owner; +use crate::cluster::RpcConnection; use crate::config::Factory; use crate::tasks::TaskHandle; @@ -76,6 +82,8 @@ const EVICT_TIMEOUT_MESSAGE: &str = pub struct ActorNamespace { this: Weak, factory: Rc, + /// The worker service the class belongs to, which a peer names to reach an actor. + service: String, class_name: String, config: ActorConfig, actor_class: Rc, @@ -91,6 +99,27 @@ pub struct ActorNamespace { actors: RefCell>>, /// The cleanup loop, started by the first request; dropped with the namespace. cleanup_task: RefCell>, + /// Set in cluster mode for a namespace on disk: its actors are owned by one instance of + /// the cluster at a time, and replicated. + cluster: OnceCell>, + /// The namespace's directory, once linked on disk. + directory: OnceCell, + /// Actors being resolved in the cluster, each woken when its first resolution ends. + resolving: RefCell>>, +} + +/// The first resolution of an actor: its end, however it ends, wakes the requests that waited. +struct Resolving<'a> { + pending: &'a RefCell>>, + key: String, +} + +impl Drop for Resolving<'_> { + fn drop(&mut self) { + if let Some(done) = self.pending.borrow_mut().remove(&self.key) { + done.notify_waiters(); + } + } } impl ActorNamespace { @@ -101,6 +130,7 @@ impl ActorNamespace { /// Workflow's props. `persistent_self_tokens` is that class's worker's flag. pub fn new( factory: Rc, + service: String, class_name: String, config: ActorConfig, actor_class: Rc, @@ -109,6 +139,7 @@ impl ActorNamespace { Rc::new_cyclic(|this| Self { this: Weak::clone(this), factory, + service, class_name, config, actor_class, @@ -116,9 +147,89 @@ impl ActorNamespace { storage: OnceCell::new(), actors: RefCell::new(LinkedHashMap::new()), cleanup_task: RefCell::new(None), + cluster: OnceCell::new(), + directory: OnceCell::new(), + resolving: RefCell::new(HashMap::new()), }) } + /// Puts a durable namespace on disk under the cluster's ownership; before `link`. + pub fn set_cluster(&self, cluster: Rc) { + let _ = self.cluster.set(cluster); + } + + /// The cluster, for a namespace linked on disk; in-memory namespaces stay local. + fn cluster(&self) -> Option<&Rc> { + self.directory.get().and_then(|_| self.cluster.get()) + } + + /// The database of root actor `key`. + fn sqlite_path(&self, key: &str) -> Result { + let directory = self + .directory + .get() + .ok_or_else(|| kj::failed!("the namespace has no directory"))?; + Ok(format!("{directory}/{key}.sqlite")) + } + + /// The owner of the actor with `id`: a container here, holding its ownership, or a session + /// with the instance that owns it. + async fn resolve(&self, id: KjOwn) -> Result { + let key = ffi::actor_id_key(&id); + if let Some(container) = self.owned_container(&key) { + return Ok(Resolved::Here(Client(container))); + } + let cluster = self + .cluster() + .ok_or_else(|| kj::failed!("not in a cluster"))?; + // One cold resolution per actor: the rest wait for it, then find its container or its + // cached route, rather than each reading and claiming the owner record in the bucket. + let pending = self.resolving.borrow().get(&key).cloned(); + let _first = if let Some(done) = pending { + done.notified().await; + if let Some(container) = self.owned_container(&key) { + return Ok(Resolved::Here(Client(container))); + } + None + } else { + let done = Rc::new(Notify::new()); + self.resolving.borrow_mut().insert(key.clone(), done); + Some(Resolving { + pending: &self.resolving, + key: key.clone(), + }) + }; + let unique_key = self.unique_key().unwrap_or_default().to_owned(); + match cluster + .route(&unique_key, &self.service, &self.class_name, &key) + .await? + { + Owner::Remote(connection) => Ok(Resolved::Remote(connection)), + Owner::Here(epoch) => { + // A concurrent request may have taken it meanwhile. + if let Some(container) = self.owned_container(&key) { + return Ok(Resolved::Here(Client(container))); + } + let container = ActorContainer::new_root(key.clone(), self, id); + let owned = Owned::new(Rc::clone(cluster), &unique_key, &key, epoch); + *container.owned.borrow_mut() = Some(owned); + let replaced = self.actors.borrow_mut().insert(key, Rc::clone(&container)); + drop(replaced); + Ok(Resolved::Here(Client(container))) + } + } + } + + fn owned_container(&self, key: &str) -> Option> { + let actors = self.actors.borrow(); + let container = actors.get(key)?; + container + .owned + .borrow() + .is_some() + .then(|| Rc::clone(container)) + } + fn storage(&self) -> Result<&ffi::ActorStorage> { self.storage.get().map(|storage| &**storage).ok_or_else(|| { kj::failed!( @@ -198,6 +309,14 @@ impl ActorNamespace { /// The channel of the actor with `id`, started on first use. `persistent` is recorded on the /// channel: whether stubs to it may be stored. pub fn channel(&self, id: KjOwn, persistent: Persistent) -> Rc { + if self.cluster().is_some() { + return Rc::new(RoutedChannel { + namespace: Weak::clone(&self.this), + id, + persistent, + bound: Rc::default(), + }); + } Rc::new(ActorChannel { container: Client(self.container_for(id)), persistent, @@ -277,6 +396,9 @@ impl ActorNamespace { } ActorConfig::Ephemeral { .. } => ("", ""), }; + if !path.is_empty() { + let _ = self.directory.set(format!("{path}/{unique_key}")); + } let storage = ffi::factory_new_actor_storage( self.factory.raw(), path, @@ -326,6 +448,118 @@ impl ActorNamespaceHandle { .container_for(id) .start_request(ffi::new_request_metadata(KjMaybe::None, KjMaybe::None)) } + + fn namespace(&self) -> Option> { + self.0.upgrade() + } + + pub(crate) fn replicated(&self) -> bool { + self.namespace().is_some_and(|ns| ns.cluster().is_some()) + } + + pub(crate) fn check_open(&self) -> Result<()> { + match self.namespace().as_ref().and_then(|ns| ns.cluster()) { + Some(node) if !node.cluster.lease_holds() => Err(kj::failed!( + "this instance's cluster lease in the bucket has lapsed" + )), + _ => Ok(()), + } + } + + pub(crate) fn database_reset(&self, root_key: &str) { + if let Some(ns) = self.namespace() + && let (Some(node), Some(unique_key)) = (ns.cluster(), ns.unique_key()) + { + node.cluster.database_reset(unique_key, root_key); + } + } + + pub(crate) fn wal_write(&self, file: &str, offset: u64, data: &[u8]) { + if let Some(ns) = self.namespace() + && let (Some(node), Some(unique_key)) = (ns.cluster(), ns.unique_key()) + { + node.cluster.wal_write(unique_key, file, offset, data); + } + } +} + +/// Where a request to a clustered actor goes. +enum Resolved { + Here(Client), + Remote(Rc), +} + +/// A stub's channel to an actor in a clustered namespace. Its first request finds the owner; +/// the stub stays with that instance, as a local stub stays with its container. +struct RoutedChannel { + namespace: Weak, + id: KjOwn, + persistent: Persistent, + bound: Rc>>>, +} + +impl Channel for RoutedChannel { + fn start_request( + &self, + mut metadata: KjOwn, + ) -> Result> { + ffi::request_metadata_set_from_persistent_stub(metadata.as_mut(), self.persistent); + let namespace = self + .namespace + .upgrade() + .ok_or_else(|| kj::disconnected!("the Durable Object namespace is gone"))?; + let id = ffi::actor_id_clone(&self.id); + let bound = Rc::clone(&self.bound); + Ok(PromisedInterface::new(async move { + let resolved = bound.borrow().clone(); + let resolved = if let Some(resolved) = resolved { + resolved + } else { + let resolved = Rc::new(namespace.resolve(id).await?); + Rc::clone(bound.borrow_mut().get_or_insert(resolved)) + }; + match &*resolved { + Resolved::Here(container) => container.start_request(metadata), + Resolved::Remote(connection) => Ok(attach( + connection.start_request(metadata), + Rc::clone(connection), + )), + } + }) + .into_kj()) + } + + fn token(&self, usage: TokenUsage) -> Result> { + let namespace = self + .namespace + .upgrade() + .ok_or_else(|| kj::disconnected!("the Durable Object namespace is gone"))?; + let unique_key = namespace + .unique_key() + .ok_or_else(|| kj::failed!("only durable actors have channel tokens"))?; + Ok(ffi::factory_encode_actor_token( + namespace.factory.raw(), + unique_key, + &self.id, + self.persistent, + usage, + )?) + } + + fn evict_for_test(&self, hibernate: bool) -> LocalBoxFuture<'_, Result<()>> { + Box::pin(async move { + let namespace = self + .namespace + .upgrade() + .ok_or_else(|| kj::disconnected!("the Durable Object namespace is gone"))?; + match namespace.resolve(ffi::actor_id_clone(&self.id)).await? { + Resolved::Here(container) => container.evict_for_test(hibernate).await, + Resolved::Remote(_) => Err(kj::failed!( + "jsg.Error: Cannot evict a Durable Object another instance owns." + )), + } + }) + } } // ===================================================================================== @@ -377,6 +611,8 @@ pub struct ActorContainer { on_broken_task: RefCell>, /// The idle shutdown, armed when the last request ends and cancelled by the next. shutdown_task: RefCell>, + /// A clustered root's ownership, for as long as the container is in its namespace's map. + owned: RefCell>, } impl ActorContainer { @@ -439,6 +675,7 @@ impl ActorContainer { idle: Notify::new(), on_broken_task: RefCell::new(None), shutdown_task: RefCell::new(None), + owned: RefCell::new(None), } } } @@ -549,6 +786,7 @@ impl ActorContainer { let class = self.class()?; class.when_ready().await?; self.require_not_broken()?; + self.prepare_storage().await?; // A concurrent request may have started the actor meanwhile. if !self.is_running() { self.start(&class)?; @@ -560,6 +798,33 @@ impl ActorContainer { .ok_or_else(|| kj::failed!("the actor did not start")) } + /// In cluster mode, readies the actor's database before the actor opens it: it is taken over + /// (recovered from earlier epochs). An actor whose takeover fails leaves its namespace, so the + /// next request claims anew. + async fn prepare_storage(&self) -> Result<()> { + let ns = self.namespace()?; + if ns.cluster().is_none() { + return Ok(()); + } + let root_path = ns.sqlite_path(&self.root_key)?; + if self.is_root() { + let Some(restore) = self.owned.borrow().as_ref().map(|o| o.restore(root_path)) else { + return Err(kj::failed!("a clustered actor without ownership")); + }; + if let Err(e) = restore.await { + self.abort(Some(e.clone())); + ns.actors.borrow_mut().remove(&self.key); + return Err(e); + } + } else { + // TODO(cluster): facets, each a log of its own in the root's epoch. + return Err(kj::failed!( + "jsg.Error: Facets are not supported in cluster mode." + )); + } + Ok(()) + } + /// Constructs the actor and starts watching it for breakage. fn start(&self, class: &Rc) -> Result<()> { let ns = self.namespace()?; @@ -570,6 +835,7 @@ impl ActorContainer { enable_sql: ns.enable_sql(), root_key: self.root_key.clone(), facet_id: self.facet_id()?.into(), + replicated: ns.cluster().is_some(), }; let actor = Rc::new(class.new_actor(NewActor { id, @@ -619,6 +885,13 @@ impl Client { actor: &ffi::ActorHandle, ) -> Result> { let ns = self.namespace()?; + // A node stalled past its lease may have been taken over: serving + // even a read from its database could return stale data. + if ns.cluster().is_some_and(|node| !node.cluster.lease_holds()) { + return Err(kj::failed!( + "this instance's cluster lease in the bucket has lapsed" + )); + } let inner = { let state = self.class_and_id.borrow(); let ClassState::Ready { class, id } = &*state else { @@ -734,9 +1007,24 @@ impl ActorContainer { } } drop(manager); + self.release(actor.as_deref()); drop(actor); } + /// Closes a clustered actor's database and gives up its ownership, so nothing writes behind + /// the next owner. + fn release(&self, actor: Option<&KjOwn>) { + if let Some(actor) = actor + && self + .namespace + .upgrade() + .is_some_and(|ns| ns.cluster().is_some()) + { + ffi::actor_close_storage(actor); + } + drop(self.owned.borrow_mut().take()); + } + /// Aborts the actor and its facets; every request on them fails with `reason`. Without one /// the actors are shut down, failing nothing in flight, and later requests fail for an /// unknown reason. The caller removes the container from any map that could route traffic to @@ -759,6 +1047,7 @@ impl ActorContainer { *self.broken_reason.borrow_mut() = Some( reason.unwrap_or_else(|| kj::failed!("jsg.Error: Actor aborted for unknown reason.")), ); + self.release(actor.as_deref()); drop(actor); } @@ -918,9 +1207,11 @@ impl Drop for ActorContainer { for facet in self.facets.get_mut().values() { facet.abort(None); } - if let Some(actor) = self.actor.get_mut().take() { - ffi::actor_abort(&actor, &AbortReason(None)); + let actor = self.actor.get_mut().take(); + if let Some(actor) = &actor { + ffi::actor_abort(actor, &AbortReason(None)); } + self.release(actor.as_deref()); } } @@ -1027,6 +1318,13 @@ impl ActorContainer { } fn clone_facet(&self, src: &str, dst: &str) -> Result<()> { + let ns = self.namespace()?; + if ns.cluster().is_some() { + // TODO(cluster): clone in the bucket's terms. + return Err(kj::failed!( + "jsg.Error: Cloning facets is not supported in cluster mode." + )); + } // Replacing a facet implies aborting it. self.abort_facet(dst, kj::failed!("jsg.Error: Facet was cloned-over.")); if src == dst { @@ -1034,7 +1332,6 @@ impl ActorContainer { // abort matches `delete(dst)`, and the storage stays as it is. return Ok(()); } - let ns = self.namespace()?; ffi::actor_storage_clone_facet( ns.storage()?, &self.root_key, @@ -1047,6 +1344,23 @@ impl ActorContainer { } impl ActorHooks { + pub(crate) fn committer(&self) -> Result> { + // Only a clustered actor's storage asks; see `ActorStorageSpec::replicated`. + let container = self.container()?; + let ns = container + .namespace + .upgrade() + .ok_or_else(|| kj::disconnected!("shut down"))?; + let node = ns + .cluster() + .ok_or_else(|| kj::failed!("not in a cluster"))?; + Ok(Box::new(Committer { + cluster: Rc::clone(&node.cluster), + namespace: ns.unique_key().unwrap_or_default().to_owned(), + object: container.root_key.clone(), + })) + } + fn container(&self) -> Result> { self.0 .upgrade() diff --git a/src/workerd/server/server/bridge.rs b/src/workerd/server/server/bridge.rs index bf4eeddfcb4..4feaa6b4bca 100644 --- a/src/workerd/server/server/bridge.rs +++ b/src/workerd/server/server/bridge.rs @@ -25,6 +25,9 @@ pub use crate::channels::ServerHandle; pub use crate::channels::SubrequestChannel; pub use crate::channels::WorkerInterfaceList; pub use crate::channels::WorkerStub; +pub use crate::cluster::Committer; +pub use crate::cluster::ResultTask; +pub use crate::cluster::result_task_run; pub use crate::entry::PendingCommand; pub use crate::entry::run_pending_command; pub use crate::in_process::InProcessServer; @@ -164,6 +167,8 @@ pub mod ffi { root_key: String, /// A facet's number within its root actor's storage; none for a root actor. facet_id: KjMaybe, + /// The cluster replicates the actor's commits: each waits for `Committer::commit`. + replicated: bool, } /// What a channel token is for (`IoChannelFactory::ChannelTokenUsage`): restoring a stub over @@ -423,6 +428,8 @@ pub mod ffi { ) -> Result; /// Resets the actor's SQLite database while its connection is open (`deleteAllActors`). fn actor_reset_storage(actor: &ActorHandle); + /// Closes the actor's database, if it has one; later use of it throws. + fn actor_close_storage(actor: &ActorHandle); /// The actor's hibernation manager, to hand to the actor that replaces it after eviction. fn actor_hibernation_manager(actor: &ActorHandle) -> KjMaybe>; type HibernationManager; @@ -559,6 +566,15 @@ pub mod ffi { factory: &WorkerFactory, stream: KjOwn, ) -> KjOwn; + /// An actor on a cluster peer, by `WorkerdDebugPort.getActor` over `session`, a peer + /// session from `new_rpc_client`. Pipelined: no round trip until its first event. It + /// keeps the session alive. + fn rpc_client_debug_actor( + session: &RpcClient, + service: &str, + class_name: &str, + actor_id: &str, + ) -> KjOwn; /// Sends a custom event (a trace, a JS RPC session, a tail stream, a UDP flow) to the /// peer's `WorkerdBootstrap`. async fn rpc_client_custom_event( @@ -567,6 +583,12 @@ pub mod ffi { cf_blob_json: KjMaybe<&str>, ) -> Result; async fn rpc_client_on_disconnect(client: &RpcClient); + /// Starts an event on the peer's `WorkerdBootstrap`: HTTP, WebSockets and JS RPC all + /// cross as capnp. The client must outlive the returned interface. + fn rpc_client_start_request( + client: &RpcClient, + metadata: KjOwn, + ) -> KjOwn; /// Starts the inspector on `address` on its own thread; every isolate the factory /// creates afterwards registers with it. Returns the bound port. @@ -820,6 +842,15 @@ pub mod ffi { self: &ActorNamespaceHandle, id: KjOwn, ) -> Result>; + /// Whether the cluster replicates the namespace's actors. + fn replicated(self: &ActorNamespaceHandle) -> bool; + /// Called as SQLite opens one of the namespace's files: fails once this instance's + /// cluster lease has lapsed. + fn check_open(self: &ActorNamespaceHandle) -> Result<()>; + /// A write SQLite made to one of the namespace's WAL files. + fn wal_write(self: &ActorNamespaceHandle, file: &str, offset: u64, data: &[u8]); + /// The database of the root actor `root_key` was deleted. + fn database_reset(self: &ActorNamespaceHandle, root_key: &str); /// An actor's way back to the server: the requests it raises for itself, its facets /// (named child actors sharing its storage), and its transitions between idle and active. @@ -844,6 +875,15 @@ pub mod ffi { fn active(self: &ActorHooks); /// The actor's last request ended. fn inactive(self: &ActorHooks); + /// What makes the actor's commits durable, when the cluster replicates it. + fn committer(self: &ActorHooks) -> Result>; + + type Committer; + /// Makes the commits captured so far durable. + fn commit(self: &Committer) -> Box; + /// A Rust future C++ awaits with `result_task_run`. + type ResultTask; + async fn result_task_run(task: Box) -> Result<()>; /// The reason of an `actor_abort`. `raise` returns it as the `Err`, which C++ catches as /// the equivalent `kj::Exception`; `Ok` when there is none. diff --git a/src/workerd/server/server/cluster.rs b/src/workerd/server/server/cluster.rs new file mode 100644 index 00000000000..c2a7e392367 --- /dev/null +++ b/src/workerd/server/server/cluster.rs @@ -0,0 +1,487 @@ +// Copyright (c) 2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +//! Cluster mode (`Config.cluster`): this instance's membership in a cluster whose shared state +//! is a bucket (the `cluster` crate), and the port peers reach it on. +//! +//! The port speaks HTTP/1.1 through kj-hyper. `CONNECT`, its MAC in `cf-cluster-auth`, opens the +//! one capnp session an instance keeps to each peer: the tunnel carries `WorkerdDebugPort`, +//! whose `getActor` gives a `WorkerdBootstrap` for one Durable Object, so HTTP, WebSockets and +//! JS RPC to the object cross unchanged. A request for an object this instance +//! does not own routes on through its own channel to the owner. + +use std::cell::RefCell; +use std::collections::HashMap; +use std::future::Future; +use std::pin::Pin; +use std::rc::Rc; + +use cluster::Cluster; +use cluster::Route; +use futures::FutureExt; +use futures::TryFutureExt; +use futures::future::LocalBoxFuture; +use kj::http::Headers; +use kj::http::HeadersRef; +use kj::http::Method; +use kj::http::ServiceResponse; +use kj_hyper::client::Dialed; +use kj_hyper::server::Connect; +use kj_hyper::server::Handler; +use kj_hyper::server::ServerSettings; +use kj_hyper::server::Shutdown; +use kj_rs::KjOwn; +use tokio::io::AsyncReadExt; +use tokio::io::AsyncWriteExt; +use tokio::net::TcpListener; +use tokio::net::TcpStream; +use workerd_capnp::cluster_config; + +use crate::Result; +use crate::bridge::ffi; +use crate::config::Factory; +use crate::services::header_table; +use crate::services::send_error; + +/// A session's CONNECT carries `..`: see `connect_mac`. +const AUTH_HEADER: &str = "cf-cluster-auth"; +/// How far a CONNECT's timestamp may be from this instance's clock. +const AUTH_WINDOW_MS: u64 = 30_000; +/// How long dialing a peer and its answer to the CONNECT may take. +const TUNNEL_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5); + +/// This instance's cluster: the crate's state and the factory sessions to owners start from. +pub struct ClusterNode { + pub cluster: Rc, + factory: Rc, + /// One session per peer address, dropped when it disconnects. + sessions: Rc>>>, + /// The address peers dial, which their CONNECTs are bound to. + advertise: String, + /// The nonces of CONNECTs accepted within twice the window, by timestamp. + nonces: RefCell>, +} + +pub struct RpcConnection(KjOwn); + +/// Binds the cluster port, before the services start. +/// Returns the listener and the address to advertise. +pub async fn bind( + conf: cluster_config::Reader<'_>, + address: Option<&str>, + advertise: Option<&str>, +) -> Result<(TcpListener, String)> { + let address = address.map_or_else(|| text(conf.get_address()), |a| Ok(a.to_owned()))?; + refuse_public(conf, &address)?; + let listener = TcpListener::bind(&address) + .await + .map_err(|e| kj::failed!("cluster address {address}: {e}"))?; + let bound = listener.local_addr().map_err(|e| kj::failed!("{e}"))?; + let advertise = match advertise { + Some(a) => a.to_owned(), + None if conf.has_advertise() => text(conf.get_advertise())?, + None if bound.ip().is_unspecified() => { + return Err(kj::failed!( + "cluster address {address} is a wildcard, which peers cannot reach: set \ + cluster.advertise or --cluster-advertise" + )); + } + None => bound.to_string(), + }; + refuse_public(conf, &advertise)?; + Ok((listener, advertise)) +} + +/// Fails for an `address` that is a public IP, unless the config allows it. +fn refuse_public(conf: cluster_config::Reader<'_>, address: &str) -> Result<()> { + let Ok(ip) = address.parse::().map(|a| a.ip()) else { + return Ok(()); + }; + if is_public(ip) && !conf.get_unsafe_public_address() { + return Err(kj::failed!( + "cluster address {ip} is public, and traffic between instances is not encrypted: \ + use a private network (a VPC, WireGuard, Tailscale), or set \ + cluster.unsafePublicAddress" + )); + } + Ok(()) +} + +/// Whether `ip` is on a network others route: not loopback, private, link-local, shared +/// (100.64.0.0/10, which Tailscale uses), or a wildcard, which this cannot judge. +fn is_public(ip: std::net::IpAddr) -> bool { + match ip { + std::net::IpAddr::V4(v4) => { + let [a, b, ..] = v4.octets(); + let shared = a == 100 && (64..128).contains(&b); + let local = v4.is_loopback() || v4.is_private() || v4.is_link_local(); + !(local || shared || v4.is_unspecified()) + } + std::net::IpAddr::V6(v6) => { + let local = v6.is_loopback() || v6.is_unique_local() || v6.is_unicast_link_local(); + !(local || v6.is_unspecified()) + } + } +} + +/// The MAC a CONNECT to `target` carries, at `ms` with `nonce`: a header seen on the wire is +/// refused at any other target, once used, or once stale. +fn connect_mac(cluster: &Cluster, target: &str, ms: u64, nonce: &str) -> Vec { + cluster.mac(format!("workerd cluster connect v1\n{target}\n{ms}\n{nonce}").as_bytes()) +} + +impl ClusterNode { + /// Joins the cluster: the crate's state for `conf`, this instance at `address`. + pub fn new( + factory: Rc, + conf: cluster_config::Reader<'_>, + address: String, + ) -> Result> { + let options = conf.get_bucket_options().map_err(|e| capnp(&e))?; + let setup = cluster::Setup { + url: text(conf.get_bucket())?, + options: options.iter().map(text).collect::>()?, + node: cluster::hex(&rand::random::<[u8; 16]>()), + address: address.clone(), + ttl_ms: cluster_logic::lease::LEASE_MS, + key: text(conf.get_key())?.into_bytes(), + }; + let cluster = Cluster::new(setup).map_err(|e| kj::failed!("cluster: {e}"))?; + Ok(Rc::new(Self { + cluster, + factory, + sessions: Rc::default(), + advertise: address, + nonces: RefCell::default(), + })) + } + + /// The owner of `actor` in `namespace`, the class `class` of worker `service`: here, at an + /// epoch, or a capnp session elsewhere. + pub async fn route( + &self, + namespace: &str, + service: &str, + class: &str, + actor: &str, + ) -> Result { + let now = cluster::now_ms(); + let route = self + .cluster + .acquire_or_route(namespace, actor, now) + .await + .map_err(|e| kj::disconnected!("{namespace}/{actor}: {e}"))?; + let (node, address, stale) = match route { + Route::Owned(epoch) => return Ok(Owner::Here(epoch)), + Route::Remote { + node, + address, + stale, + } => (node, address, stale), + }; + if stale { + let cluster = Rc::clone(&self.cluster); + let (namespace, actor) = (namespace.to_owned(), actor.to_owned()); + self.factory.spawn_detached(async move { + let _ = cluster.refresh_route(&namespace, &actor, now).await; + }); + } + let session = self + .session(&address) + .await + .map_err(|e| kj::disconnected!("{namespace}/{actor} on {node}: {e}"))?; + // A channel per stub, so a stub stays with the instance it reached. + Ok(Owner::Remote(Rc::new(RpcConnection( + ffi::rpc_client_debug_actor(&session.0, service, class, actor), + )))) + } + + /// The session to the peer at `address`: the open one, or a new one. + async fn session(&self, address: &str) -> std::io::Result> { + if let Some(session) = self.sessions.borrow().get(address) { + return Ok(Rc::clone(session)); + } + // A stalled peer accepts the connection and never answers. + let tunnel = tokio::time::timeout(TUNNEL_TIMEOUT, self.tunnel(address)); + let stream = tunnel.await.map_err(|_| { + std::io::Error::new(std::io::ErrorKind::TimedOut, "no answer to CONNECT") + })??; + let session = Rc::new(RpcConnection(ffi::new_rpc_client( + self.factory.raw(), + Dialed::from(stream).into_kj(), + ))); + let replaced = self + .sessions + .borrow_mut() + .insert(address.to_owned(), Rc::clone(&session)); + drop(replaced); + let (sessions, address, watched) = ( + Rc::clone(&self.sessions), + address.to_owned(), + Rc::clone(&session), + ); + self.factory.spawn_detached(async move { + let _ = ffi::rpc_client_on_disconnect(&watched.0).await; + let mut sessions = sessions.borrow_mut(); + if sessions + .get(&address) + .is_some_and(|s| Rc::ptr_eq(s, &watched)) + { + sessions.remove(&address); + } + }); + Ok(session) + } + + /// A TCP connection to the peer at `address`, through its CONNECT for a session. + async fn tunnel(&self, address: &str) -> std::io::Result { + let mut stream = TcpStream::connect(address).await?; + stream.set_nodelay(true)?; + let ms = cluster::now_ms(); + let nonce = cluster::hex(&rand::random::<[u8; 16]>()); + let mac = cluster::hex(&connect_mac(&self.cluster, address, ms, &nonce)); + let auth = format!("{AUTH_HEADER}: {ms}.{nonce}.{mac}"); + let request = format!("CONNECT cluster:1 HTTP/1.1\r\nhost: cluster:1\r\n{auth}\r\n\r\n"); + stream.write_all(request.as_bytes()).await?; + // The response head, read a byte at a time so that nothing past it is consumed. + let mut head = Vec::new(); + while !head.ends_with(b"\r\n\r\n") { + if head.len() > 4096 { + return Err(std::io::Error::other("response head too long")); + } + head.push(stream.read_u8().await?); + } + if !head.starts_with(b"HTTP/1.1 200") { + let status = String::from_utf8_lossy(head.split(|&b| b == b'\r').next().unwrap_or(&[])); + return Err(std::io::Error::other(format!("refused: {status}"))); + } + Ok(stream) + } +} + +/// Where a Durable Object runs, for a request about to start on it. +pub enum Owner { + Here(u64), + Remote(Rc), +} + +impl RpcConnection { + /// Starts a request on the remote owner; the caller keeps the session alive with it. + pub fn start_request( + &self, + metadata: KjOwn, + ) -> KjOwn { + ffi::rpc_client_start_request(&self.0, metadata) + } +} + +// ===================================================================================== +// The port + +/// Serves the cluster port until the server is dropped. +pub async fn serve(listener: TcpListener, node: Rc) { + let settings = Rc::new(ServerSettings::default()); + let shutdown = Rc::new(Shutdown::new()); + loop { + let Ok((stream, _)) = listener.accept().await else { + continue; + }; + let _ = stream.set_nodelay(true); + let hangup = kj_rs_io::when_write_disconnected(&stream) + .err_into() + .boxed(); + let handler: Rc = Rc::new(PeerConnection { + node: Rc::clone(&node), + }); + let (settings, shutdown) = (Rc::clone(&settings), Rc::clone(&shutdown)); + let factory = Rc::clone(&node.factory); + node.factory.spawn_detached(async move { + let table = header_table(&factory); + if let Err(e) = kj_hyper::server::serve_connection( + stream, hangup, table, settings, handler, &shutdown, + ) + .await + { + tracing::warn!("cluster connection failed: {}", e.description()); + } + }); + } +} + +struct PeerConnection { + node: Rc, +} + +impl PeerConnection { + /// Checks a CONNECT's `AUTH_HEADER`: its MAC, its clock, and that its nonce is new. + fn verify(&self, header: Option<&[u8]>) -> Result<()> { + let bad = || kj::failed!("cluster CONNECT without a valid {AUTH_HEADER}"); + let header = header + .and_then(|h| std::str::from_utf8(h).ok()) + .ok_or_else(bad)?; + let [ms, nonce, mac] = header.split('.').collect::>()[..] else { + return Err(bad()); + }; + let ms: u64 = ms.parse().map_err(|_| bad())?; + let mac = cluster::unhex(mac).ok_or_else(bad)?; + let node = &self.node; + if !constant_time_eq( + &connect_mac(&node.cluster, &node.advertise, ms, nonce), + &mac, + ) { + return Err(bad()); + } + let now = cluster::now_ms(); + if now.abs_diff(ms) > AUTH_WINDOW_MS { + return Err(kj::failed!( + "cluster CONNECT {}ms off this clock", + now.abs_diff(ms) + )); + } + let mut nonces = node.nonces.borrow_mut(); + nonces.retain(|_, t| now.abs_diff(*t) <= 2 * AUTH_WINDOW_MS); + if nonces.insert(nonce.to_owned(), ms).is_some() { + return Err(kj::failed!("cluster CONNECT replayed")); + } + Ok(()) + } +} + +#[async_trait::async_trait(?Send)] +impl Handler for PeerConnection { + async fn request<'a>( + &'a self, + _method: Method, + _url: &'a [u8], + _headers: HeadersRef<'a>, + _body: Pin<&'a mut kj::io::AsyncInputStream>, + response: ServiceResponse<'a>, + ) -> Result<()> { + let table = header_table(&self.node.factory); + send_error(response, 404, "Not Found", &Headers::new(table)).await + } + + async fn connect<'a>( + &'a self, + _host: &'a [u8], + headers: HeadersRef<'a>, + connect: Connect, + ) -> Result<()> { + let table = header_table(&self.node.factory); + let empty = Headers::new(table); + if let Err(e) = self.verify(headers.get_by_name(AUTH_HEADER)) { + tracing::warn!("cluster CONNECT refused: {}", e.description()); + connect.reject(403, "Forbidden", HeadersRef::from(&empty), Some(0))?; + return Ok(()); + } + let tunnel = connect.accept(200, "OK", HeadersRef::from(&empty))?; + // A peer holds the cluster key, so it gets the debug port's interface. + Ok(ffi::factory_accept_debug_port(self.node.factory.raw(), tunnel.into_kj()).await?) + } +} + +fn constant_time_eq(a: &[u8], b: &[u8]) -> bool { + a.len() == b.len() && a.iter().zip(b).fold(0, |acc, (x, y)| acc | (x ^ y)) == 0 +} + +fn text(reader: capnp::Result>) -> Result { + reader + .map_err(|e| capnp(&e))? + .to_string() + .map_err(|e| kj::failed!("{e}")) +} + +fn capnp(e: &capnp::Error) -> crate::Error { + kj::failed!("{e}") +} + +/// A Rust future C++ awaits (`result_task_run`). +pub struct ResultTask(pub LocalBoxFuture<'static, Result<()>>); + +pub async fn result_task_run(task: Box) -> Result<()> { + task.0.await +} + +/// What makes one actor's commits durable: `sync_actor` on its object. +pub struct Committer { + pub cluster: Rc, + pub namespace: String, + pub object: String, +} + +impl Committer { + pub fn commit(&self) -> Box { + let cluster = Rc::clone(&self.cluster); + let (namespace, object) = (self.namespace.clone(), self.object.clone()); + Box::new(ResultTask( + async move { + cluster + .sync_actor(&namespace, &object) + .await + .map_err(|e| kj::failed!("{namespace}/{object}: commit failed: {e}")) + } + .boxed_local(), + )) + } +} + +/// This instance's ownership of one Durable Object at an epoch. +/// +/// Its container holds it while the container is in its namespace's map. Dropping it stops capturing the object and keeps it +/// dormant here, uploading what the bucket lacks; the next request wakes it. +pub struct Owned { + node: Rc, + namespace: String, + actor: String, + epoch: u64, + /// The takeover, shared by the requests that wait for it. + restore: RefCell>>>>, +} + +impl Owned { + pub fn new(node: Rc, namespace: &str, actor: &str, epoch: u64) -> Self { + Self { + node, + namespace: namespace.to_owned(), + actor: actor.to_owned(), + epoch, + restore: RefCell::new(None), + } + } + + pub fn cluster(&self) -> &Rc { + &self.node.cluster + } + + /// Takes the object over into the database at `path`, once: recovers what earlier epochs + /// committed and opens this one. + pub fn restore(&self, path: String) -> impl Future> + use<> { + let mut restore = self.restore.borrow_mut(); + let cluster = Rc::clone(&self.node.cluster); + let (namespace, actor, epoch) = (self.namespace.clone(), self.actor.clone(), self.epoch); + restore + .get_or_insert_with(|| { + async move { + cluster + .restore_actor(&namespace, &actor, epoch, &path) + .await + .map_err(|e| kj::failed!("{namespace}/{actor}: takeover failed: {e}")) + } + .boxed_local() + .shared() + }) + .clone() + } +} + +impl Drop for Owned { + fn drop(&mut self) { + let cluster = Rc::clone(&self.node.cluster); + cluster.close_actor(&self.namespace, &self.actor); + let (namespace, actor, epoch) = (self.namespace.clone(), self.actor.clone(), self.epoch); + self.node.factory.spawn_detached(async move { + cluster.evict_actor(&namespace, &actor, epoch).await; + }); + } +} diff --git a/src/workerd/server/server/config.rs b/src/workerd/server/server/config.rs index 31f38933477..cf46daf0557 100644 --- a/src/workerd/server/server/config.rs +++ b/src/workerd/server/server/config.rs @@ -51,6 +51,7 @@ use crate::channels::ServerHandle; use crate::channels::SubrequestChannel; use crate::channels::TokenUsage; use crate::channels::WorkerInterface; +use crate::cluster::ClusterNode; use crate::listen::loopback::Loopback; use crate::loader::WorkerLoaderNamespace; use crate::services::DiskDirectoryService; @@ -312,6 +313,8 @@ pub struct Server { /// Durable namespaces by `uniqueKey`, for actor tokens. namespaces_by_unique_key: HashMap>, worker_loaders: RefCell, + /// This instance's cluster, in cluster mode. + cluster: Option>, /// Declared last, so that it is dropped last: the services (and their isolates) refer to it. factory: Rc, } @@ -332,6 +335,11 @@ impl Server { lookup_service(&self.services.0, &self.report, designator) } + #[must_use] + pub fn cluster(&self) -> Option<&Rc> { + self.cluster.as_ref() + } + /// Every service, in config order. pub fn services(&self) -> impl Iterator { self.services @@ -1007,6 +1015,7 @@ impl Server { overrides: &Overrides, report: Reporter, mut inbound_listeners: HashMap>, + cluster_address: Option, ) -> Result> { let message = factory.config()?; let config = message.get_root::().map_err(capnp_error)?; @@ -1062,11 +1071,30 @@ impl Server { &mut namespaces_by_unique_key, )?; + let cluster = match cluster_address { + Some(address) if config.has_cluster() => { + make_cluster(&factory, config, address, &report)? + } + _ => None, + }; + if let Some(cluster) = &cluster { + // Actor databases open only under a lease. + cluster + .cluster + .join() + .await + .map_err(|e| kj::failed!("joining the cluster: {e}"))?; + for namespace in namespaces_by_unique_key.values() { + namespace.set_cluster(Rc::clone(cluster)); + } + } + let server = Rc::new(Self { report, services, namespaces_by_unique_key, worker_loaders: RefCell::new(WorkerLoaders::default()), + cluster, factory, }); let handle = Box::new(ServerHandle(Rc::downgrade(&server))); @@ -1475,3 +1503,18 @@ impl Server { #[cfg(test)] #[path = "config-test.rs"] mod tests; + +/// The cluster of `Config.cluster`, joined at `address`. `None` after reporting a config error. +fn make_cluster( + factory: &Rc, + config: config::Reader<'_>, + address: String, + report: &Reporter, +) -> Result>> { + if !factory.experimental() { + report.error("Cluster mode is experimental; it requires --experimental.".to_owned()); + return Ok(None); + } + let conf = config.get_cluster().map_err(capnp_error)?; + Ok(Some(ClusterNode::new(Rc::clone(factory), conf, address)?)) +} diff --git a/src/workerd/server/server/lib.rs b/src/workerd/server/server/lib.rs index 04f9aeb34ab..913e3e6fdd3 100644 --- a/src/workerd/server/server/lib.rs +++ b/src/workerd/server/server/lib.rs @@ -15,6 +15,7 @@ pub mod bench; pub mod bindings; pub mod bridge; pub mod channels; +pub mod cluster; pub mod config; pub mod entry; pub mod in_process; diff --git a/src/workerd/server/server/run.rs b/src/workerd/server/server/run.rs index 58a72111dca..8e1ea48d24f 100644 --- a/src/workerd/server/server/run.rs +++ b/src/workerd/server/server/run.rs @@ -29,6 +29,8 @@ use crate::bindings::Designator; use crate::bindings::len_u32; use crate::bridge::ffi; use crate::channels::Channel; +use crate::cluster; +use crate::cluster::ClusterNode; use crate::config::Factory; use crate::config::Overrides; use crate::config::Reporter; @@ -61,6 +63,9 @@ pub struct RunOptions { pub control: Option, /// `--debug-port`. pub debug_port: Option, + /// `--cluster-addr` and `--cluster-advertise`. + pub cluster_address: Option, + pub cluster_advertise: Option, } /// The control channel (`--control-fd`), written line by line; a failed write is logged. The @@ -139,11 +144,21 @@ async fn start( control.listen_inspector(port); } + // The cluster port is bound before the services, like the sockets: its address goes in the + // lease record. + let cluster_port = if config.has_cluster() { + let conf = config.get_cluster().map_err(capnp_error)?; + let (address, advertise) = (&options.cluster_address, &options.cluster_advertise); + Some(cluster::bind(conf, address.as_deref(), advertise.as_deref()).await?) + } else { + None + }; let server = Server::start( Rc::clone(&factory), &options.overrides, report, bound.inbound, + cluster_port.as_ref().map(|(_, address)| address.clone()), ) .await?; report_unmatched_overrides(&server, &options.overrides, config, for_test)?; @@ -187,6 +202,9 @@ async fn start( for (name, port) in ports { control.listen(&name, port); } + if let (Some(node), Some((listener, _))) = (server.cluster(), cluster_port) { + start_cluster(&factory, node, listener); + } if let Some(address) = &options.debug_port { let listener = listen::listen(address, 0, factory.loopback()).await?; @@ -204,6 +222,22 @@ async fn start( })) } +/// Runs this instance's part in the cluster: the peer port, the lease, and compaction. They run for +/// the life of the process; a lease the instance cannot keep ends it. +fn start_cluster(factory: &Rc, node: &Rc, listener: tokio::net::TcpListener) { + factory.spawn_detached(cluster::serve(listener, Rc::clone(node))); + let cluster = Rc::clone(&node.cluster); + factory.spawn_detached(async move { + let reason = cluster.maintain().await; + // Peers may take this instance's objects over once its lease lapses: it must stop + // serving first. + tracing::error!("leaving the cluster: {reason}"); + std::process::exit(1); + }); + let cluster = Rc::clone(&node.cluster); + factory.spawn_detached(async move { cluster.tier_loop().await }); +} + /// The listener for one bound socket, or none for a socket whose type is unknown (reported). fn listener( context: Rc, @@ -371,8 +405,19 @@ pub async fn run( match futures::future::select(serving, drain).await { Either::Left((result, _)) => result?, Either::Right(((), serving)) => { + // A draining instance claims nothing more and lets its objects go, so peers take + // them over while its connections finish. + if let Some(node) = server.cluster() { + node.cluster.start_draining(); + server.abort_all_actors(None); + } draining.send_replace(true); serving.await?; + if let Some(node) = server.cluster() + && let Err(e) = node.cluster.deregister().await + { + tracing::warn!("leaving the cluster: {e}"); + } } } diff --git a/src/workerd/server/server/worker.rs b/src/workerd/server/server/worker.rs index 2d526ac5d1f..d0c10c78620 100644 --- a/src/workerd/server/server/worker.rs +++ b/src/workerd/server/server/worker.rs @@ -462,6 +462,7 @@ impl WorkerService { }); let namespace = ActorNamespace::new( Rc::clone(&self.factory), + self.service_name.clone().unwrap_or_default(), class_name.clone(), config.clone(), actor_class, @@ -485,6 +486,7 @@ impl WorkerService { ) -> Result> { let namespace = ActorNamespace::new( Rc::clone(&self.factory), + self.service_name.clone().unwrap_or_default(), key.to_owned(), config, actor_class, diff --git a/src/workerd/server/tests/cluster/BUILD.bazel b/src/workerd/server/tests/cluster/BUILD.bazel new file mode 100644 index 00000000000..ee9543cf79c --- /dev/null +++ b/src/workerd/server/tests/cluster/BUILD.bazel @@ -0,0 +1,23 @@ +load("@aspect_rules_js//js:defs.bzl", "js_test") + +# Takeover waits out a dead owner's 20 s lease, several times over. +# TODO(cluster): under the 60 s "long" timeout. +js_test( + name = "cluster-test", + size = "large", + timeout = "eternal", + data = [ + ":cluster-test.wd-test", + ":cluster-worker.js", + ":fake-s3.js", + ":fake-s3.wd-test", + "//src/workerd/server:workerd", + ], + entry_point = "cluster-test.mjs", + env = { + "WORKERD_BINARY": "$(rootpath //src/workerd/server:workerd)", + "WD_TEST_CONFIG": "$(rootpath :cluster-test.wd-test)", + "FAKE_S3_CONFIG": "$(rootpath :fake-s3.wd-test)", + }, + tags = ["js-test"], +) diff --git a/src/workerd/server/tests/cluster/cluster-test.mjs b/src/workerd/server/tests/cluster/cluster-test.mjs new file mode 100644 index 00000000000..ac803b6fb4b --- /dev/null +++ b/src/workerd/server/tests/cluster/cluster-test.mjs @@ -0,0 +1,1098 @@ +// Copyright (c) 2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 +// +// Integration test for cluster mode. The test starts a fake S3 (fake-s3.js, in +// a workerd of its own) and several workerd instances whose cluster state lives +// in it, each with its SQLite files on a disk of its own, then exercises: +// 1. Consistent DO state across instances (single-writer correctness). +// 2. Transparent forwarding when a request arrives at the "wrong" instance. +// 3. Takeover after a node is killed. +// 4. Each running instance holds a lease record in the bucket. +// 5. Storing/loading a DO stub without calling it claims nothing. +// 6. A broken DO does not stay stuck while stubs still pin it: it can be +// re-instantiated instead of hanging requests. +// 7. Persistent stubs to restored RpcTargets rooted at a DO on one node can be +// stored and redeemed from a DO on another node; the restore runs on the +// root DO's owner. This holds whether the stub arrives as a token or was +// minted in-process and passed through props. +// 8. A stub that has resolved to a remote DO keeps failing once that DO +// breaks, while a fresh stub reaches the re-instantiated DO. + +import { spawn } from 'node:child_process'; +import { mkdir, mkdtemp, readdir, rm } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { connect, createServer } from 'node:net'; +import { createHmac, randomBytes } from 'node:crypto'; +import { join } from 'node:path'; +import { env } from 'node:process'; +import { test } from 'node:test'; +import assert from 'node:assert'; +import { setTimeout as sleep } from 'node:timers/promises'; + +assert( + env.WORKERD_BINARY !== undefined, + 'You must set the WORKERD_BINARY environment variable.' +); +assert( + env.WD_TEST_CONFIG !== undefined, + 'You must set the WD_TEST_CONFIG environment variable.' +); +assert( + env.FAKE_S3_CONFIG !== undefined, + 'You must set the FAKE_S3_CONFIG environment variable.' +); + +const CONTROL_FD = 3; + +// A minimal harness specialized for the cluster test. Unlike server-harness.mjs +// it: +// - allows overriding bindings via process environment (NODE_ID), +// - points each instance at the fake S3 and at a disk of its own, +// - supports SIGKILL in addition to SIGTERM for the takeover test. +class ClusterNode { + #binary; + #config; + #localPath; + #s3Port; + #nodeId; + #child = null; + #httpPort = null; + #closed = null; + #stderrBuffer = ''; + #extraArgs; + + constructor({ binary, config, localPath, s3Port, nodeId, extraArgs = [] }) { + this.#extraArgs = extraArgs; + this.#binary = binary; + this.#config = config; + this.#localPath = localPath; + this.#s3Port = s3Port; + this.#nodeId = nodeId; + } + + get localPath() { + return this.#localPath; + } + + get nodeId() { + return this.#nodeId; + } + + get httpPort() { + assert(this.#httpPort !== null, 'node not started'); + return this.#httpPort; + } + + // Returns the contents of the node's stderr observed so far. + get stderr() { + return this.#stderrBuffer; + } + + async start() { + assert.strictEqual(this.#child, null); + + const args = [ + 'serve', + this.#config, + '--experimental', + '--verbose', + `--control-fd=${CONTROL_FD}`, + `--socket-addr=http=127.0.0.1:0`, + ]; + if (this.#localPath !== undefined) { + args.push(`--directory-path=local=${this.#localPath}`); + } + args.push(...this.#extraArgs); + + const child = spawn(this.#binary, args, { + stdio: ['ignore', 'inherit', 'pipe', 'pipe'], + env: { + ...env, + NODE_ID: this.#nodeId, + AWS_ENDPOINT_URL: `http://127.0.0.1:${this.#s3Port}`, + }, + }); + this.#child = child; + + child.stderr.on('data', (data) => { + const chunk = data.toString('utf8'); + this.#stderrBuffer += chunk; + // Mirror to our own stderr so test output remains useful. + process.stderr.write(`[${this.#nodeId}] ${chunk}`); + }); + + // Watch the control fd for the assigned http port. + const portPromise = new Promise((resolve, reject) => { + let buffer = ''; + const onData = (data) => { + buffer += data.toString('utf8'); + let nl; + while ((nl = buffer.indexOf('\n')) !== -1) { + const line = buffer.slice(0, nl).trim(); + buffer = buffer.slice(nl + 1); + if (!line) continue; + try { + const parsed = JSON.parse(line); + if (parsed.event === 'listen' && parsed.socket === 'http') { + child.stdio[CONTROL_FD].off('data', onData); + resolve(parsed.port); + return; + } + } catch (err) { + reject( + new Error( + `Failed to parse control message from node ${this.#nodeId}: ${line}` + ) + ); + return; + } + } + }; + child.stdio[CONTROL_FD].on('data', onData); + child.once('error', reject); + child.once('exit', (code, signal) => { + if (this.#httpPort === null) { + reject( + new Error( + `Node ${this.#nodeId} exited before listening (code=${code}, signal=${signal})` + ) + ); + } + }); + }); + + this.#closed = new Promise((resolve) => { + child.once('exit', (code, signal) => resolve({ code, signal })); + }); + + await new Promise((resolve, reject) => { + child.once('spawn', resolve).once('error', reject); + }); + + this.#httpPort = await portPromise; + } + + async stop({ signal = 'SIGTERM', timeoutMs = 10_000 } = {}) { + if (this.#child === null) return null; + const child = this.#child; + this.#child = null; + child.kill(signal); + + const killTimer = setTimeout(() => { + try { + child.kill('SIGKILL'); + } catch (_) { + // process may already be gone. + } + }, timeoutMs); + + const result = await this.#closed; + clearTimeout(killTimer); + this.#httpPort = null; + return result; + } +} + +// Fetches `path` from the given node and parses the JSON response. When +// `timeoutMs` is given, a request that does not complete in time fails with a +// clear error instead of hanging the test (used where a bug would manifest as +// a request that never completes). +async function fetchJson(port, path, { timeoutMs } = {}) { + const url = `http://127.0.0.1:${port}${path}`; + let res; + try { + res = await fetch(url, { + signal: + timeoutMs === undefined ? undefined : AbortSignal.timeout(timeoutMs), + }); + } catch (err) { + if (err.name === 'TimeoutError') { + throw new Error( + `request to ${url} did not complete within ${timeoutMs}ms` + ); + } + throw err; + } + const text = await res.text(); + let body; + try { + body = JSON.parse(text); + } catch (err) { + throw new Error( + `Non-JSON response from ${url} (status ${res.status}): ${text}` + ); + } + return { status: res.status, body }; +} + +// The fake S3 the cluster's state lives in, in a workerd of its own. +class FakeS3 { + #node; + + constructor() { + // A cluster node's harness starts any workerd; this one has no cluster. + this.#node = new ClusterNode({ + binary: env.WORKERD_BINARY, + config: env.FAKE_S3_CONFIG, + nodeId: 's3', + }); + } + + get port() { + return this.#node.httpPort; + } + + start() { + return this.#node.start(); + } + + stop() { + return this.#node.stop({ timeoutMs: 5000 }); + } + + // The keys under `prefix` in the cluster's bucket, whose layout is under + // `cluster/v1/`. + async list(prefix) { + const query = new URLSearchParams({ + 'list-type': '2', + prefix: `cluster/v1/${prefix}`, + }); + const res = await fetch(`http://127.0.0.1:${this.port}/wdtest/?${query}`); + const xml = await res.text(); + return [...xml.matchAll(/([^<]*)<\/Key>/g)].map((m) => + m[1].slice('cluster/v1/'.length) + ); + } + + async get(key) { + const res = await fetch( + `http://127.0.0.1:${this.port}/wdtest/cluster/v1/${key}` + ); + return res.status === 404 ? null : await res.text(); + } + + // The key of the node that owns the Counter with this ID, or null when no + // owner record exists. + async owner(id) { + const record = await this.get(`do/counter-test-namespace/${id}/own.json`); + return record === null ? null : JSON.parse(record).node; + } + + // How many GET, PUT and DELETE requests the bucket served for `key`. + async requests(key) { + const query = new URLSearchParams({ key }); + return ( + await fetch(`http://127.0.0.1:${this.port}/_requests?${query}`) + ).json(); + } + + // The keys of the nodes with a lease record. + async nodes() { + // Not the `.probe` a starting node writes and deletes. + return (await this.list('nodes/')) + .filter((key) => key.endsWith('.json')) + .map((key) => key.slice('nodes/'.length, -'.json'.length)); + } +} + +// Starts a fake S3 and `numNodes` workerd instances and runs `fn` with: +// nodes the ClusterNode instances, in start order +// bucket the FakeS3 +// nodeKeys Map from nodeId to the node's key (32-char hex), which is also +// what owner records name +async function withCluster(numNodes, fn, { extraArgs = () => [] } = {}) { + // Each test run gets its own bucket and disks, so state doesn't leak + // between tests. + const root = await mkdtemp(join(tmpdir(), 'workerd-cluster-')); + const bucket = new FakeS3(); + const nodes = []; + const nodeKeys = new Map(); + try { + await bucket.start(); + for (let i = 0; i < numNodes; i++) { + const localPath = join(root, `node${i}`); + await mkdir(localPath); + const node = new ClusterNode({ + binary: env.WORKERD_BINARY, + config: env.WD_TEST_CONFIG, + localPath, + s3Port: bucket.port, + nodeId: `node${i}`, + extraArgs: await extraArgs(i), + }); + await node.start(); + nodes.push(node); + + // A node writes its lease record in its first maintenance round. Since + // nodes start one at a time, the record that is new is this node's. + const known = new Set(nodeKeys.values()); + const deadline = Date.now() + 10_000; + let fresh = []; + while (Date.now() < deadline) { + fresh = (await bucket.nodes()).filter((key) => !known.has(key)); + if (fresh.length > 0) break; + await sleep(50); + } + assert.strictEqual( + fresh.length, + 1, + `expected exactly one new lease record after starting ${node.nodeId}, got: [${fresh.join(', ')}]` + ); + nodeKeys.set(node.nodeId, fresh[0]); + } + await fn({ nodes, bucket, nodeKeys }); + } finally { + // Stop all nodes (ignore errors -- some may already be stopped by the + // test itself). + for (const node of nodes) { + try { + await node.stop({ timeoutMs: 5000 }); + } catch (_) { + // ignore + } + } + await bucket.stop(); + await rm(root, { recursive: true, force: true }); + } +} + +// --------------------------------------------------------------------------- +// Tests + +test('cluster: DO state is consistent across instances (single writer)', async () => { + await withCluster(3, async ({ nodes }) => { + // Send a series of /increment requests to a round-robin of nodes, all for + // the same DO name. Track the maximum returned count -- it must be a + // strictly-increasing sequence (1, 2, 3, ...) because only one instance is + // the writer at any given time and the DO state is persistent. + const totalRequests = 12; + const expected = []; + for (let i = 1; i <= totalRequests; i++) expected.push(i); + + const observed = []; + const ownerIds = new Set(); + for (let i = 0; i < totalRequests; i++) { + const node = nodes[i % nodes.length]; + const { status, body } = await fetchJson( + node.httpPort, + '/increment?name=consistency' + ); + assert.strictEqual( + status, + 200, + `request to ${node.nodeId} failed: ${JSON.stringify(body)}` + ); + observed.push(body.count); + ownerIds.add(body.nodeId); + } + observed.sort((a, b) => a - b); + assert.deepStrictEqual( + observed, + expected, + `each /increment must be a unique strictly-increasing count` + ); + + // The DO is owned by exactly one node at a time. Even though we directed + // requests at all 3 nodes, every observed response should report the same + // nodeId -- the actual owner -- because incoming requests are forwarded. + assert.strictEqual( + ownerIds.size, + 1, + `expected exactly one DO owner across ${totalRequests} requests, got: ${[...ownerIds].join(', ')}` + ); + }); +}); + +test('cluster: requests to non-owner are transparently forwarded', async () => { + await withCluster(2, async ({ nodes }) => { + // Prime the DO by sending an increment to node 0. That node should claim + // ownership and respond. Then send a /get to node 1. Node 1 should forward + // to node 0 (the owner) and return the same state, and node 0 should be + // identified as the responder. + const first = await fetchJson( + nodes[0].httpPort, + '/increment?name=forwarding' + ); + assert.strictEqual(first.status, 200); + assert.strictEqual(first.body.count, 1); + const owner = first.body.nodeId; + + const second = await fetchJson(nodes[1].httpPort, '/get?name=forwarding'); + assert.strictEqual(second.status, 200); + assert.strictEqual( + second.body.count, + 1, + `forwarded /get must see prior increment` + ); + assert.strictEqual( + second.body.nodeId, + owner, + `forwarded request must be served by the original owner (${owner}), not the forwarding node (${nodes[1].nodeId})` + ); + + // Same id must be reported for both calls (sanity check for idFromName + // determinism across instances). + assert.strictEqual(first.body.id, second.body.id); + }); +}); + +test('cluster: killing the owner allows another node to take over', async () => { + await withCluster(2, async ({ nodes }) => { + // Prime the DO on whichever node ends up owning it. + const initial = await fetchJson( + nodes[0].httpPort, + '/increment?name=takeover' + ); + assert.strictEqual(initial.status, 200); + assert.strictEqual(initial.body.count, 1); + const originalOwner = initial.body.nodeId; + const ownerIndex = nodes.findIndex((n) => n.nodeId === originalOwner); + assert.notStrictEqual(ownerIndex, -1); + const survivorIndex = ownerIndex === 0 ? 1 : 0; + const survivor = nodes[survivorIndex]; + + // Hard-kill the owner. SIGKILL leaves its lease record in place, so the + // survivor takes over once that lease expires. + await nodes[ownerIndex].stop({ signal: 'SIGKILL', timeoutMs: 2000 }); + + // Hit the survivor repeatedly. Requests fail until the dead owner's lease + // expires; then the survivor takes ownership. The persisted count must be + // preserved. Each request is bounded: one that hung once held this test + // for nine minutes. + let response = null; + let lastErr = null; + const deadline = Date.now() + 30_000; + while (Date.now() < deadline) { + try { + response = await fetchJson( + survivor.httpPort, + '/increment?name=takeover', + { timeoutMs: 10_000 } + ); + if (response.status === 200) break; + } catch (err) { + lastErr = err; + } + await sleep(200); + } + assert( + response !== null && response.status === 200, + `survivor never succeeded; last error: ${lastErr}` + ); + assert.strictEqual( + response.body.nodeId, + survivor.nodeId, + `survivor (${survivor.nodeId}) must own the DO after takeover, got ${response.body.nodeId}` + ); + assert.strictEqual( + response.body.count, + 2, + `persisted state must be preserved across takeover` + ); + + // Further requests to the survivor continue to work and increment. + const followup = await fetchJson( + survivor.httpPort, + '/increment?name=takeover' + ); + assert.strictEqual(followup.status, 200); + assert.strictEqual(followup.body.count, 3); + assert.strictEqual(followup.body.nodeId, survivor.nodeId); + }); +}); + +test('cluster: draining node still routes in-flight requests to a remote owner', async () => { + await withCluster(2, async ({ nodes }) => { + // Prime the DO so that exactly one node owns it. + const initial = await fetchJson(nodes[0].httpPort, '/increment?name=drain'); + assert.strictEqual(initial.status, 200); + assert.strictEqual(initial.body.count, 1); + const owner = nodes.find((n) => n.nodeId === initial.body.nodeId); + const nonOwner = nodes.find((n) => n.nodeId !== initial.body.nodeId); + assert(owner !== undefined && nonOwner !== undefined); + + // Put a request in flight on the non-owner whose stateless handler waits + // before calling the DO, then SIGTERM the non-owner while it is waiting. + // Draining lets in-flight requests finish, so this request must still be + // able to route to the owner over cluster RPC after draining began. + const requestPromise = fetchJson( + nonOwner.httpPort, + '/increment?name=drain&delay=1000', + { timeoutMs: 10_000 } + ); + await sleep(250); + const exitPromise = nonOwner.stop({ signal: 'SIGTERM', timeoutMs: 10_000 }); + + const response = await requestPromise; + assert.strictEqual( + response.status, + 200, + `in-flight request on draining node failed: ${JSON.stringify(response.body)}` + ); + assert.strictEqual(response.body.nodeId, owner.nodeId); + assert.strictEqual(response.body.count, 2); + + // The draining node must have exited cleanly once the request finished. + const exit = await exitPromise; + assert.deepStrictEqual(exit, { code: 0, signal: null }); + }); +}); + +test('cluster: draining node refuses to claim an unowned DO', async () => { + await withCluster(2, async ({ nodes, bucket, nodeKeys }) => { + // Put a request in flight on node 0 for a DO that nobody owns yet, then + // SIGTERM node 0 while its stateless handler is still waiting. By the time + // the handler calls the DO, node 0 is draining: its cluster listener no + // longer accepts connections, so claiming the DO there would leave it + // unreachable from the rest of the cluster until node 0 exits. The claim + // must be refused instead. + const requestPromise = fetchJson( + nodes[0].httpPort, + '/increment?name=lame-duck&delay=1000', + { timeoutMs: 10_000 } + ); + await sleep(250); + const exitPromise = nodes[0].stop({ signal: 'SIGTERM', timeoutMs: 10_000 }); + + const response = await requestPromise; + assert.strictEqual( + response.status, + 500, + `draining node must not claim the DO: ${JSON.stringify(response.body)}` + ); + // The refusal is a DISCONNECTED exception, which JS sees as a generic + // connection-loss error. + assert.match(response.body.error, /Network connection lost/); + + const exit = await exitPromise; + assert.deepStrictEqual(exit, { code: 0, signal: null }); + + // The DO was never instantiated anywhere: the survivor claims it fresh. + const survivor = await fetchJson( + nodes[1].httpPort, + '/increment?name=lame-duck' + ); + assert.strictEqual(survivor.status, 200, JSON.stringify(survivor.body)); + assert.strictEqual(survivor.body.count, 1); + assert.strictEqual(survivor.body.nodeId, nodes[1].nodeId); + assert.strictEqual( + await bucket.owner(survivor.body.id), + nodeKeys.get(nodes[1].nodeId) + ); + }); +}); + +test('cluster: concurrent first requests to a DO read its owner record once per node', async () => { + await withCluster(2, async ({ nodes, bucket }) => { + // Without coalescing, each of these reads own.json and the losers retry + // their claims: the stampede that starved lease renewal on the fleet. + const results = await Promise.all( + Array.from({ length: 64 }, (_, i) => + fetchJson(nodes[i % 2].httpPort, '/increment?name=herd') + ) + ); + for (const r of results) + assert.strictEqual(r.status, 200, JSON.stringify(r.body)); + const counts = results.map((r) => r.body.count).sort((a, b) => a - b); + assert.deepStrictEqual( + counts, + Array.from({ length: 64 }, (_, i) => i + 1) + ); + const requests = await bucket.requests( + `do/counter-test-namespace/${results[0].body.id}/own.json` + ); + assert(requests <= 8, `${requests} owner record requests for one cold DO`); + }); +}); + +// A port that was free a moment ago. +async function freePort() { + const server = createServer(); + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + const { port } = server.address(); + await new Promise((resolve) => server.close(resolve)); + return port; +} + +test('cluster: an instance bound to a wildcard advertises --cluster-advertise', async () => { + const ports = []; + const extraArgs = async (i) => { + ports[i] = await freePort(); + return [ + `--cluster-addr=0.0.0.0:${ports[i]}`, + `--cluster-advertise=127.0.0.1:${ports[i]}`, + ]; + }; + await withCluster( + 2, + async ({ nodes, bucket, nodeKeys }) => { + for (const [i, node] of nodes.entries()) { + const record = JSON.parse( + await bucket.get(`nodes/${nodeKeys.get(node.nodeId)}.json`) + ); + assert.strictEqual(record.address, `127.0.0.1:${ports[i]}`); + } + // Peers reach each other at the advertised addresses. + const first = await fetchJson( + nodes[0].httpPort, + '/increment?name=advertised' + ); + const second = await fetchJson( + nodes[1].httpPort, + '/increment?name=advertised' + ); + assert.strictEqual(first.status, 200, JSON.stringify(first.body)); + assert.strictEqual(second.status, 200, JSON.stringify(second.body)); + assert.strictEqual(second.body.count, 2); + }, + { extraArgs } + ); +}); + +// The status line a CONNECT to the cluster port at `port` gets, with `auth` +// as its cf-cluster-auth header, if any. +async function clusterConnect(port, auth) { + const socket = connect(port, '127.0.0.1'); + await new Promise((resolve) => socket.once('connect', resolve)); + const header = auth === undefined ? '' : `cf-cluster-auth: ${auth}\r\n`; + socket.write( + `CONNECT cluster:1 HTTP/1.1\r\nhost: cluster:1\r\n${header}\r\n` + ); + const [data] = await Promise.race([ + new Promise((resolve) => socket.once('data', (d) => resolve([d]))), + sleep(5000).then(() => [Buffer.from('timeout')]), + ]); + socket.destroy(); + return data.toString().split('\r\n')[0]; +} + +// A cf-cluster-auth header for a CONNECT to `target`, as an instance with the +// test config's key makes it. +function clusterAuth(target, ms = Date.now(), nonce = randomBytes(16)) { + const n = nonce.toString('hex'); + const mac = createHmac('sha256', 'test-cluster-shared-secret') + .update(`workerd cluster connect v1\n${target}\n${ms}\n${n}`) + .digest('hex'); + return `${ms}.${n}.${mac}`; +} + +test('cluster: a CONNECT is refused when replayed, stale, misdirected or unsigned', async () => { + const port = await freePort(); + const extraArgs = async () => [`--cluster-addr=127.0.0.1:${port}`]; + await withCluster( + 1, + async () => { + const target = `127.0.0.1:${port}`; + const auth = clusterAuth(target); + assert.match(await clusterConnect(port, auth), / 200 /); + assert.match(await clusterConnect(port, auth), / 403 /, 'replayed'); + const stale = clusterAuth(target, Date.now() - 60_000); + assert.match(await clusterConnect(port, stale), / 403 /, 'stale'); + const elsewhere = clusterAuth(`127.0.0.1:${port + 1}`); + assert.match(await clusterConnect(port, elsewhere), / 403 /, 'target'); + assert.match(await clusterConnect(port), / 403 /, 'unsigned'); + }, + { extraArgs } + ); +}); + +test('cluster: a public cluster address is refused', async () => { + const local = await mkdtemp(join(tmpdir(), 'workerd-cluster-')); + for (const flag of ['--cluster-addr', '--cluster-advertise']) { + const child = spawn( + env.WORKERD_BINARY, + [ + 'serve', + env.WD_TEST_CONFIG, + '--experimental', + '--socket-addr=http=127.0.0.1:0', + `${flag}=8.8.8.8:7000`, + `--directory-path=local=${local}`, + ], + { stdio: ['ignore', 'ignore', 'pipe'] } + ); + let stderr = ''; + child.stderr.on('data', (d) => (stderr += d)); + const code = await new Promise((resolve) => child.once('exit', resolve)); + assert.notStrictEqual(code, 0, flag); + assert.match(stderr, /8\.8\.8\.8 is public/, flag); + } + await rm(local, { recursive: true }); +}); + +test('cluster: each running instance holds a lease record', async () => { + await withCluster(2, async ({ nodes, bucket, nodeKeys }) => { + const records = await bucket.nodes(); + assert.deepStrictEqual( + records.sort(), + [...nodeKeys.values()].sort(), + `expected one lease record per instance` + ); + for (const key of records) { + const record = JSON.parse(await bucket.get(`nodes/${key}.json`)); + assert(record.expires_ms > Date.now(), `lease of ${key} has expired`); + } + }); +}); + +test('cluster: setting an alarm is refused', async () => { + await withCluster(1, async ({ nodes }) => { + const res = await fetchJson(nodes[0].httpPort, '/set-alarm?name=alarm'); + assert.match(res.body.error ?? '', /not yet supported in cluster mode/); + }); +}); + +test('cluster: a DO reopened where it was closed resumes its epoch', async () => { + await withCluster(1, async ({ nodes, bucket }) => { + const [node] = nodes; + await fetchJson(node.httpPort, '/increment?name=dormant'); + const before = await fetchJson(node.httpPort, '/increment?name=dormant'); + assert.strictEqual(before.body.count, 2, JSON.stringify(before.body)); + const id = before.body.id; + // Breaking it drops its container, as expiry does: the database + // closes, and the bucket gets what it lacks. + await fetchJson(node.httpPort, '/break?name=dormant'); + await sleep(500); + const after = await fetchJson(node.httpPort, '/increment?name=dormant'); + assert.strictEqual(after.body.count, 3, JSON.stringify(after.body)); + const prefix = `do/counter-test-namespace/${id}`; + const record = JSON.parse(await bucket.get(`${prefix}/own.json`)); + assert.strictEqual(record.epoch, 1, JSON.stringify(record)); + assert.deepStrictEqual(await bucket.list(`${prefix}/ltx/e2/`), []); + assert.match(node.stderr, /epoch 1 resumes at 2/); + }); +}); + +test('cluster: storing and loading a stub claims nothing', async () => { + await withCluster(2, async ({ nodes, bucket }) => { + // DO "holder" (on whichever node claims it) stores a stub to DO "lazy" and + // reads it back. Serializing and deserializing the stub must not claim + // "lazy": no owner record, and no storage for it. + const stored = await fetchJson( + nodes[1].httpPort, + '/store-stub?name=holder&target=lazy' + ); + assert.strictEqual(stored.status, 200, JSON.stringify(stored.body)); + assert.strictEqual(stored.body.hasStub, true); + const holderId = stored.body.id; + const targetId = stored.body.targetId; + assert.notStrictEqual(holderId, targetId); + assert.notStrictEqual( + await bucket.owner(holderId), + null, + `the holder DO itself must be claimed` + ); + assert.strictEqual( + await bucket.owner(targetId), + null, + `storing/loading a stub must not claim its target` + ); + for (const node of nodes) { + let entries = []; + try { + entries = await readdir(join(node.localPath, 'counter-test-namespace')); + } catch (_) { + // No storage at all on this node. + } + assert( + !entries.some((name) => name.startsWith(targetId)), + `storing/loading a stub must not create actor storage; found: ${entries.join(', ')}` + ); + } + + // Calling the loaded stub claims it. + const called = await fetchJson( + nodes[1].httpPort, + '/call-stored-stub?name=holder' + ); + assert.strictEqual(called.status, 200, JSON.stringify(called.body)); + assert.strictEqual(called.body.target.id, targetId); + assert.strictEqual(called.body.target.count, 0); + assert.notStrictEqual( + await bucket.owner(targetId), + null, + `calling the stub must claim its target` + ); + }); +}); + +// Requests that would hang forever if a broken actor kept its ownership lock +// are bounded by this timeout so the failure is a clear error. +const HANG_TIMEOUT_MS = 5000; + +test('cluster: a broken DO pinned by a stub does not keep its ownership lock', async () => { + await withCluster(2, async ({ nodes, bucket, nodeKeys }) => { + // Node 0 claims "pinned". DO "holder", served by node 1, then obtains a + // stub to it, uses it, and keeps it for as long as "holder" lives. Through + // node 0's bootstrap for that stub, node 0's container for "pinned" stays + // referenced for the rest of the test. + const first = await fetchJson(nodes[0].httpPort, '/increment?name=pinned'); + assert.strictEqual(first.status, 200, JSON.stringify(first.body)); + assert.strictEqual(first.body.count, 1); + const owner = first.body.nodeId; + const objectId = first.body.id; + assert.strictEqual(await bucket.owner(objectId), nodeKeys.get(owner)); + + const held = await fetchJson( + nodes[1].httpPort, + '/hold-stub?name=holder&target=pinned' + ); + assert.strictEqual(held.status, 200, JSON.stringify(held.body)); + assert.strictEqual(held.body.target.nodeId, owner); + + // Break the DO. Its container leaves the owner's map but lives on, pinned + // by the parked stub. + const broken = await fetchJson(nodes[1].httpPort, '/break?name=pinned'); + assert.strictEqual(broken.status, 500, JSON.stringify(broken.body)); + + // The lock must have been released along with the map entry. Otherwise a + // request from the non-owner routes to the owner, which finds no container + // and routes to itself, forever. + const other = nodes.find((n) => n.nodeId !== owner); + const after = await fetchJson(other.httpPort, '/increment?name=pinned', { + timeoutMs: HANG_TIMEOUT_MS, + }); + assert.strictEqual(after.status, 200, JSON.stringify(after.body)); + assert.strictEqual(after.body.count, 2, 'state must survive the break'); + assert.strictEqual( + await bucket.owner(objectId), + nodeKeys.get(after.body.nodeId), + 'the lock file must name the node that re-instantiated the DO' + ); + }); +}); + +test('cluster: a DO whose constructor fails releases its ownership lock', async () => { + await withCluster(2, async ({ nodes, bucket, nodeKeys }) => { + const first = await fetchJson(nodes[0].httpPort, '/increment?name=ctor'); + assert.strictEqual(first.status, 200, JSON.stringify(first.body)); + const objectId = first.body.id; + + // Arm the one-shot constructor failure and abort the running instance so + // the next request has to construct a new one. + const armed = await fetchJson( + nodes[1].httpPort, + '/arm-constructor-failure?name=ctor' + ); + assert.strictEqual(armed.status, 500, JSON.stringify(armed.body)); + + // Re-instantiate through a stub kept by DO "holder". The constructor + // throws, which breaks the new instance; the kept stub pins its hollowed + // container on whichever node claimed the DO. + const held = await fetchJson( + nodes[1].httpPort, + '/hold-stub?name=holder&target=ctor', + { timeoutMs: HANG_TIMEOUT_MS } + ); + assert.strictEqual(held.status, 500, JSON.stringify(held.body)); + assert.match(held.body.error, /constructor failed on purpose/); + + // A fresh request from the other node must not hang. The failed constructor + // committed the cleared flag before throwing, so this instantiation + // succeeds and sees the earlier increment. + const after = await fetchJson(nodes[0].httpPort, '/increment?name=ctor', { + timeoutMs: HANG_TIMEOUT_MS, + }); + assert.strictEqual(after.status, 200, JSON.stringify(after.body)); + assert.strictEqual(after.body.count, 2); + assert.strictEqual( + await bucket.owner(objectId), + nodeKeys.get(after.body.nodeId), + 'the lock file must name the node that re-instantiated the DO' + ); + }); +}); + +test('cluster: aborting a DO with a request in flight lets it be re-instantiated immediately', async () => { + await withCluster(2, async ({ nodes, bucket, nodeKeys }) => { + const first = await fetchJson( + nodes[0].httpPort, + '/increment?name=inflight' + ); + assert.strictEqual(first.status, 200, JSON.stringify(first.body)); + const owner = first.body.nodeId; + const objectId = first.body.id; + + // Start a request the DO holds open, give it time to reach the DO, then + // abort the DO underneath it. The in-flight request keeps a reference to + // the old Worker::Actor until it unwinds, but the storage must be closed + // and the lock released as soon as the actor breaks, not when that + // reference goes away. + const holding = fetchJson(nodes[0].httpPort, '/hold?name=inflight', { + timeoutMs: HANG_TIMEOUT_MS, + }); + await sleep(250); + const broken = await fetchJson(nodes[0].httpPort, '/break?name=inflight'); + assert.strictEqual(broken.status, 500, JSON.stringify(broken.body)); + + const other = nodes.find((n) => n.nodeId !== owner); + const after = await fetchJson(other.httpPort, '/increment?name=inflight', { + timeoutMs: HANG_TIMEOUT_MS, + }); + assert.strictEqual(after.status, 200, JSON.stringify(after.body)); + assert.strictEqual( + after.body.count, + 2, + 'the new instance must see the state committed before the abort' + ); + assert.strictEqual( + await bucket.owner(objectId), + nodeKeys.get(after.body.nodeId), + 'the lock file must name the node that re-instantiated the DO' + ); + + const held = await holding; + assert.strictEqual(held.status, 500, JSON.stringify(held.body)); + }); +}); + +// Sets up the cross-node persistent-stub scenario of the test below: DO "root" +// is claimed by node 0, then DO "holder", served through node 1 (and therefore +// claimed by node 1), asks "root" to vend a persistent stub of the given kind +// and stores it. Returns root's node ID. +async function vendAcrossNodes(nodes, kind) { + const first = await fetchJson(nodes[0].httpPort, '/increment?name=root'); + assert.strictEqual(first.status, 200, JSON.stringify(first.body)); + const rootOwner = first.body.nodeId; + + const vended = await fetchJson( + nodes[1].httpPort, + `/vend-stub?name=holder&kind=${kind}&target=root`, + { timeoutMs: HANG_TIMEOUT_MS } + ); + assert.strictEqual(vended.status, 200, JSON.stringify(vended.body)); + assert.strictEqual(vended.body.hasStub, true); + assert.notStrictEqual( + vended.body.nodeId, + rootOwner, + 'the test requires root and holder to live on different nodes' + ); + + // Vending ran [restore]() live. Storing and loading the stub must not have + // replayed it. + const state = await fetchJson(nodes[1].httpPort, '/get?name=root'); + assert.strictEqual(state.status, 200, JSON.stringify(state.body)); + assert.strictEqual(state.body.restores, 1); + + return rootOwner; +} + +// TODO(cluster): the token-carrying peer call (bucket-mvp's getChannelFromToken). +test( + 'cluster: a restored RpcTarget stub is redeemed on the root DO owner', + { todo: true }, + async () => { + await withCluster(2, async ({ nodes }) => { + const rootOwner = await vendAcrossNodes(nodes, 'rpc'); + + // The RpcTarget mutates root's own storage, so it must run in root's + // context on root's owner. + const used = await fetchJson( + nodes[1].httpPort, + '/use-stub?name=holder&kind=rpc&amount=10', + { timeoutMs: HANG_TIMEOUT_MS } + ); + assert.strictEqual(used.status, 200, JSON.stringify(used.body)); + assert.deepStrictEqual(used.body.result, { + count: 11, + nodeId: rootOwner, + }); + + const state = await fetchJson(nodes[0].httpPort, '/get?name=root'); + assert.strictEqual(state.body.count, 11); + assert.strictEqual(state.body.restores, 2); + }); + } +); + +test('cluster: a stub resolved to a remote DO keeps failing after that DO breaks', async () => { + await withCluster(2, async ({ nodes }) => { + const first = await fetchJson(nodes[0].httpPort, '/increment?name=sticky'); + assert.strictEqual(first.status, 200, JSON.stringify(first.body)); + const owner = first.body.nodeId; + + // DO "holder" on node 1 obtains a stub to "sticky", uses it once (resolving + // it to node 0's instance), and keeps it. + const held = await fetchJson( + nodes[1].httpPort, + '/hold-stub?name=holder&target=sticky' + ); + assert.strictEqual(held.status, 200, JSON.stringify(held.body)); + assert.strictEqual(held.body.target.nodeId, owner); + assert.notStrictEqual(held.body.nodeId, owner); + + const broken = await fetchJson(nodes[1].httpPort, '/break?name=sticky'); + assert.strictEqual(broken.status, 500, JSON.stringify(broken.body)); + + // The held stub points at the broken instance and must keep failing, even + // after a fresh stub has re-instantiated the DO. + const heldAfter = await fetchJson( + nodes[1].httpPort, + '/call-held-stub?name=holder&target=sticky', + { timeoutMs: HANG_TIMEOUT_MS } + ); + assert.strictEqual(heldAfter.status, 500, JSON.stringify(heldAfter.body)); + assert.match(heldAfter.body.error, /broken on purpose/); + + const fresh = await fetchJson(nodes[1].httpPort, '/increment?name=sticky', { + timeoutMs: HANG_TIMEOUT_MS, + }); + assert.strictEqual(fresh.status, 200, JSON.stringify(fresh.body)); + assert.strictEqual(fresh.body.count, 2); + + const heldAgain = await fetchJson( + nodes[1].httpPort, + '/call-held-stub?name=holder&target=sticky', + { timeoutMs: HANG_TIMEOUT_MS } + ); + assert.strictEqual(heldAgain.status, 500, JSON.stringify(heldAgain.body)); + assert.match(heldAgain.body.error, /broken on purpose/); + }); +}); + +test('cluster: a freshly minted RpcTarget stub passed via props reaches its DO after it breaks', async () => { + await withCluster(2, async ({ nodes }) => { + const first = await fetchJson(nodes[0].httpPort, '/increment?name=root'); + assert.strictEqual(first.status, 200, JSON.stringify(first.body)); + const originalOwner = first.body.nodeId; + + // Root mints an RpcTarget stub and hands it to a Consumer entrypoint on the + // same node via props. The Consumer holds the stub's channel only -- the + // very channel object ctx.restore() created, not a decoded token -- and + // waits for the go signal before using it. + const vended = await fetchJson( + nodes[0].httpPort, + '/vend-to-consumer?name=root' + ); + assert.strictEqual(vended.status, 200, JSON.stringify(vended.body)); + + // Break root, then re-instantiate it through the other node. A broken DO + // keeps its owner, which re-instantiates it. + const broken = await fetchJson(nodes[1].httpPort, '/break?name=root'); + assert.strictEqual(broken.status, 500, JSON.stringify(broken.body)); + const moved = await fetchJson(nodes[1].httpPort, '/increment?name=root', { + timeoutMs: HANG_TIMEOUT_MS, + }); + assert.strictEqual(moved.status, 200, JSON.stringify(moved.body)); + assert.strictEqual(moved.body.count, 2); + assert.strictEqual(moved.body.nodeId, originalOwner); + + // Now let the Consumer use its stub: it reaches the re-instantiated DO. + const go = await fetchJson(nodes[0].httpPort, '/set-go?name=sink'); + assert.strictEqual(go.status, 200, JSON.stringify(go.body)); + + const deadline = Date.now() + HANG_TIMEOUT_MS; + let recorded; + while (recorded === undefined) { + assert(Date.now() < deadline, 'Consumer never recorded a result'); + await sleep(50); + const sink = await fetchJson(nodes[0].httpPort, '/get?name=sink'); + recorded = sink.body.recorded; + } + assert.deepStrictEqual(recorded, { count: 3, nodeId: moved.body.nodeId }); + + const state = await fetchJson(nodes[1].httpPort, '/get?name=root'); + assert.strictEqual(state.body.restores, 2); + }); +}); diff --git a/src/workerd/server/tests/cluster/cluster-test.wd-test b/src/workerd/server/tests/cluster/cluster-test.wd-test new file mode 100644 index 00000000000..5ac6c1f01d2 --- /dev/null +++ b/src/workerd/server/tests/cluster/cluster-test.wd-test @@ -0,0 +1,53 @@ +# Copyright (c) 2026 Cloudflare, Inc. +# Licensed under the Apache 2.0 license found in the LICENSE file or at: +# https://opensource.org/licenses/Apache-2.0 + +# Each instance keeps its SQLite files on its own disk ("local", one directory per instance), and +# membership and Durable Object ownership live in a bucket: a fake S3 (fake-s3.wd-test) that the +# test starts, whose address it passes in AWS_ENDPOINT_URL. + +using Workerd = import "/workerd/workerd.capnp"; + +const config :Workerd.Config = ( + services = [ + ( name = "local", + disk = ( + writable = true, + allowDotfiles = true, + ) + ), + + ( name = "main", + worker = ( + modules = [ + ( name = "cluster-worker.js", esModule = embed "cluster-worker.js" ), + ], + compatibilityDate = "2026-01-01", + compatibilityFlags = ["nodejs_compat", "allow_irrevocable_stub_storage"], + bindings = [ + ( name = "COUNTER", durableObjectNamespace = "Counter" ), + ( name = "NODE_ID", fromEnvironment = "NODE_ID" ), + ], + durableObjectNamespaces = [ + ( className = "Counter", uniqueKey = "counter-test-namespace" ), + ], + durableObjectStorage = ( localDisk = "local" ), + ) + ), + ], + + sockets = [ + ( name = "http", address = "*:0", http = (), service = "main" ), + ], + + cluster = ( + bucket = "s3://wdtest/cluster", + bucketOptions = [ + "aws_region=us-east-1", + "aws_access_key_id=wdtest", + "aws_secret_access_key=wdtest-secret", + ], + address = "127.0.0.1:0", + key = "test-cluster-shared-secret", + ), +); diff --git a/src/workerd/server/tests/cluster/cluster-worker.js b/src/workerd/server/tests/cluster/cluster-worker.js new file mode 100644 index 00000000000..fc07787b697 --- /dev/null +++ b/src/workerd/server/tests/cluster/cluster-worker.js @@ -0,0 +1,314 @@ +// Copyright (c) 2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +// Test worker for the cluster integration test. Exposes a Counter Durable Object +// and a top-level fetch() that routes requests to the DO by name. Requests that +// fail are reported as JSON `{ error }` with status 500 rather than letting the +// exception propagate, so the test can assert on the message. +// +// The DO supports the following request paths: +// /increment Increments and returns the current counter value. +// /get Returns the current counter value without incrementing it, +// along with `restores`, the number of times this DO's +// [restore]() method has run. +// /identity Returns information identifying this instance (NODE_ID env) +// along with the DO id, so the test can verify which instance +// served the request. +// /store-stub Stores a stub to the DO named by the `target` query param in +// this DO's storage, then reads it back without calling it. +// Returns the target's id. +// /call-stored-stub +// Loads the stub stored by /store-stub and calls /get on it. +// Returns the target's response. +// /hold-stub Obtains a stub to the DO named by the `target` query param, +// calls /identity on it, and keeps the stub on this instance +// for as long as this instance lives. (A stub is owned by the +// request context that created it, so a module-level variable +// would not keep it alive past the request; an instance field +// of a long-lived DO does.) Returns the target's response. +// /call-held-stub Calls /identity on the stub kept by /hold-stub for the DO +// named by the `target` query param. Returns the target's +// response. +// /hold Waits one second before responding, so that the test can +// break the DO while this request is still in flight. +// /break Aborts the DO via ctx.abort(). The request fails. +// /arm-constructor-failure +// Sets a storage flag that makes the *next* instantiation of +// this DO throw from its constructor (the flag is cleared and +// committed before throwing, so the instantiation after that +// succeeds), then aborts the DO so that the next request +// re-instantiates it. +// +// Persistent-stub paths. Kind `rpc` names a CounterAdder RpcTarget, vended by +// the DO given by the `target` query param via its [restore]() method, that +// mutates the target's counter: +// /vend-stub?kind=K&target=T +// Asks the target (over RPC) to vend a persistent stub of the +// given kind, stores it in this DO's storage under `kind`, and +// reads it back without calling it. +// /use-stub?kind=K[&amount=N] +// Loads the stub stored by /vend-stub and calls add(amount) +// on it. Returns the result. +// /vend-to-consumer +// Vends an `rpc` stub of *this* DO and hands it, via props, +// to a Consumer entrypoint in the same process, which keeps +// only the stub's channel (props carry no live capability). +// The Consumer waits until DO "sink" has its `go` flag set, +// then calls the stub and records the result in "sink". +// /set-go Sets this DO's `go` flag (read by Consumer via /get). +// /record?value=V Stores V (JSON) as this DO's `recorded` value. +// +// The top-level fetch() additionally honors a `delay` query parameter (ms): it +// waits that long *before* calling the DO. This lets a test put a request in +// flight on a node and SIGTERM that node while the request has not yet routed +// to the DO's owner. + +import { + DurableObject, + RpcTarget, + WorkerEntrypoint, + restore, +} from 'cloudflare:workers'; + +export default { + async fetch(request, env) { + const url = new URL(request.url); + + // The DO name to act on is given via the `name` query parameter (defaults + // to "default"). All instances using the same name will route to the same + // DO id. + const name = url.searchParams.get('name') ?? 'default'; + + // Synthesize a stub URL for the DO. + const id = env.COUNTER.idFromName(name); + const stub = env.COUNTER.get(id); + + const delay = url.searchParams.get('delay'); + if (delay !== null) { + await scheduler.wait(Number(delay)); + } + + // Forward to the DO with the same path and query. + const forwardUrl = new URL(url.pathname + url.search, 'http://do/'); + try { + return await stub.fetch(forwardUrl.toString(), { + method: request.method, + headers: request.headers, + }); + } catch (err) { + return Response.json({ error: String(err) }, { status: 500 }); + } + }, +}; + +// RpcTarget vended by Counter's [restore]() that mutates the counter's storage. +// It runs in the Counter's context on whichever node owns the Counter. +class CounterAdder extends RpcTarget { + constructor(counter) { + super(); + this.counter = counter; + } + + async add(amount) { + const count = + ((await this.counter.state.storage.get('count')) ?? 0) + amount; + await this.counter.state.storage.put('count', count); + return { count, nodeId: this.counter.env.NODE_ID ?? '' }; + } +} + +// See /vend-to-consumer. The stub in `props` was minted by ctx.restore() in this +// process and arrives here without its live capability, so the first call on it +// restores through the channel object the minting DO created, not through a +// decoded token. +export class Consumer extends WorkerEntrypoint { + useLater() { + const adder = this.ctx.props.adder; + const sink = this.env.COUNTER.get(this.env.COUNTER.idFromName('sink')); + this.ctx.waitUntil( + (async () => { + for (;;) { + const state = await (await sink.fetch('http://do/get')).json(); + if (state.go) break; + await scheduler.wait(50); + } + let result; + try { + result = await adder.add(1); + } catch (err) { + result = { error: String(err) }; + } + await sink.fetch( + 'http://do/record?value=' + encodeURIComponent(JSON.stringify(result)) + ); + })() + ); + } +} + +const BREAK_ON_CONSTRUCT = 'break-on-construct'; + +export class Counter extends DurableObject { + constructor(state, env) { + super(state, env); + this.state = state; + this.env = env; + // See /hold-stub. Maps target DO name -> stub. + this.heldStubs = new Map(); + + state.blockConcurrencyWhile(async () => { + if (await state.storage.get(BREAK_ON_CONSTRUCT)) { + // One-shot: make sure the cleared flag is durable before breaking, so + // the next instantiation succeeds. + await state.storage.delete(BREAK_ON_CONSTRUCT); + await state.storage.sync(); + throw new Error('constructor failed on purpose'); + } + }); + } + + async fetch(request) { + const url = new URL(request.url); + const nodeId = this.env.NODE_ID ?? ''; + const idHex = this.state.id.toString(); + + if (url.pathname === '/increment') { + let count = (await this.state.storage.get('count')) ?? 0; + count += 1; + await this.state.storage.put('count', count); + return Response.json({ + count, + nodeId, + id: idHex, + }); + } else if (url.pathname === '/get') { + const count = (await this.state.storage.get('count')) ?? 0; + const restores = (await this.state.storage.get('restores')) ?? 0; + const go = (await this.state.storage.get('go')) ?? false; + const recorded = await this.state.storage.get('recorded'); + return Response.json({ + count, + restores, + go, + recorded, + nodeId, + id: idHex, + }); + } else if (url.pathname === '/identity') { + return Response.json({ nodeId, id: idHex }); + } else if (url.pathname === '/store-stub') { + // Only stubs from the ctx.exports self-binding are storable. + const target = url.searchParams.get('target'); + const ns = this.state.exports.Counter; + const targetId = ns.idFromName(target); + await this.state.storage.put('stub', ns.get(targetId)); + // Read it back so that the stub is also deserialized, but do not call it. + const stub = await this.state.storage.get('stub'); + return Response.json({ + nodeId, + id: idHex, + targetId: targetId.toString(), + hasStub: typeof stub?.fetch === 'function', + }); + } else if (url.pathname === '/call-stored-stub') { + const stub = await this.state.storage.get('stub'); + const response = await stub.fetch('http://do/get'); + return Response.json({ + nodeId, + id: idHex, + target: await response.json(), + }); + } else if (url.pathname === '/hold-stub') { + const target = url.searchParams.get('target'); + const stub = this.env.COUNTER.get(this.env.COUNTER.idFromName(target)); + this.heldStubs.set(target, stub); + const response = await stub.fetch('http://do/identity'); + return Response.json({ + nodeId, + id: idHex, + target: await response.json(), + }); + } else if (url.pathname === '/call-held-stub') { + const target = url.searchParams.get('target'); + const stub = this.heldStubs.get(target); + if (stub === undefined) { + return Response.json({ error: 'no held stub' }, { status: 400 }); + } + const response = await stub.fetch('http://do/identity'); + return Response.json({ + nodeId, + id: idHex, + target: await response.json(), + }); + } else if (url.pathname === '/vend-stub') { + const kind = url.searchParams.get('kind'); + const target = url.searchParams.get('target'); + const ns = this.state.exports.Counter; + const stub = await ns.get(ns.idFromName(target)).vend(kind); + await this.state.storage.put(kind, stub); + // Read it back so that the stub is also deserialized, but do not call it. + const loaded = await this.state.storage.get(kind); + return Response.json({ + nodeId, + id: idHex, + hasStub: loaded !== undefined, + }); + } else if (url.pathname === '/use-stub') { + const kind = url.searchParams.get('kind'); + const stub = await this.state.storage.get(kind); + const result = await stub.add( + Number(url.searchParams.get('amount') ?? '1') + ); + return Response.json({ nodeId, id: idHex, result }); + } else if (url.pathname === '/vend-to-consumer') { + const adder = await this.state.restore({ kind: 'rpc' }); + const consumer = this.state.exports.Consumer({ props: { adder } }); + await consumer.useLater(); + return Response.json({ nodeId, id: idHex }); + } else if (url.pathname === '/set-go') { + await this.state.storage.put('go', true); + return Response.json({ nodeId, id: idHex }); + } else if (url.pathname === '/record') { + await this.state.storage.put( + 'recorded', + JSON.parse(url.searchParams.get('value')) + ); + return Response.json({ nodeId, id: idHex }); + } else if (url.pathname === '/hold') { + await scheduler.wait(1000); + return Response.json({ nodeId, id: idHex }); + } else if (url.pathname === '/set-alarm') { + try { + await this.state.storage.setAlarm(Date.now() + 60_000); + return Response.json({ set: true }); + } catch (e) { + return Response.json({ error: String(e) }); + } + } else if (url.pathname === '/break') { + this.state.abort('broken on purpose'); + } else if (url.pathname === '/arm-constructor-failure') { + await this.state.storage.put(BREAK_ON_CONSTRUCT, true); + await this.state.storage.sync(); + this.state.abort('armed constructor failure'); + } + + return new Response('Not found', { status: 404 }); + } + + // Present so `setAlarm` reaches the cluster's refusal. + async alarm() {} + + // RPC method used by /vend-stub on the *target* DO. + vend(kind) { + return this.state.restore({ kind }); + } + + async [restore](params) { + const restores = ((await this.state.storage.get('restores')) ?? 0) + 1; + await this.state.storage.put('restores', restores); + + if (params.kind === 'rpc') return new CounterAdder(this); + throw new Error(`unexpected restore params: ${JSON.stringify(params)}`); + } +} diff --git a/src/workerd/server/tests/cluster/fake-s3.js b/src/workerd/server/tests/cluster/fake-s3.js new file mode 100644 index 00000000000..5712066236d --- /dev/null +++ b/src/workerd/server/tests/cluster/fake-s3.js @@ -0,0 +1,115 @@ +// Copyright (c) 2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 +// +// The S3 calls a bucket cluster makes, for tests: GET, PUT (conditional on +// If-Match or If-None-Match: *), DELETE, and ListObjectsV2, path-style +// (`//`). One Durable Object holds every key, so a conditional +// write is atomic. Requests are not authenticated. + +import { DurableObject } from 'cloudflare:workers'; + +const PAGE = 1000; +// Storage values are capped at 128 KiB: a body is stored in chunks, under keys +// outside every listed prefix. +const CHUNK = 64 * 1024; +const chunkKey = (key, i) => `\0${key}\0${i}`; + +function escape(s) { + return s.replace(/&/g, '&').replace(//g, '>'); +} + +export class Bucket extends DurableObject { + next = 0; + // Requests per object key, for tests that bound a cluster's bucket traffic. + requests = new Map(); + + async fetch(request) { + const storage = this.ctx.storage; + const url = new URL(request.url); + if (url.pathname === '/_requests') { + return Response.json(this.requests.get(url.searchParams.get('key')) ?? 0); + } + const path = decodeURIComponent(url.pathname).slice(1); + const slash = path.indexOf('/'); + const key = slash < 0 ? '' : path.slice(slash + 1); + const headers = { date: new Date().toUTCString() }; + + if (request.method === 'GET' && key === '') { + const prefix = url.searchParams.get('prefix') ?? ''; + const after = url.searchParams.get('continuation-token'); + const listed = await storage.list({ + prefix, + ...(after === null ? {} : { startAfter: after }), + limit: PAGE + 1, + }); + const page = [...listed].slice(0, PAGE); + const truncated = listed.size > PAGE; + const xml = + '' + + `${truncated}` + + page + .map( + ([k, v]) => + `${escape(k)}${v.size}` + ) + .join('') + + (truncated + ? `${escape(page.at(-1)[0])}` + : '') + + ''; + return new Response(xml, { headers }); + } + + // Read the body before the object: storage calls keep other requests + // out, but awaiting the body would let one in between the check and the + // write. + const counted = key.slice(key.indexOf('/') + 1); + this.requests.set(counted, (this.requests.get(counted) ?? 0) + 1); + const body = request.method === 'PUT' ? await request.arrayBuffer() : null; + const object = await storage.get(key); + switch (request.method) { + case 'GET': + if (!object) return new Response(null, { status: 404, headers }); + const keys = Array.from({ length: object.chunks }, (_, i) => + chunkKey(key, i) + ); + const chunks = keys.length === 0 ? new Map() : await storage.get(keys); + const out = new Uint8Array(object.size); + keys.forEach((k, i) => + out.set(new Uint8Array(chunks.get(k)), i * CHUNK) + ); + return new Response(out, { + headers: { ...headers, etag: object.etag }, + }); + case 'PUT': { + const ifMatch = request.headers.get('if-match'); + const ifNoneMatch = request.headers.get('if-none-match'); + if ( + (ifNoneMatch === '*' && object) || + (ifMatch !== null && object?.etag !== ifMatch) + ) { + return new Response(null, { status: 412, headers }); + } + const etag = `"${Date.now()}-${this.next++}"`; + const chunks = Math.ceil(body.byteLength / CHUNK); + const entries = {}; + for (let i = 0; i < chunks; i++) + entries[chunkKey(key, i)] = body.slice(i * CHUNK, (i + 1) * CHUNK); + entries[key] = { etag, size: body.byteLength, chunks }; + await storage.put(entries); + return new Response(null, { headers: { ...headers, etag } }); + } + case 'DELETE': + await storage.delete(key); + return new Response(null, { status: 204, headers }); + } + return new Response(null, { status: 405, headers }); + } +} + +export default { + fetch(request, env) { + return env.BUCKET.get(env.BUCKET.idFromName('bucket')).fetch(request); + }, +}; diff --git a/src/workerd/server/tests/cluster/fake-s3.wd-test b/src/workerd/server/tests/cluster/fake-s3.wd-test new file mode 100644 index 00000000000..da369a03ef0 --- /dev/null +++ b/src/workerd/server/tests/cluster/fake-s3.wd-test @@ -0,0 +1,31 @@ +# Copyright (c) 2026 Cloudflare, Inc. +# Licensed under the Apache 2.0 license found in the LICENSE file or at: +# https://opensource.org/licenses/Apache-2.0 + +# The bucket for cluster-test: fake-s3.js on an ephemeral port, in memory. + +using Workerd = import "/workerd/workerd.capnp"; + +const config :Workerd.Config = ( + services = [ + ( name = "s3", + worker = ( + modules = [ + ( name = "fake-s3.js", esModule = embed "fake-s3.js" ), + ], + compatibilityDate = "2026-01-01", + bindings = [ + ( name = "BUCKET", durableObjectNamespace = "Bucket" ), + ], + durableObjectNamespaces = [ + ( className = "Bucket", uniqueKey = "fake-s3-bucket" ), + ], + durableObjectStorage = ( inMemory = void ), + ) + ), + ], + + sockets = [ + ( name = "http", address = "*:0", http = (), service = "s3" ), + ], +); diff --git a/src/workerd/server/workerd.capnp b/src/workerd/server/workerd.capnp index eb86a225909..c2756a10c70 100644 --- a/src/workerd/server/workerd.capnp +++ b/src/workerd/server/workerd.capnp @@ -94,6 +94,65 @@ struct Config { logging @6 : LoggingOptions; # Console and Stdio logging configuration options. + + cluster @7 :ClusterConfig; + # Set when this instance should join a cluster. Requires `--experimental`. +} + +struct ClusterConfig { + # workerd instances can work together in a cluster, to distribute Durable Objects among them. + # Each Durable Object runs on one instance at a time, its owner; any instance routes requests + # for it there. + # + # The cluster's shared state lives in an S3-compatible bucket: each instance renews a lease + # record naming its address, and each Durable Object's owner is a record claimed by a conditional + # write. Taking an object over from an owner whose lease expired bumps its ownership epoch. + # Instances need no shared filesystem: `durableObjectStorage.localDisk` is each instance's own + # disk. Durable Objects stored in memory stay local to each instance. + # + # All instances in the cluster MUST have identical configuration: any instance may forward any + # request to any other and it must produce the same result. During a rolling upgrade, instances + # may differ in backwards-compatible ways. + + bucket @0 :Text; + # The bucket that holds the cluster's shared state: `s3://bucket/prefix`. + + bucketOptions @1 :List(Text); + # `key=value` settings for `bucket`: `aws_endpoint=https://...`, `aws_region=auto`, + # `aws_access_key_id=...`, `aws_secret_access_key=...`, and `aws_token=...`. In place of the + # last three, `aws_credentials_file=` names a file of `AWS_ACCESS_KEY_ID=`, + # `AWS_SECRET_ACCESS_KEY=` and `AWS_SESSION_TOKEN=` lines, reread whenever it changes, so + # short-lived credentials rotate without a restart. Replace it by rename, not in place. Each + # setting not given falls back to an environment variable: `AWS_ENDPOINT_URL`, `AWS_REGION`, + # `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `AWS_SESSION_TOKEN`, `AWS_CREDENTIALS_FILE`. + + address @2 :Text; + # Where this instance listens for other instances: `host:port`, a port of 0 for an ephemeral + # one. Overridden by `--cluster-addr`. Holders of `key` get full access to every service in the + # process through it, and its traffic is not encrypted: only the cluster's instances should + # reach it, on a private network (a VPC, WireGuard, Tailscale). A public IP is refused unless + # `unsafePublicAddress` is set; a wildcard (`0.0.0.0`) listens on every interface, so firewall + # the port. + + advertise @4 :Text; + # The address other instances reach this one at, which goes in its lease record in the bucket. + # Optional: by default, the address bound, with the port it got. Required when that is a + # wildcard (`0.0.0.0:7000`) or not what peers can reach (NAT, a container). Overridden by + # `--cluster-advertise`. A public IP is refused unless `unsafePublicAddress` is set. + + key @3 :Text; + # A secret known only to members of the cluster. Each connection between instances opens with + # a MAC keyed by it, over the target, a timestamp and a nonce, so a header seen on the wire + # cannot be replayed; the instances' clocks must agree within 30 seconds. It also keys channel + # tokens, which pass service and Durable Object stubs over RPC. Traffic is not encrypted: run + # the cluster on a private network (a VPC, WireGuard, Tailscale). + # + # TODO(someday): Support some way to gradually roll out a rotated key. + + unsafePublicAddress @5 :Bool = false; + # Allow a public IP as `address` or `advertise`. Refused by default, since traffic between + # instances is not encrypted. Private, loopback, link-local and shared (100.64.0.0/10, as + # Tailscale uses) addresses, wildcards and hostnames are allowed. } struct LoggingOptions { @@ -816,8 +875,8 @@ struct Worker { # extensions `.sqlite-wal`, and `.sqlite-shm` may also be present.) } - # TODO(someday): Support distributing objects across a cluster. At present, objects are always - # local to one instance of the runtime. + # In a cluster (`Config.cluster`), objects stored on `localDisk` are distributed across the + # instances; see `ClusterConfig`. moduleFallback @13 :Text; diff --git a/src/workerd/util/sqlite-kv-test.c++ b/src/workerd/util/sqlite-kv-test.c++ index 57f2e24b9f9..b092050ca53 100644 --- a/src/workerd/util/sqlite-kv-test.c++ +++ b/src/workerd/util/sqlite-kv-test.c++ @@ -338,5 +338,25 @@ KJ_TEST("SQLite-KV multi-put with allowUnconfirmed") { KJ_EXPECT(called); } +KJ_TEST("SQLite-KV reads throw after close()") { + auto dir = kj::newInMemoryDirectory(kj::nullClock()); + SqliteDatabase::Vfs vfs(*dir); + SqliteDatabase db(vfs, kj::Path({"foo"}), kj::WriteMode::CREATE | kj::WriteMode::MODIFY); + SqliteKv kv(db); + + kv.put("foo", "abc"_kj.asBytes()); + kv.putExternals("foo", kj::arr(kj::heapArray("tok"_kj.asBytes()))); + + db.close(); + + // Unlike reset(), close() must not make the data appear deleted: the file still has it. + KJ_EXPECT_THROW_MESSAGE("database has been closed", + kv.get("foo", [](kj::ArrayPtr) { KJ_FAIL_EXPECT("should not be called"); })); + KJ_EXPECT_THROW_MESSAGE( + "database has been closed", kv.list(nullptr, kj::none, kj::none, SqliteKv::FORWARD)->next()); + KJ_EXPECT_THROW_MESSAGE("database has been closed", kv.getExternals("foo")); + KJ_EXPECT_THROW_MESSAGE("database has been closed", kv.put("bar", "def"_kj.asBytes())); +} + } // namespace } // namespace workerd diff --git a/src/workerd/util/sqlite-kv.c++ b/src/workerd/util/sqlite-kv.c++ index 6dd044b8b30..047e8bf1e33 100644 --- a/src/workerd/util/sqlite-kv.c++ +++ b/src/workerd/util/sqlite-kv.c++ @@ -276,10 +276,16 @@ void SqliteKv::putExternals(kj::StringPtr key, kj::Array> tokens } } -void SqliteKv::beforeSqliteReset() { - // We'll need to recreate the tables on the next operation. - tableCreated = false; - externalsTableCreated = false; +void SqliteKv::beforeSqliteClose(SqliteDatabase::CloseReason reason) { + if (reason == SqliteDatabase::CloseReason::RESET) { + // The tables are deleted along with the database and will need to be recreated on the next + // operation. + tableCreated = false; + externalsTableCreated = false; + } + // On close(), leave the flags alone: the tables still exist in the file. Clearing them would + // make get()/list()/getExternals() report an empty database without consulting the (closed) + // connection; with the flags intact those reads go through a Statement and throw instead. } void SqliteKv::rollbackMultiPut(Initialized& stmts, WriteOptions options) { diff --git a/src/workerd/util/sqlite-kv.h b/src/workerd/util/sqlite-kv.h index a2e85670a41..815b25e302f 100644 --- a/src/workerd/util/sqlite-kv.h +++ b/src/workerd/util/sqlite-kv.h @@ -226,7 +226,7 @@ class SqliteKv: private SqliteDatabase::ResetListener { // with `allowUnconfirmed = true` since the paired KV write decides confirmation semantics. void clearExternalsIfPresent(KeyPtr key, SqliteQueryId queryId); - void beforeSqliteReset() override; + void beforeSqliteClose(SqliteDatabase::CloseReason reason) override; // Returns `queryId` if set, otherwise mints a fresh one. Used so every statement of a single // top-level KV operation shares one grouping id, while recursive operations (e.g. multi-put) diff --git a/src/workerd/util/sqlite-metadata-test.c++ b/src/workerd/util/sqlite-metadata-test.c++ index b153a44e781..f899f8c545f 100644 --- a/src/workerd/util/sqlite-metadata-test.c++ +++ b/src/workerd/util/sqlite-metadata-test.c++ @@ -65,5 +65,23 @@ KJ_TEST("SQLite-METADATA") { KJ_EXPECT(metadata.getAlarm() == anAlarmTime2); } +KJ_TEST("SQLite-METADATA reads throw after close()") { + auto dir = kj::newInMemoryDirectory(kj::nullClock()); + SqliteDatabase::Vfs vfs(*dir); + SqliteDatabase db(vfs, kj::Path({"foo"}), kj::WriteMode::CREATE | kj::WriteMode::MODIFY); + SqliteMetadata metadata(db); + + constexpr kj::Date anAlarmTime = kj::UNIX_EPOCH + 1734099316 * kj::SECONDS; + metadata.setAlarm(anAlarmTime, /*allowUnconfirmed=*/false); + KJ_EXPECT(metadata.getAlarm() == anAlarmTime); + + db.close(); + + // Unlike reset(), close() must not make the alarm appear unset: the file still has it. + KJ_EXPECT_THROW_MESSAGE("database has been closed", metadata.getAlarm()); + KJ_EXPECT_THROW_MESSAGE( + "database has been closed", metadata.setAlarm(kj::none, /*allowUnconfirmed=*/false)); +} + } // namespace } // namespace workerd diff --git a/src/workerd/util/sqlite-metadata.c++ b/src/workerd/util/sqlite-metadata.c++ index 6acdbdb2528..7cc4d5625a0 100644 --- a/src/workerd/util/sqlite-metadata.c++ +++ b/src/workerd/util/sqlite-metadata.c++ @@ -100,10 +100,18 @@ SqliteMetadata::Initialized& SqliteMetadata::ensureInitialized(bool allowUnconfi KJ_UNREACHABLE; } -void SqliteMetadata::beforeSqliteReset() { - // We'll need to recreate the table on the next operation. - tableCreated = false; +void SqliteMetadata::beforeSqliteClose(SqliteDatabase::CloseReason reason) { + // Drop the alarm cache in both cases. After a reset() the alarm is gone; after a close() the + // next read must reach the closed connection and throw rather than report a stale value. cacheState = kj::none; + + if (reason == SqliteDatabase::CloseReason::RESET) { + // The table is deleted along with the database and will need to be recreated on the next + // operation. Leave `tableCreated` alone on close(): the table still exists in the file, and + // clearing the flag would make getAlarmUncached() report "no alarm" without consulting the + // (closed) database. + tableCreated = false; + } } } // namespace workerd diff --git a/src/workerd/util/sqlite-metadata.h b/src/workerd/util/sqlite-metadata.h index c365166a213..9df50dbca6d 100644 --- a/src/workerd/util/sqlite-metadata.h +++ b/src/workerd/util/sqlite-metadata.h @@ -75,7 +75,7 @@ class SqliteMetadata final: private SqliteDatabase::ResetListener { // first write. // ResetListener interface: - void beforeSqliteReset() override; + void beforeSqliteClose(SqliteDatabase::CloseReason reason) override; }; } // namespace workerd diff --git a/src/workerd/util/sqlite-test.c++ b/src/workerd/util/sqlite-test.c++ index 6f4fe81fd8b..c28da858b49 100644 --- a/src/workerd/util/sqlite-test.c++ +++ b/src/workerd/util/sqlite-test.c++ @@ -292,6 +292,39 @@ KJ_TEST("SQLite backed by real disk") { } } +KJ_TEST("SQLite VFS reports WAL writes") { + TempDirOnDisk dir; + uint headers = 0, frames = 0; + SqliteDatabase::Vfs vfs(*dir, + {.walWrite = kj::Function)>( + [&](kj::StringPtr file, uint64_t offset, kj::ArrayPtr data) { + KJ_EXPECT(file.endsWith("foo-wal"), file); + // The 32-byte WAL header at offset 0, then each frame's header and page. + if (offset == 0) { + KJ_EXPECT(data.size() == 32); + ++headers; + } else { + ++frames; + } + })}); + + SqliteDatabase db(vfs, kj::Path({"foo"}), kj::WriteMode::CREATE | kj::WriteMode::MODIFY); + setupSql(db); + checkSql(db); + KJ_EXPECT(headers >= 1); + KJ_EXPECT(frames >= 2); +} + +KJ_TEST("SQLite VFS refuses a WAL it cannot report") { + auto dir = kj::newInMemoryDirectory(kj::nullClock()); + SqliteDatabase::Vfs vfs(*dir, + {.walWrite = kj::Function)>( + [](kj::StringPtr, uint64_t, kj::ArrayPtr) {})}); + SqliteDatabase db(vfs, kj::Path({"foo"}), kj::WriteMode::CREATE | kj::WriteMode::MODIFY); + // The VFS reports the refusal to SQLite as a failure to open the WAL. + KJ_EXPECT_THROW_MESSAGE("SQLITE_CANTOPEN", setupSql(db)); +} + // Tests that a read-only database client picks up changes made to the database by a read/write // client. void doReadOnlyUpdateTest(const kj::Directory& dir) { diff --git a/src/workerd/util/sqlite.c++ b/src/workerd/util/sqlite.c++ index 62470afd1d1..bec3e6d3919 100644 --- a/src/workerd/util/sqlite.c++ +++ b/src/workerd/util/sqlite.c++ @@ -696,7 +696,11 @@ SqliteDatabase::~SqliteDatabase() noexcept(false) { } SqliteDatabase::operator sqlite3*() { - return &KJ_ASSERT_NONNULL(maybeDb, "previous reset() failed"); + KJ_IF_SOME(db, maybeDb) { + return &db; + } + KJ_REQUIRE(!closed, "database has been closed"); + KJ_FAIL_ASSERT("previous reset() failed"); } SqliteMemoryScope SqliteDatabase::enterMemoryScope() { @@ -722,7 +726,7 @@ void SqliteDatabase::handleCriticalError(kj::Maybe errorCode, if (code == SQLITE_FULL || code == SQLITE_IOERR || code == SQLITE_NOMEM || code == SQLITE_INTERRUPT) { - sqlite3* db = &KJ_ASSERT_NONNULL(maybeDb, "previous reset() failed"); + sqlite3* db = *this; // We are in a transaction if (inTransaction || !savepoints.empty()) { // The transaction was auto-rolledback, re-enabling the auto commit mode, so we should fail @@ -847,7 +851,7 @@ SqliteDatabase::StatementAndEffect SqliteDatabase::prepareSql(StaticRegulator re Multi multi, kj::Maybe&> prelude, kj::Maybe queryId) { - sqlite3* db = &KJ_ASSERT_NONNULL(maybeDb, "previous reset() failed"); + sqlite3* db = *this; ParseContext parseContext; KJ_ASSERT(currentParseContext == kj::none, "recursive prepareSql()?"); @@ -1054,6 +1058,7 @@ void SqliteDatabase::executeWithRegulator( void SqliteDatabase::reset() { KJ_REQUIRE(!readOnly, "can't reset() read-only database"); + KJ_REQUIRE(!closed, "can't reset() a database that has been closed"); // If transactions are open during reset(), whatever had the transaction open is going to get // confused at best, or lose data at worst. Let's just not allow this. @@ -1067,7 +1072,7 @@ void SqliteDatabase::reset() { KJ_IF_SOME(db, maybeDb) { for (auto& listener: resetListeners) { - listener.beforeSqliteReset(); + listener.beforeSqliteClose(CloseReason::RESET); } auto err = sqlite3_close(&db); @@ -1086,6 +1091,44 @@ void SqliteDatabase::reset() { } } +void SqliteDatabase::close() noexcept { + if (closed) return; + + // If we're currently executing a SQLite statement, there's no safe way for us to close the DB. + // But, callers of close() rely on the fact that the DB will in fact be closed, and they often + // sit on error-handling paths that cannot tolerate further exceptions. These asserts, if they + // fail, will abort the process, due to close() being noexcept. As of this writing it is believed + // that no real code path is likely to hit these, so these are just safety checks / internal + // invariants. + KJ_ASSERT(currentStatement == kj::none, "close() called while a query is executing"); + KJ_ASSERT(currentRegulator == kj::none, "close() called while a query is compiling"); + + auto memoryScope = enterMemoryScope(); + + KJ_IF_SOME(db, maybeDb) { + for (auto& listener: resetListeners) { + listener.beforeSqliteClose(CloseReason::CLOSE); + } + + // Every prepared statement was finalized by the listener walk above, so SQLITE_BUSY here + // indicates a statement that isn't tracked by a ResetListener. Do not fall back to + // sqlite3_close_v2(): a lazily-closed handle keeps the file open, which is exactly what this + // method exists to prevent. + auto err = sqlite3_close(&db); + KJ_ASSERT( + err == SQLITE_OK, "close() found dependent objects that still exist", sqlite3_errstr(err)); + + maybeDb = kj::none; + } + + // sqlite3_close() rolled back any open transaction. Nothing may read the database again, so the + // rollback callbacks (which exist to keep in-memory caches in sync with it) are simply dropped. + inTransaction = false; + savepoints.clear(); + rollbackCallbacks.clear(); + closed = true; +} + bool SqliteDatabase::isAuthorized(int actionCode, kj::Maybe param1, kj::Maybe param2, @@ -1550,10 +1593,22 @@ SqliteDatabase::StatementAndEffect& SqliteDatabase::Statement::prepareForExecuti return KJ_ASSERT_NONNULL(stmt.tryGet()); } -void SqliteDatabase::Statement::beforeSqliteReset() { +void SqliteDatabase::Statement::beforeSqliteClose(CloseReason reason) { + // Regardless of `reason`, the prepared statement must be finalized (by destroying the + // StatementAndEffect) so that the connection can be closed. KJ_IF_SOME(prepared, stmt.tryGet()) { - // Pull the original SQL code out of the statement and store it. - stmt = kj::str(sqlite3_sql(prepared.statement)); + switch (reason) { + case CloseReason::RESET: + // Keep the SQL text so that the statement can be reprepared against the new database. + stmt = kj::str(sqlite3_sql(prepared.statement)); + break; + case CloseReason::CLOSE: + // The statement can never be reprepared, so don't bother copying the SQL. An empty string + // (no allocation) suffices: prepareForExecution() will attempt to prepare it and fail + // with "database has been closed" before ever looking at the text. + stmt = kj::String(); + break; + } } } @@ -1856,15 +1911,20 @@ bool SqliteDatabase::Query::isNull(uint column) { SqliteDatabase::StatementAndEffect& SqliteDatabase::Query::getStatementAndEffect() { return KJ_UNWRAP_OR(maybeStatement, { - regulator->onError(kj::none, "SQLite query was canceled because the database was deleted."); - KJ_FAIL_REQUIRE("query canceled because reset() was called on the database"); + if (db.closed) { + regulator->onError(kj::none, "SQLite query was canceled because the database was closed."); + KJ_FAIL_REQUIRE("query canceled because the database has been closed"); + } else { + regulator->onError(kj::none, "SQLite query was canceled because the database was deleted."); + KJ_FAIL_REQUIRE("query canceled because reset() was called on the database"); + } }); } -void SqliteDatabase::Query::beforeSqliteReset() { +void SqliteDatabase::Query::beforeSqliteClose(CloseReason reason) { // Note that if we don't own the statement, then `maybeStatement` is probably already dangling // here. Luckily, we don't need to reset it or anything because the statement will be destroyed - // by Statement::beforeSqliteReset(). + // by Statement::beforeSqliteClose(). maybeStatement = kj::none; ownStatement = {}; } @@ -1942,6 +2002,11 @@ struct SqliteDatabase::Vfs::WrappedNativeFileImpl: public sqlite3_file { const Vfs* vfs; int rootFd; + // Set for a WAL file when the VFS reports WAL writes (see VfsOptions::walWrite). SQLite keeps + // `walName` valid until the file closes. + kj::Function)>* walWrite; + const char* walName; + // It's expected that the wrapped sqlite_file begins in memory immediately after this object. sqlite3_file* getWrapped() { return reinterpret_cast(this + 1); @@ -2001,7 +2066,17 @@ const sqlite3_io_methods SqliteDatabase::Vfs::WrappedNativeFileImpl::METHOD_TABL WRAP(xClose), WRAP(xRead), - WRAP(xWrite), + .xWrite = [](sqlite3_file* file, const void* data, int amount, sqlite3_int64 offset) + noexcept -> int { + int result = MethodWrapperHack::wrapper(file, data, amount, offset); + auto wrapper = static_cast(file); + if (result == SQLITE_OK && wrapper->walWrite != nullptr) { + (*wrapper->walWrite)(wrapper->walName, offset, + kj::arrayPtr(static_cast(data), amount)); + } + return result; + }, WRAP(xTruncate), WRAP(xSync), WRAP(xFileSize), @@ -2082,7 +2157,7 @@ sqlite3_vfs SqliteDatabase::Vfs::makeWrappedNativeVfs() { file->pMethods = nullptr; // Set up currentVfsRoot. - auto& self = *reinterpret_cast(vfs->pAppData); + auto& self = *reinterpret_cast(vfs->pAppData); KJ_ASSERT(currentVfsRoot == AT_FDCWD); currentVfsRoot = self.rootFd; KJ_DEFER(currentVfsRoot = AT_FDCWD); @@ -2094,9 +2169,26 @@ sqlite3_vfs SqliteDatabase::Vfs::makeWrappedNativeVfs() { if (file->pMethods == nullptr) { wrapper->pMethods = nullptr; } else { + KJ_IF_SOME(afterOpen, self.options.afterOpen) { + try { + afterOpen(); + } catch (kj::Exception& e) { + reportVfsErrorCaught(kj::mv(e)); + file->pMethods->xClose(file); + wrapper->pMethods = nullptr; + return SQLITE_CANTOPEN; + } + } wrapper->pMethods = &WrappedNativeFileImpl::METHOD_TABLE; wrapper->vfs = &self; wrapper->rootFd = self.rootFd; + wrapper->walWrite = nullptr; + KJ_IF_SOME(walWrite, self.options.walWrite) { + if (flags & SQLITE_OPEN_WAL) { + wrapper->walWrite = &walWrite; + wrapper->walName = zName; + } + } } return result; @@ -2398,7 +2490,7 @@ sqlite3_vfs SqliteDatabase::Vfs::makeKjVfs() { .pAppData = this, #define WRAP_METHOD(errorCode, block) \ - auto& self KJ_UNUSED = *static_cast(vfs->pAppData); \ + auto& self KJ_UNUSED = *static_cast(vfs->pAppData); \ try block catch (kj::Exception& e) { \ reportVfsErrorCaught(kj::mv(e)); \ return errorCode; \ @@ -2415,6 +2507,11 @@ sqlite3_vfs SqliteDatabase::Vfs::makeKjVfs() { auto path = kj::Path::parse(zName); auto kjFile = KJ_UNWRAP_OR(self.directory.tryOpenFile(path), { return SQLITE_CANTOPEN; }); + KJ_IF_SOME(afterOpen, self.options.afterOpen) { + afterOpen(); + } else { + // squelch spurious "dangling else" warning from clang + } kj::Maybe> lock; if (flags & SQLITE_OPEN_MAIN_DB) { lock = self.lockManager.lock(path, *kjFile); @@ -2430,6 +2527,11 @@ sqlite3_vfs SqliteDatabase::Vfs::makeKjVfs() { KJ_ASSERT(flags & SQLITE_OPEN_DELETEONCLOSE); KJ_ASSERT(!(flags & SQLITE_OPEN_MAIN_DB), "main DB can't be a temporary file"); kjFile = self.directory.createTemporary(); + KJ_IF_SOME(afterOpen, self.options.afterOpen) { + afterOpen(); + } else { + // squelch spurious "dangling else" warning from clang + } } else { kj::WriteMode mode; if (flags & SQLITE_OPEN_CREATE) { @@ -2442,9 +2544,18 @@ sqlite3_vfs SqliteDatabase::Vfs::makeKjVfs() { mode = kj::WriteMode::MODIFY; } + // Only the native VFS reports WAL writes; a WAL a caller asked to hear from but cannot + // would commit transactions it never sees. + KJ_REQUIRE(self.options.walWrite == kj::none || !(flags & SQLITE_OPEN_WAL), + "walWrite needs a disk directory"); auto path = kj::Path::parse(zName); kjFile = KJ_UNWRAP_OR(self.directory.tryOpenFile(path, mode), { return SQLITE_CANTOPEN; }); + KJ_IF_SOME(afterOpen, self.options.afterOpen) { + afterOpen(); + } else { + // squelch spurious "dangling else" warning from clang + } if (flags & SQLITE_OPEN_MAIN_DB) { lock = self.lockManager.lock(path, *kjFile); } diff --git a/src/workerd/util/sqlite.h b/src/workerd/util/sqlite.h index bbe1e538948..2258806c278 100644 --- a/src/workerd/util/sqlite.h +++ b/src/workerd/util/sqlite.h @@ -307,7 +307,31 @@ class SqliteDatabase { // may throw if they depend on tables that haven't been recreated yet). void reset(); - // Objects that need to be notified when reset() is called may inherit `ResetListener`. + // Closes the underlying SQLite connection without reopening it. Every subsequent attempt to + // use the database, including through existing Statements and Queries, throws. Any open + // transaction is rolled back (never committed). Idempotent. + // + // This exists so that an actor's storage can be severed at the moment the actor is shut down, + // even if references to the SqliteDatabase live on: closing the last connection checkpoints and + // deletes the WAL and releases any file locks, which the next opener of the same file needs. + // + // This is declared noexcept because it is expected to be used in shutdown paths which are not + // prepared to catch further errors and strongly depend on the database actually becoming closed + // (so that they can safely release file locks). If close() is unable to complete (e.g. because + // a sqlite statement is currently being executed) then we have no choice but to abort the + // process. + void close() noexcept; + + // Why a connection is being closed. Passed to `ResetListener::beforeSqliteClose()`. + enum class CloseReason { + // reset(): the database file is deleted and a fresh connection opened. + RESET, + // close(): the connection is gone for good. + CLOSE, + }; + + // Objects that need to be notified when reset() or close() is called may inherit + // `ResetListener`. class ResetListener { public: ResetListener(SqliteDatabase& db): db(db) { @@ -321,9 +345,16 @@ class SqliteDatabase { db.resetListeners.add(*this); } - // When the database's `reset()` method is called, all listeners' `beforeSqliteReset()` will be - // called before actually resetting the database. - virtual void beforeSqliteReset() = 0; + // When the database's `reset()` or `close()` method is called, all listeners' + // `beforeSqliteClose()` will be called before actually closing the connection. Any prepared + // statements must be finalized here so that `sqlite3_close()` can succeed. + // + // The `reason` matters for listeners that cache facts about the database's contents: after + // RESET the database is empty and will be reopened, so such caches must be cleared; after + // CLOSE nothing may read the database again, so such caches should be left alone so that + // attempted reads reach the closed connection and throw rather than reporting an empty + // database. + virtual void beforeSqliteClose(CloseReason reason) = 0; protected: // so that subclasses don't have to store their own copy of the `db` reference SqliteDatabase& db; @@ -357,6 +388,11 @@ class SqliteDatabase { // When a rollback occurs, callbacks are invoked in the reverse of the order in which they were // registered. The database content is rolled back first, before invoking any callbacks. // Callbacks may read from the database, but must not write to it. + // + // NOTE: close() technically rolls back all outstanding transactions, but does NOT call rollback + // callbacks. Instead, callers who need to handle the close() case should register a + // ResetListener. In practices, in-memory caches are generally moot after close() since the + // actor has shut down anyway. void onRollback(kj::Function callback) { if (inTransaction || !savepoints.empty()) { rollbackCallbacks.add(kj::mv(callback)); @@ -388,9 +424,12 @@ class SqliteDatabase { kj::Maybe actorAccountLimits; - // This pointer can be left null if a call to reset() failed to re-open the database. + // Null after a call to reset() failed to re-open the database, or after close(). kj::Maybe maybeDb; + // Set by close(). Distinguishes the closed state from a failed reset() in error messages. + bool closed = false; + // Set while a query is compiling. kj::Maybe currentRegulator; @@ -560,7 +599,7 @@ class SqliteDatabase::Statement final: private ResetListener { regulator(regulator), stmt(kj::mv(sqlCode)) {} - void beforeSqliteReset() override; + void beforeSqliteClose(CloseReason reason) override; // Get the underlying StatementAndEffect, which the caller will then execute. If `prelude` is // non-empty, prepareForExecution() actually executes the prelude. @@ -833,7 +872,7 @@ class SqliteDatabase::Query final: private ResetListener { return getStatementAndEffect().statement; } - void beforeSqliteReset() override; + void beforeSqliteClose(CloseReason reason) override; void nextRow(bool first); }; @@ -857,6 +896,16 @@ struct SqliteDatabase::VfsOptions { // will fall back to the native VFS implementation. In that case, the options you set here will // be ORed with the ones set by the underlying VFS. int deviceCharacteristics = 0x00001000; // = SQLITE_FCNTL_POWERSAFE_OVERWRITE + + // Called immediately after SQLite opens any file through this VFS. Clustered Durable Object + // storage uses it to refuse opening a file once this instance's cluster lease has lapsed. + kj::Maybe> afterOpen; + + // Called after each successful write to a WAL file, with the file's name as SQLite gave it. + // Bucket-backed Durable Objects replicate by cutting each committed transaction's frames from + // these writes. Only the native VFS (a disk directory) reports them. + kj::Maybe)>> + walWrite; }; // Implements a SQLite VFS based on a KJ directory. From 9b3378a1ad4ece23c46fc5e0fc02579f1eaa4e40 Mon Sep 17 00:00:00 2001 From: Ryan Dahl Date: Mon, 26 Oct 2026 13:01:22 -0700 Subject: [PATCH 02/38] cluster: fix CI on Windows, ASan and clippy capture.rs read the WAL with the Unix-only read_exact_at; it seeks and reads instead. The ltx test VFS copied 160 bytes out of SQLite's 152-byte io_methods into a struct whose tail is a non-null fn pointer; each file now keeps its patched table in its own allocation, so nothing is overread or leaked. Drop a word clippy's doc_markdown flags. --- src/workerd/server/cli/args.rs | 2 +- src/workerd/server/cluster/capture.rs | 8 +++-- src/workerd/server/cluster/ltx_tests.rs | 48 ++++++++++++++----------- 3 files changed, 33 insertions(+), 25 deletions(-) diff --git a/src/workerd/server/cli/args.rs b/src/workerd/server/cli/args.rs index 2ae0b173457..520e3ca2706 100644 --- a/src/workerd/server/cli/args.rs +++ b/src/workerd/server/cli/args.rs @@ -338,7 +338,7 @@ pub struct ServeArgs { /// Bind the cluster's peer port to instead of the config's `cluster.address`. The /// port gives full access to every service in the process to holders of the cluster key, and /// its traffic is not encrypted: only the cluster's instances should reach it, on a private - /// network (a VPC, WireGuard, Tailscale). A public IP is refused unless the config sets + /// network (a VPC or Tailscale). A public IP is refused unless the config sets /// `cluster.unsafePublicAddress`; a wildcard (0.0.0.0) listens on every interface, so /// firewall the port. #[arg(long = "cluster-addr", value_name = "addr")] diff --git a/src/workerd/server/cluster/capture.rs b/src/workerd/server/cluster/capture.rs index bbc39a94ace..db2e3498e60 100644 --- a/src/workerd/server/cluster/capture.rs +++ b/src/workerd/server/cluster/capture.rs @@ -18,8 +18,9 @@ use std::collections::BTreeMap; use std::collections::HashMap; use std::fs::File; use std::io::Read; +use std::io::Seek; +use std::io::SeekFrom; use std::io::Write; -use std::os::unix::fs::FileExt; use std::path::Path; use std::path::PathBuf; @@ -150,13 +151,14 @@ impl Capture { Ok(wal) if wal.metadata().is_ok_and(|m| m.len() == 0) => None, wal => Some(wal.map_err(crate::ltx::Error::from)?), }; - if let Some(wal) = wal.filter(|_| !self.pages.is_empty()) { + if let Some(mut wal) = wal.filter(|_| !self.pages.is_empty()) { for (&pgno, &offset) in &self.pages { let at = (pgno as usize - 1) * page_size; if image.len() < at + page_size { image.resize(at + page_size, 0); } - wal.read_exact_at(&mut image[at..at + page_size], offset) + wal.seek(SeekFrom::Start(offset)) + .and_then(|_| wal.read_exact(&mut image[at..at + page_size])) .map_err(crate::ltx::Error::from)?; } image.truncate(self.commit as usize * page_size); diff --git a/src/workerd/server/cluster/ltx_tests.rs b/src/workerd/server/cluster/ltx_tests.rs index 8cdc9a52059..2b6875ac7df 100644 --- a/src/workerd/server/cluster/ltx_tests.rs +++ b/src/workerd/server/cluster/ltx_tests.rs @@ -42,7 +42,7 @@ struct Vfs { rest: [*const c_void; 15], } -/// `sqlite3_io_methods`, likewise, followed by the method it replaces. +/// `sqlite3_io_methods`, likewise. #[repr(C)] #[derive(Clone, Copy)] struct Methods { @@ -51,7 +51,15 @@ struct Methods { read: *const c_void, write: unsafe extern "C" fn(*mut File, *const c_void, c_int, i64) -> c_int, rest: [*const c_void; 15], +} + +/// A file's own method table, followed by the method it replaces and +/// whether the file is a WAL. It sits past the wrapped file's state. +#[repr(C)] +struct Patched { + methods: Methods, wrapped_write: unsafe extern "C" fn(*mut File, *const c_void, c_int, i64) -> c_int, + is_wal: bool, } #[repr(C)] @@ -155,11 +163,10 @@ thread_local! { static CAPTURE: RefCell> = const { RefCell::new(None) }; } -/// The wrapped VFS, and where a file's "is a WAL" flag sits past the -/// wrapped file's own state. +/// The wrapped VFS, and where a file's `Patched` sits. struct Wrapped { vfs: usize, - flag_at: usize, + patched_at: usize, } fn wrapped() -> &'static Wrapped { @@ -168,26 +175,21 @@ fn wrapped() -> &'static Wrapped { // registered here is leaked, as `SQLite` requires. WRAPPED.get_or_init(|| unsafe { let native = sqlite3_vfs_find(std::ptr::null()); - let flag_at = usize::try_from((*native).file_size) + let patched_at = usize::try_from((*native).file_size) .unwrap() .next_multiple_of(8); let mut vfs = *native; vfs.name = c"capture-test".as_ptr(); - vfs.file_size = c_int::try_from(flag_at + 8).unwrap(); + vfs.file_size = c_int::try_from(patched_at + size_of::()).unwrap(); vfs.open = open; assert_eq!(sqlite3_vfs_register(Box::leak(Box::new(vfs)), 0), 0); Wrapped { vfs: native as usize, - flag_at, + patched_at, } }) } -unsafe fn is_wal(file: *mut File) -> *mut bool { - // SAFETY: the flag is within the `file_size` bytes `SQLite` allocated. - unsafe { file.cast::().add(wrapped().flag_at).cast() } -} - unsafe extern "C" fn open( _: *mut Vfs, name: *const c_char, @@ -195,17 +197,20 @@ unsafe extern "C" fn open( flags: c_int, out: *mut c_int, ) -> c_int { - // SAFETY: `SQLite` passes a file of the wrapper's size; the patched - // method table is leaked, so it outlives the file. + // SAFETY: `SQLite` passes a file of the wrapper's size, so the + // patched table lives as long as the file. unsafe { let native = wrapped().vfs as *mut Vfs; let rc = ((*native).open)(native, name, file, flags, out); if rc == 0 && !(*file).methods.is_null() { - *is_wal(file) = flags & SQLITE_OPEN_WAL != 0; - let mut methods = *(*file).methods; - methods.wrapped_write = methods.write; - methods.write = write; - (*file).methods = Box::leak(Box::new(methods)); + let methods = *(*file).methods; + let patched: *mut Patched = file.cast::().add(wrapped().patched_at).cast(); + patched.write(Patched { + methods: Methods { write, ..methods }, + wrapped_write: methods.write, + is_wal: flags & SQLITE_OPEN_WAL != 0, + }); + (*file).methods = patched.cast_const().cast(); } rc } @@ -214,8 +219,9 @@ unsafe extern "C" fn open( unsafe extern "C" fn write(file: *mut File, data: *const c_void, n: c_int, offset: i64) -> c_int { // SAFETY: `data` holds `n` bytes, and the file was opened by `open`. unsafe { - let rc = ((*(*file).methods).wrapped_write)(file, data, n, offset); - if rc == 0 && *is_wal(file) { + let patched = &*(*file).methods.cast::(); + let rc = (patched.wrapped_write)(file, data, n, offset); + if rc == 0 && patched.is_wal { let data = std::slice::from_raw_parts(data.cast::(), usize::try_from(n).unwrap()); let offset = u64::try_from(offset).unwrap(); CAPTURE.with_borrow_mut(|c| c.as_mut().unwrap().write(offset, data)); From 3b3d847b69e59381b89a799ef0059313229e40de Mon Sep 17 00:00:00 2001 From: Ryan Dahl Date: Mon, 26 Oct 2026 13:15:28 -0700 Subject: [PATCH 03/38] cluster: refuse walWrite on Windows Windows SQLite uses the KJ VFS, which cannot report WAL writes, so a cluster object there acknowledged commits it never uploaded: the takeover test on Windows recovered nothing. Refuse walWrite on Windows and skip the WAL-report and cluster tests there. --- src/workerd/server/tests/cluster/BUILD.bazel | 5 +++++ src/workerd/util/sqlite-test.c++ | 2 ++ src/workerd/util/sqlite.c++ | 2 ++ 3 files changed, 9 insertions(+) diff --git a/src/workerd/server/tests/cluster/BUILD.bazel b/src/workerd/server/tests/cluster/BUILD.bazel index ee9543cf79c..fdfc05e0aed 100644 --- a/src/workerd/server/tests/cluster/BUILD.bazel +++ b/src/workerd/server/tests/cluster/BUILD.bazel @@ -20,4 +20,9 @@ js_test( "FAKE_S3_CONFIG": "$(rootpath :fake-s3.wd-test)", }, tags = ["js-test"], + # SQLite on Windows cannot report WAL writes, which cluster mode needs. + target_compatible_with = select({ + "@platforms//os:windows": ["@platforms//:incompatible"], + "//conditions:default": [], + }), ) diff --git a/src/workerd/util/sqlite-test.c++ b/src/workerd/util/sqlite-test.c++ index c28da858b49..dd1272fd00c 100644 --- a/src/workerd/util/sqlite-test.c++ +++ b/src/workerd/util/sqlite-test.c++ @@ -292,6 +292,7 @@ KJ_TEST("SQLite backed by real disk") { } } +#if !_WIN32 KJ_TEST("SQLite VFS reports WAL writes") { TempDirOnDisk dir; uint headers = 0, frames = 0; @@ -324,6 +325,7 @@ KJ_TEST("SQLite VFS refuses a WAL it cannot report") { // The VFS reports the refusal to SQLite as a failure to open the WAL. KJ_EXPECT_THROW_MESSAGE("SQLITE_CANTOPEN", setupSql(db)); } +#endif // Tests that a read-only database client picks up changes made to the database by a read/write // client. diff --git a/src/workerd/util/sqlite.c++ b/src/workerd/util/sqlite.c++ index bec3e6d3919..329aed7c8b9 100644 --- a/src/workerd/util/sqlite.c++ +++ b/src/workerd/util/sqlite.c++ @@ -2863,6 +2863,8 @@ SqliteDatabase::Vfs::Vfs(const kj::Directory& directory, Options options) options(kj::mv(options)), native(*sqlite3_vfs_find(nullptr)) { #if _WIN32 + // Only the native VFS reports WAL writes, and Windows uses the KJ VFS. + KJ_REQUIRE(this->options.walWrite == kj::none, "walWrite is not supported on Windows"); vfs = kj::heap(makeKjVfs()); #else KJ_IF_SOME(fd, directory.getFd()) { From fbb5d75e1b55ad878dcd1f8a7f2f3b8c4f59c463 Mon Sep 17 00:00:00 2001 From: Ryan Dahl Date: Mon, 26 Oct 2026 14:07:04 -0700 Subject: [PATCH 04/38] cluster: keep the cluster crate's tests out of downstream builds The capture tests register a Rust VFS with SQLite. A downstream release build compiles SQLite with -fsanitize=cfi-icall, so its first call into the Rust xOpen traps (SIGILL). Production calls Rust from C++ directly through cxx and is unaffected. --- src/workerd/server/cluster/BUILD.bazel | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/workerd/server/cluster/BUILD.bazel b/src/workerd/server/cluster/BUILD.bazel index c4d6432180d..4b1d43e95e9 100644 --- a/src/workerd/server/cluster/BUILD.bazel +++ b/src/workerd/server/cluster/BUILD.bazel @@ -7,6 +7,9 @@ wd_rust_crate( proc_macro_deps = ["@crates_vendor//:async-trait"], # The capture's tests drive a real SQLite. test_deps = ["@sqlite3"], + # The capture's tests hand SQLite Rust callbacks, which a downstream + # build's cfi-icall rejects. + test_tags = ["no-downstream"], visibility = ["//src/workerd/server:__subpackages__"], deps = [ "//src/workerd/server/cluster/logic:cluster-logic", From b464e41c3d9e2adc173e23a2422dfc62c368e959 Mon Sep 17 00:00:00 2001 From: Ryan Dahl Date: Mon, 26 Oct 2026 14:33:26 -0700 Subject: [PATCH 05/38] cluster: capture WAL writes through a directory, not VFS hooks Clustered actors' SQLite opens files through a kj::Directory wrapper, as edgeworker's storage relay does: opening checks the lease, and a WAL file reports each successful write. It has no fd, so SQLite uses the KJ VFS on every platform. This drops VfsOptions::afterOpen and walWrite and their native-VFS plumbing, leaving sqlite.c++'s VFS as on main, and the cluster test runs on Windows again. --- .../server/factory/worker-factory-actor.c++ | 166 ++++++++++++++++-- .../server/factory/worker-factory-impl.h | 6 +- src/workerd/server/tests/cluster/BUILD.bazel | 5 - src/workerd/util/sqlite-test.c++ | 35 ---- src/workerd/util/sqlite.c++ | 59 +------ src/workerd/util/sqlite.h | 10 -- 6 files changed, 157 insertions(+), 124 deletions(-) diff --git a/src/workerd/server/factory/worker-factory-actor.c++ b/src/workerd/server/factory/worker-factory-actor.c++ index 34b3dae33b4..69f8eaf914b 100644 --- a/src/workerd/server/factory/worker-factory-actor.c++ +++ b/src/workerd/server/factory/worker-factory-actor.c++ @@ -15,6 +15,152 @@ namespace workerd::server { // ======================================================================================= // ActorStorage +namespace { + +// The directory a clustered namespace's SQLite opens its files through. Opening fails once this +// instance's cluster lease has lapsed, and writes to a WAL are reported to the cluster, which cuts +// commits from them. It has no fd, so SQLite uses the KJ VFS, on every platform. +class ClusterDirectory final: public kj::Directory { + public: + ClusterDirectory(kj::Own inner, const ActorNamespaceHandle& actors) + : inner(kj::mv(inner)), + actors(actors) {} + + Metadata stat() const override { + return inner->stat(); + } + void sync() const override { + inner->sync(); + } + void datasync() const override { + inner->datasync(); + } + kj::Own cloneFsNode() const override { + return kj::heap(inner->clone(), actors); + } + kj::Array listNames() const override { + return inner->listNames(); + } + kj::Array listEntries() const override { + return inner->listEntries(); + } + bool exists(kj::PathPtr path) const override { + return inner->exists(path); + } + kj::Maybe tryLstat(kj::PathPtr path) const override { + return inner->tryLstat(path); + } + kj::Maybe> tryOpenFile(kj::PathPtr path) const override { + actors.check_open(); + return inner->tryOpenFile(path); + } + kj::Maybe> tryOpenSubdir(kj::PathPtr path) const override { + return inner->tryOpenSubdir(path); + } + kj::Maybe tryReadlink(kj::PathPtr path) const override { + return inner->tryReadlink(path); + } + kj::Maybe> tryOpenFile( + kj::PathPtr path, kj::WriteMode mode) const override { + actors.check_open(); + auto name = path.toString(); + return inner->tryOpenFile(path, mode) + .map([&](kj::Own&& file) -> kj::Own { + if (!name.endsWith(".sqlite-wal")) return kj::mv(file); + return kj::heap(kj::mv(file), kj::mv(name), actors); + }); + } + kj::Own> replaceFile(kj::PathPtr path, kj::WriteMode mode) const override { + return inner->replaceFile(path, mode); + } + kj::Own createTemporary() const override { + actors.check_open(); + return inner->createTemporary(); + } + kj::Maybe> tryAppendFile( + kj::PathPtr path, kj::WriteMode mode) const override { + return inner->tryAppendFile(path, mode); + } + kj::Maybe> tryOpenSubdir( + kj::PathPtr path, kj::WriteMode mode) const override { + return inner->tryOpenSubdir(path, mode); + } + kj::Own> replaceSubdir( + kj::PathPtr path, kj::WriteMode mode) const override { + return inner->replaceSubdir(path, mode); + } + bool trySymlink(kj::PathPtr linkpath, kj::StringPtr content, kj::WriteMode mode) const override { + return inner->trySymlink(linkpath, content, mode); + } + bool tryTransfer(kj::PathPtr toPath, + kj::WriteMode toMode, + const kj::Directory& fromDirectory, + kj::PathPtr fromPath, + kj::TransferMode mode) const override { + return inner->tryTransfer(toPath, toMode, fromDirectory, fromPath, mode); + } + bool tryRemove(kj::PathPtr path) const override { + return inner->tryRemove(path); + } + + private: + // A WAL file: each write, once it succeeds, goes to the cluster. + class WalFile final: public kj::File { + public: + WalFile(kj::Own inner, kj::String name, const ActorNamespaceHandle& actors) + : inner(kj::mv(inner)), + name(kj::mv(name)), + actors(actors) {} + + Metadata stat() const override { + return inner->stat(); + } + void sync() const override { + inner->sync(); + } + void datasync() const override { + inner->datasync(); + } + kj::Own cloneFsNode() const override { + return kj::heap(inner->clone(), kj::str(name), actors); + } + size_t read(uint64_t offset, kj::ArrayPtr buffer) const override { + return inner->read(offset, buffer); + } + kj::Array mmap(uint64_t offset, uint64_t size) const override { + return inner->mmap(offset, size); + } + kj::Array mmapPrivate(uint64_t offset, uint64_t size) const override { + return inner->mmapPrivate(offset, size); + } + void write(uint64_t offset, kj::ArrayPtr data) const override { + inner->write(offset, data); + actors.wal_write( + toRust(name), offset, ::rust::Slice(data.begin(), data.size())); + } + void zero(uint64_t offset, uint64_t size) const override { + inner->zero(offset, size); + } + void truncate(uint64_t size) const override { + inner->truncate(size); + } + kj::Own mmapWritable( + uint64_t offset, uint64_t size) const override { + return inner->mmapWritable(offset, size); + } + + private: + kj::Own inner; + kj::String name; + const ActorNamespaceHandle& actors; + }; + + kj::Own inner; + const ActorNamespaceHandle& actors; +}; + +} // namespace + ActorStorage::Impl::Impl(const WorkerFactory& factory, kj::Maybe> directoryParam, kj::String uniqueKey, @@ -25,7 +171,10 @@ ActorStorage::Impl::Impl(const WorkerFactory& factory, .orDefault([&]() { return kj::newInMemoryDirectory(kj::systemPreciseCalendarClock()); })), - vfs(*vfsDirectory, vfsOptions()), + sqliteDirectory(actors->replicated() + ? kj::heap(vfsDirectory->clone(), *actors) + : vfsDirectory->clone()), + vfs(*sqliteDirectory), schedulerVfs(*vfsDirectory), uniqueKey(kj::mv(uniqueKey)), alarmScheduler(kj::heap(kj::systemPreciseCalendarClock(), @@ -40,21 +189,6 @@ ActorStorage::Impl::Impl(const WorkerFactory& factory, return actors->actor_for_alarm(kj::heap(kj::mv(id))); })) {} -SqliteDatabase::VfsOptions ActorStorage::Impl::vfsOptions() { - SqliteDatabase::VfsOptions options; - if (actors->replicated()) { - // A clustered namespace's files open only while this instance's lease holds, and their WAL - // writes are how the cluster captures commits. - options.afterOpen = [this]() { actors->check_open(); }; - options.walWrite = [this]( - kj::StringPtr file, uint64_t offset, kj::ArrayPtr data) { - actors->wal_write( - toRust(file), offset, ::rust::Slice(data.begin(), data.size())); - }; - } - return options; -} - kj::Maybe> ActorStorage::Impl::getFacetTreeIndexIfNotEmpty( const kj::Directory& dir, kj::StringPtr rootKey) { return dir.tryOpenFile(kj::Path({kj::str(rootKey, ".facets")}), kj::WriteMode::MODIFY) diff --git a/src/workerd/server/factory/worker-factory-impl.h b/src/workerd/server/factory/worker-factory-impl.h index c3c7cedaf29..caafb2e1e60 100644 --- a/src/workerd/server/factory/worker-factory-impl.h +++ b/src/workerd/server/factory/worker-factory-impl.h @@ -219,8 +219,11 @@ class ActorStorage::Impl final: public kj::Refcounted { ::rust::Box actors; kj::Maybe> directory; kj::Own vfsDirectory; // the in-memory directory when there is none + // What actors' SQLite opens files through: `vfsDirectory`, or in a cluster a wrapper that + // fences opens on the lease and reports WAL writes. + kj::Own sqliteDirectory; SqliteDatabase::Vfs vfs; - // The scheduler's database is no actor's, so it opens without the cluster's VFS hooks. + // The scheduler's database is no actor's, so it opens outside the cluster's wrapper. SqliteDatabase::Vfs schedulerVfs; kj::String uniqueKey; kj::Own alarmScheduler; @@ -231,7 +234,6 @@ class ActorStorage::Impl final: public kj::Refcounted { const kj::Directory& dir, kj::StringPtr rootKey); kj::Path getSqlitePath(kj::StringPtr rootKey, uint facetId, kj::StringPtr suffix = ""_kj); - SqliteDatabase::VfsOptions vfsOptions(); void deleteFacet( const kj::Directory& dir, kj::StringPtr rootKey, FacetTreeIndex& index, uint facetId); void deleteDescendantStorage(const kj::Directory& dir, kj::StringPtr rootKey, uint parentId); diff --git a/src/workerd/server/tests/cluster/BUILD.bazel b/src/workerd/server/tests/cluster/BUILD.bazel index fdfc05e0aed..ee9543cf79c 100644 --- a/src/workerd/server/tests/cluster/BUILD.bazel +++ b/src/workerd/server/tests/cluster/BUILD.bazel @@ -20,9 +20,4 @@ js_test( "FAKE_S3_CONFIG": "$(rootpath :fake-s3.wd-test)", }, tags = ["js-test"], - # SQLite on Windows cannot report WAL writes, which cluster mode needs. - target_compatible_with = select({ - "@platforms//os:windows": ["@platforms//:incompatible"], - "//conditions:default": [], - }), ) diff --git a/src/workerd/util/sqlite-test.c++ b/src/workerd/util/sqlite-test.c++ index dd1272fd00c..6f4fe81fd8b 100644 --- a/src/workerd/util/sqlite-test.c++ +++ b/src/workerd/util/sqlite-test.c++ @@ -292,41 +292,6 @@ KJ_TEST("SQLite backed by real disk") { } } -#if !_WIN32 -KJ_TEST("SQLite VFS reports WAL writes") { - TempDirOnDisk dir; - uint headers = 0, frames = 0; - SqliteDatabase::Vfs vfs(*dir, - {.walWrite = kj::Function)>( - [&](kj::StringPtr file, uint64_t offset, kj::ArrayPtr data) { - KJ_EXPECT(file.endsWith("foo-wal"), file); - // The 32-byte WAL header at offset 0, then each frame's header and page. - if (offset == 0) { - KJ_EXPECT(data.size() == 32); - ++headers; - } else { - ++frames; - } - })}); - - SqliteDatabase db(vfs, kj::Path({"foo"}), kj::WriteMode::CREATE | kj::WriteMode::MODIFY); - setupSql(db); - checkSql(db); - KJ_EXPECT(headers >= 1); - KJ_EXPECT(frames >= 2); -} - -KJ_TEST("SQLite VFS refuses a WAL it cannot report") { - auto dir = kj::newInMemoryDirectory(kj::nullClock()); - SqliteDatabase::Vfs vfs(*dir, - {.walWrite = kj::Function)>( - [](kj::StringPtr, uint64_t, kj::ArrayPtr) {})}); - SqliteDatabase db(vfs, kj::Path({"foo"}), kj::WriteMode::CREATE | kj::WriteMode::MODIFY); - // The VFS reports the refusal to SQLite as a failure to open the WAL. - KJ_EXPECT_THROW_MESSAGE("SQLITE_CANTOPEN", setupSql(db)); -} -#endif - // Tests that a read-only database client picks up changes made to the database by a read/write // client. void doReadOnlyUpdateTest(const kj::Directory& dir) { diff --git a/src/workerd/util/sqlite.c++ b/src/workerd/util/sqlite.c++ index 329aed7c8b9..56daba7b64d 100644 --- a/src/workerd/util/sqlite.c++ +++ b/src/workerd/util/sqlite.c++ @@ -2002,11 +2002,6 @@ struct SqliteDatabase::Vfs::WrappedNativeFileImpl: public sqlite3_file { const Vfs* vfs; int rootFd; - // Set for a WAL file when the VFS reports WAL writes (see VfsOptions::walWrite). SQLite keeps - // `walName` valid until the file closes. - kj::Function)>* walWrite; - const char* walName; - // It's expected that the wrapped sqlite_file begins in memory immediately after this object. sqlite3_file* getWrapped() { return reinterpret_cast(this + 1); @@ -2066,17 +2061,7 @@ const sqlite3_io_methods SqliteDatabase::Vfs::WrappedNativeFileImpl::METHOD_TABL WRAP(xClose), WRAP(xRead), - .xWrite = [](sqlite3_file* file, const void* data, int amount, sqlite3_int64 offset) - noexcept -> int { - int result = MethodWrapperHack::wrapper(file, data, amount, offset); - auto wrapper = static_cast(file); - if (result == SQLITE_OK && wrapper->walWrite != nullptr) { - (*wrapper->walWrite)(wrapper->walName, offset, - kj::arrayPtr(static_cast(data), amount)); - } - return result; - }, + WRAP(xWrite), WRAP(xTruncate), WRAP(xSync), WRAP(xFileSize), @@ -2157,7 +2142,7 @@ sqlite3_vfs SqliteDatabase::Vfs::makeWrappedNativeVfs() { file->pMethods = nullptr; // Set up currentVfsRoot. - auto& self = *reinterpret_cast(vfs->pAppData); + auto& self = *reinterpret_cast(vfs->pAppData); KJ_ASSERT(currentVfsRoot == AT_FDCWD); currentVfsRoot = self.rootFd; KJ_DEFER(currentVfsRoot = AT_FDCWD); @@ -2169,26 +2154,9 @@ sqlite3_vfs SqliteDatabase::Vfs::makeWrappedNativeVfs() { if (file->pMethods == nullptr) { wrapper->pMethods = nullptr; } else { - KJ_IF_SOME(afterOpen, self.options.afterOpen) { - try { - afterOpen(); - } catch (kj::Exception& e) { - reportVfsErrorCaught(kj::mv(e)); - file->pMethods->xClose(file); - wrapper->pMethods = nullptr; - return SQLITE_CANTOPEN; - } - } wrapper->pMethods = &WrappedNativeFileImpl::METHOD_TABLE; wrapper->vfs = &self; wrapper->rootFd = self.rootFd; - wrapper->walWrite = nullptr; - KJ_IF_SOME(walWrite, self.options.walWrite) { - if (flags & SQLITE_OPEN_WAL) { - wrapper->walWrite = &walWrite; - wrapper->walName = zName; - } - } } return result; @@ -2490,7 +2458,7 @@ sqlite3_vfs SqliteDatabase::Vfs::makeKjVfs() { .pAppData = this, #define WRAP_METHOD(errorCode, block) \ - auto& self KJ_UNUSED = *static_cast(vfs->pAppData); \ + auto& self KJ_UNUSED = *static_cast(vfs->pAppData); \ try block catch (kj::Exception& e) { \ reportVfsErrorCaught(kj::mv(e)); \ return errorCode; \ @@ -2507,11 +2475,6 @@ sqlite3_vfs SqliteDatabase::Vfs::makeKjVfs() { auto path = kj::Path::parse(zName); auto kjFile = KJ_UNWRAP_OR(self.directory.tryOpenFile(path), { return SQLITE_CANTOPEN; }); - KJ_IF_SOME(afterOpen, self.options.afterOpen) { - afterOpen(); - } else { - // squelch spurious "dangling else" warning from clang - } kj::Maybe> lock; if (flags & SQLITE_OPEN_MAIN_DB) { lock = self.lockManager.lock(path, *kjFile); @@ -2527,11 +2490,6 @@ sqlite3_vfs SqliteDatabase::Vfs::makeKjVfs() { KJ_ASSERT(flags & SQLITE_OPEN_DELETEONCLOSE); KJ_ASSERT(!(flags & SQLITE_OPEN_MAIN_DB), "main DB can't be a temporary file"); kjFile = self.directory.createTemporary(); - KJ_IF_SOME(afterOpen, self.options.afterOpen) { - afterOpen(); - } else { - // squelch spurious "dangling else" warning from clang - } } else { kj::WriteMode mode; if (flags & SQLITE_OPEN_CREATE) { @@ -2544,18 +2502,9 @@ sqlite3_vfs SqliteDatabase::Vfs::makeKjVfs() { mode = kj::WriteMode::MODIFY; } - // Only the native VFS reports WAL writes; a WAL a caller asked to hear from but cannot - // would commit transactions it never sees. - KJ_REQUIRE(self.options.walWrite == kj::none || !(flags & SQLITE_OPEN_WAL), - "walWrite needs a disk directory"); auto path = kj::Path::parse(zName); kjFile = KJ_UNWRAP_OR(self.directory.tryOpenFile(path, mode), { return SQLITE_CANTOPEN; }); - KJ_IF_SOME(afterOpen, self.options.afterOpen) { - afterOpen(); - } else { - // squelch spurious "dangling else" warning from clang - } if (flags & SQLITE_OPEN_MAIN_DB) { lock = self.lockManager.lock(path, *kjFile); } @@ -2863,8 +2812,6 @@ SqliteDatabase::Vfs::Vfs(const kj::Directory& directory, Options options) options(kj::mv(options)), native(*sqlite3_vfs_find(nullptr)) { #if _WIN32 - // Only the native VFS reports WAL writes, and Windows uses the KJ VFS. - KJ_REQUIRE(this->options.walWrite == kj::none, "walWrite is not supported on Windows"); vfs = kj::heap(makeKjVfs()); #else KJ_IF_SOME(fd, directory.getFd()) { diff --git a/src/workerd/util/sqlite.h b/src/workerd/util/sqlite.h index 2258806c278..c51fe1e46f8 100644 --- a/src/workerd/util/sqlite.h +++ b/src/workerd/util/sqlite.h @@ -896,16 +896,6 @@ struct SqliteDatabase::VfsOptions { // will fall back to the native VFS implementation. In that case, the options you set here will // be ORed with the ones set by the underlying VFS. int deviceCharacteristics = 0x00001000; // = SQLITE_FCNTL_POWERSAFE_OVERWRITE - - // Called immediately after SQLite opens any file through this VFS. Clustered Durable Object - // storage uses it to refuse opening a file once this instance's cluster lease has lapsed. - kj::Maybe> afterOpen; - - // Called after each successful write to a WAL file, with the file's name as SQLite gave it. - // Bucket-backed Durable Objects replicate by cutting each committed transaction's frames from - // these writes. Only the native VFS (a disk directory) reports them. - kj::Maybe)>> - walWrite; }; // Implements a SQLite VFS based on a KJ directory. From 89703de5cab150a76fced3b270b18e88e35d0aff Mon Sep 17 00:00:00 2001 From: Ryan Dahl Date: Mon, 26 Oct 2026 21:30:48 -0700 Subject: [PATCH 06/38] cluster: log commits as Cap'n Proto frames of pages, not LTX An object's log is a stream of frames (frames.capnp), each a change to one lane, written by an ownership epoch in sequence. The database is a sqlitePages lane: a commit is a frame of the pages it wrote, so merging a run of commits keeps one copy of each page, and a snapshot is a header and the database. Restore and compaction replay frames into a file, which replaces LTX's k-way merge; a frame a wider file already applied is skipped by its sequence. Each file is an LZ4 frame with a content checksum. Lanes leave room for other kinds of state beside the database, and for commits that span them. --- src/workerd/server/cluster/BUILD.bazel | 6 + src/workerd/server/cluster/bucket.rs | 4 +- src/workerd/server/cluster/capture.rs | 277 +++------ src/workerd/server/cluster/frames.capnp | 83 +++ src/workerd/server/cluster/lib.rs | 16 +- src/workerd/server/cluster/log.rs | 458 +++++++++++++++ .../cluster/{ltx_tests.rs => log_tests.rs} | 108 +++- .../server/cluster/ltx/client/bucket.rs | 136 ----- src/workerd/server/cluster/ltx/client/mod.rs | 48 -- src/workerd/server/cluster/ltx/codec.rs | 354 ------------ src/workerd/server/cluster/ltx/error.rs | 22 - src/workerd/server/cluster/ltx/ltx.rs | 542 ------------------ src/workerd/server/cluster/ltx/mod.rs | 92 --- src/workerd/server/cluster/ltx/replica.rs | 157 ----- src/workerd/server/cluster/owner.rs | 236 ++------ 15 files changed, 772 insertions(+), 1767 deletions(-) create mode 100644 src/workerd/server/cluster/frames.capnp create mode 100644 src/workerd/server/cluster/log.rs rename src/workerd/server/cluster/{ltx_tests.rs => log_tests.rs} (85%) delete mode 100644 src/workerd/server/cluster/ltx/client/bucket.rs delete mode 100644 src/workerd/server/cluster/ltx/client/mod.rs delete mode 100644 src/workerd/server/cluster/ltx/codec.rs delete mode 100644 src/workerd/server/cluster/ltx/error.rs delete mode 100644 src/workerd/server/cluster/ltx/ltx.rs delete mode 100644 src/workerd/server/cluster/ltx/mod.rs delete mode 100644 src/workerd/server/cluster/ltx/replica.rs diff --git a/src/workerd/server/cluster/BUILD.bazel b/src/workerd/server/cluster/BUILD.bazel index 4b1d43e95e9..f30174c16fc 100644 --- a/src/workerd/server/cluster/BUILD.bazel +++ b/src/workerd/server/cluster/BUILD.bazel @@ -1,3 +1,4 @@ +load("//:build/wd_capnp_library.bzl", "wd_capnp_library") load("//:build/wd_rust_crate.bzl", "wd_rust_crate") # Durable Objects whose shared state is an S3-compatible bucket. Pure Rust: @@ -12,8 +13,10 @@ wd_rust_crate( test_tags = ["no-downstream"], visibility = ["//src/workerd/server:__subpackages__"], deps = [ + ":frames_capnp_rust", "//src/workerd/server/cluster/logic:cluster-logic", "@crates_vendor//:bytes", + "@crates_vendor//:capnp", "@crates_vendor//:futures", "@crates_vendor//:http", "@crates_vendor//:http-body", @@ -33,3 +36,6 @@ wd_rust_crate( "@crates_vendor//:tracing", ], ) + +# The format of an object's log in the bucket. +wd_capnp_library(src = "frames.capnp") diff --git a/src/workerd/server/cluster/bucket.rs b/src/workerd/server/cluster/bucket.rs index 5535ff9acd5..f65e38200b3 100644 --- a/src/workerd/server/cluster/bucket.rs +++ b/src/workerd/server/cluster/bucket.rs @@ -185,7 +185,7 @@ pub mod memory { let got = self.objects.borrow().get(key).cloned(); match (got, to) { (Some((body, version)), Some(to)) => { - std::fs::write(to, body).map_err(crate::ltx::Error::from)?; + std::fs::write(to, body)?; Ok(Some((Vec::new(), version))) } (got, _) => Ok(got), @@ -201,7 +201,7 @@ pub mod memory { ) -> Result> { let body = match body { Payload::Bytes(bytes) => bytes.to_vec(), - Payload::File(path) => std::fs::read(path).map_err(crate::ltx::Error::from)?, + Payload::File(path) => std::fs::read(path)?, }; let mut objects = self.objects.borrow_mut(); let current = objects.get(key).map(|(_, version)| version); diff --git a/src/workerd/server/cluster/capture.rs b/src/workerd/server/cluster/capture.rs index db2e3498e60..09f7563d2c1 100644 --- a/src/workerd/server/cluster/capture.rs +++ b/src/workerd/server/cluster/capture.rs @@ -3,38 +3,28 @@ // https://opensource.org/licenses/Apache-2.0 //! A Durable Object's commits, cut from the writes `SQLite` makes to its WAL -//! as workerd's VFS reports them: one LTX entry per transaction. +//! as workerd's VFS reports them: one log entry, a frame of the pages it +//! wrote, per transaction. //! //! `SQLite` writes a frame as its 24-byte header and then its page, at offsets //! that follow the 32-byte WAL header, and the header of a transaction's //! last frame carries the database's size, which marks the commit. A write -//! at offset 0 starts a new WAL generation, after a checkpoint has copied -//! every earlier frame into the database file. So the frames since the -//! last commit are buffered, and the write that completes a commit frame +//! at offset 0 starts a new WAL, after a checkpoint has copied every +//! earlier frame into the database file. So the frames since the last +//! commit are buffered, and the write that completes a commit frame //! completes one transaction. A rolled-back transaction's frames start //! where the next one's do, which overwrites them. use std::collections::BTreeMap; -use std::collections::HashMap; -use std::fs::File; -use std::io::Read; -use std::io::Seek; -use std::io::SeekFrom; -use std::io::Write; use std::path::Path; use std::path::PathBuf; use crate::Error; use crate::Result; use crate::Txid; -use crate::ltx::TXID; -use crate::ltx::codec::Encoder; -use crate::ltx::ltx::HEADER_FLAG_NO_CHECKSUM; -use crate::ltx::ltx::Header; -use crate::ltx::ltx::PageHeader; -use crate::ltx::ltx::VERSION; -use crate::ltx::ltx::encode_file; -use crate::ltx::ltx::lock_pgno; +use crate::log::Body; +use crate::log::Cursor; +use crate::log::push; const WAL_HEADER: u64 = 32; const FRAME_HEADER: usize = 24; @@ -42,21 +32,18 @@ const FRAME_HEADER: usize = 24; pub struct Capture { /// The database file. path: PathBuf, - /// The chain's entries not yet in the bucket or this node's journal, by - /// txid. + /// The chain's entries not yet in the bucket, by txid. entries: BTreeMap>, last: Txid, + cursor: Cursor, page_size: usize, /// Where the next transaction's first frame goes: 0 before a WAL header. start: u64, /// What was written from `start` on. frames: Vec, - /// Where this WAL generation's latest committed version of each page - /// is, for a snapshot, and the database's size at the last commit. - pages: HashMap, - commit: u32, - /// The database was deleted: the next commit is a whole image, since a - /// new database's first page reaches the file, not the WAL. + /// The lane does not exist: the next commit creates it with every page + /// of the file, since a new database's first page reaches the file, not + /// the WAL. fresh: bool, /// Why the capture can no longer be trusted, if it cannot. lost: Option, @@ -64,125 +51,75 @@ pub struct Capture { impl Capture { /// Capture the database at `path`, which no connection has open yet: - /// what it holds now is the snapshot at txid 1, held in memory. + /// what it holds now is txid 1. #[cfg(test)] pub fn open(path: &Path) -> Result { - let mut capture = Self::resume(path, 0); + let mut capture = Self::resume(path, 0, Cursor::default()); capture.restart()?; Ok(capture) } - /// Continue a chain through `last` over the database at `path`, which - /// holds exactly that chain and no connection has open yet: an object - /// woken where it was evicted. - pub fn resume(path: &Path, last: Txid) -> Self { + /// Continue a chain through `last`, at `cursor`, over the database at + /// `path`, which holds exactly that chain and no connection has open + /// yet. A chain at 0 starts with the next commit. + pub fn resume(path: &Path, last: Txid, cursor: Cursor) -> Self { Self { path: path.to_owned(), entries: BTreeMap::new(), last, + cursor, page_size: 0, start: 0, frames: Vec::new(), - pages: HashMap::new(), - commit: 0, - fresh: false, + fresh: last == 0, lost: None, } } - /// Start a new chain from a snapshot of what is committed now. + /// Start a new chain from the database file, which holds every commit: + /// no WAL, or one a checkpoint truncated. #[cfg(test)] pub fn restart(&mut self) -> Result<()> { self.entries.clear(); - self.last = 0; - if let Some(snapshot) = self.snapshot(1, 1)? { + (self.last, self.cursor.sequence) = (0, 0); + let image = match std::fs::read(&self.path) { + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Vec::new(), + image => image?, + }; + self.fresh = image.is_empty(); + if !self.fresh { + let commit = file_pages(&image); + let mut entry = Vec::new(); + push(&mut entry, &mut self.cursor, Body::NewLane); + push(&mut entry, &mut self.cursor, Body::Pages(&commit)); self.last = 1; - self.entries.insert(1, snapshot); + self.entries.insert(1, entry); } Ok(()) } - /// The database's files were deleted: the chain continues with an empty - /// database, and the next commit carries every page. + /// The database's files were deleted: the chain continues without the + /// lane, and the next commit creates it. pub fn reset(&mut self) { - (self.start, self.pages, self.commit) = (0, HashMap::new(), 0); + self.start = 0; self.frames.clear(); self.fresh = true; if self.last == 0 { return; } - let page_size = if self.page_size == 0 { - 4096 - } else { - self.page_size - }; + let mut entry = Vec::new(); + push(&mut entry, &mut self.cursor, Body::DeleteLane); self.last += 1; - match encode(self.last, self.last, page_size, 0, &[]) { - Ok(entry) => drop(self.entries.insert(self.last, entry)), - Err(e) => self.lost = Some(format!("txid {}: {e}", self.last)), - } - } - - /// The database file overlaid with this WAL generation's committed - /// pages, read back from the WAL, as one entry covering `first` through - /// `last`. `None` for an empty database. - fn snapshot(&mut self, first: Txid, last: Txid) -> Result>> { - let mut image = match std::fs::read(&self.path) { - Ok(bytes) => bytes, - Err(e) if e.kind() == std::io::ErrorKind::NotFound => Vec::new(), - Err(e) => return Err(crate::ltx::Error::from(e).into()), - }; - let page_size = match image.get(16..18) { - Some(&[0, 1]) => 65536, - Some(size) => usize::from(u16::from_be_bytes([size[0], size[1]])), - None => self.page_size, - }; - if page_size == 0 { - return Ok(None); - } - self.page_size = page_size; - let mut wal = self.path.as_os_str().to_owned(); - wal.push("-wal"); - let wal = match File::open(wal) { - // Closing the last connection checkpoints the WAL into the - // database file and deletes it: the file holds every page, until - // a reopened connection writes a new WAL header. - Err(e) if e.kind() == std::io::ErrorKind::NotFound => None, - Ok(wal) if wal.metadata().is_ok_and(|m| m.len() == 0) => None, - wal => Some(wal.map_err(crate::ltx::Error::from)?), - }; - if let Some(mut wal) = wal.filter(|_| !self.pages.is_empty()) { - for (&pgno, &offset) in &self.pages { - let at = (pgno as usize - 1) * page_size; - if image.len() < at + page_size { - image.resize(at + page_size, 0); - } - wal.seek(SeekFrom::Start(offset)) - .and_then(|_| wal.read_exact(&mut image[at..at + page_size])) - .map_err(crate::ltx::Error::from)?; - } - image.truncate(self.commit as usize * page_size); - } - let lock = lock_pgno(page_size as u32); - let pages: Vec<(u32, Vec)> = (1..) - .zip(image.chunks_exact(page_size)) - .filter(|(pgno, _)| *pgno != lock) - .map(|(pgno, page)| (pgno, page.to_vec())) - .collect(); - let commit = (image.len() / page_size) as u32; - (commit > 0) - .then(|| encode(first, last, page_size, commit, &pages)) - .transpose() + self.entries.insert(self.last, entry); } /// A write `SQLite` made to the WAL. pub fn write(&mut self, offset: u64, data: &[u8]) { if offset == 0 { - // A new generation: a checkpoint copied the last one into the - // database file. + // A new WAL: a checkpoint copied the last one into the file. if let Some(size) = data.get(8..12) { self.page_size = u32::from_be_bytes([size[0], size[1], size[2], size[3]]) as usize; - (self.start, self.pages) = (WAL_HEADER, HashMap::new()); + self.start = WAL_HEADER; self.frames.clear(); } return; @@ -198,48 +135,43 @@ impl Capture { } self.frames[at..end].copy_from_slice(data); let size = FRAME_HEADER + self.page_size; - if end % size != 0 { + if !end.is_multiple_of(size) { return; } let header = &self.frames[end - size..end - size + FRAME_HEADER]; let commit = u32::from_be_bytes([header[4], header[5], header[6], header[7]]); if commit != 0 { - self.commit_frames(end, commit); + self.commit(end, commit); } } /// The frames before `end` are one committed transaction. - fn commit_frames(&mut self, end: usize, commit: u32) { + fn commit(&mut self, end: usize, commit: u32) { + let mut entry = Vec::new(); + let image = if self.fresh { + self.fresh = false; + std::fs::read(&self.path).unwrap_or_else(|e| { + self.lost = Some(format!("the new database: {e}")); + Vec::new() + }) + } else { + Vec::new() + }; + let (_, mut pages) = file_pages(&image); + if !image.is_empty() { + push(&mut entry, &mut self.cursor, Body::NewLane); + } let size = FRAME_HEADER + self.page_size; - let mut pages = BTreeMap::new(); - for (i, frame) in self.frames[..end].chunks_exact(size).enumerate() { + for frame in self.frames[..end].chunks_exact(size) { let pgno = u32::from_be_bytes([frame[0], frame[1], frame[2], frame[3]]); - self.pages - .insert(pgno, self.start + (i * size + FRAME_HEADER) as u64); - if pgno <= commit { - pages.insert(pgno, frame[FRAME_HEADER..].to_vec()); - } + pages.insert(pgno, &frame[FRAME_HEADER..]); } - let pages: Vec<(u32, Vec)> = pages.into_iter().collect(); + pages.retain(|&pgno, _| pgno <= commit); + push(&mut entry, &mut self.cursor, Body::Pages(&(commit, pages))); self.start += end as u64; self.frames.drain(..end); - self.commit = commit; - // A chain with no snapshot yet starts with one: a new database's - // first page reaches the file, not the WAL, when WAL mode is set. - let entry = if self.last == 0 || self.fresh { - self.fresh = false; - let first = if self.last == 0 { 1 } else { self.last + 1 }; - self.snapshot(first, self.last + 1).and_then(|s| { - s.ok_or_else(|| Error::Replica("a commit to an empty database".into())) - }) - } else { - encode(self.last + 1, self.last + 1, self.page_size, commit, &pages) - }; self.last += 1; - match entry { - Ok(entry) => drop(self.entries.insert(self.last, entry)), - Err(e) => self.lost = Some(format!("txid {}: {e}", self.last)), - } + self.entries.insert(self.last, entry); } /// The last committed txid, or why the capture cannot be trusted. @@ -250,6 +182,10 @@ impl Capture { } } + pub fn cursor(&self) -> Cursor { + self.cursor + } + pub fn entry(&self, txid: Txid) -> Result> { self.entries .get(&txid) @@ -257,76 +193,19 @@ impl Capture { .ok_or_else(|| Error::Replica(format!("no captured entry {txid}"))) } - /// The bucket, or this node's journal, holds everything through `txid`. + /// The bucket holds everything through `txid`. pub fn forget(&mut self, txid: Txid) { self.entries.retain(|t, _| *t > txid); } } -/// Encodes the database at `path`, which no connection has open, as the -/// snapshot at txid 1 into `out`, a page at a time: its size, or `None` for -/// an empty database. Closing the last connection checkpointed the WAL into -/// the file. -pub fn snapshot_file(path: &Path, out: &Path) -> Result> { - let io = |e: std::io::Error| Error::from(crate::ltx::Error::from(e)); - let file = match File::open(path) { - Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(None), - file => file.map_err(io)?, - }; - let len = file.metadata().map_err(io)?.len(); - let mut reader = std::io::BufReader::new(file); - let mut first = [0; 100]; - if len < first.len() as u64 { - return Ok(None); - } - reader.read_exact(&mut first).map_err(io)?; - let page_size = match u16::from_be_bytes([first[16], first[17]]) { - 1 => 65536, - size => usize::from(size), - }; - let header = Header { - version: VERSION, - flags: HEADER_FLAG_NO_CHECKSUM, - page_size: page_size as u32, - commit: (len / page_size as u64) as u32, - min_txid: TXID(1), - max_txid: TXID(1), - ..Header::default() - }; - let mut writer = std::io::BufWriter::new(File::create(out).map_err(io)?); - let mut encoder = Encoder::new(&mut writer); - encoder.encode_header(header)?; - let mut page = vec![0; page_size]; - page[..first.len()].copy_from_slice(&first); - reader.read_exact(&mut page[first.len()..]).map_err(io)?; - for pgno in 1..=header.commit { - if pgno > 1 { - reader.read_exact(&mut page).map_err(io)?; - } - if pgno != lock_pgno(header.page_size) { - encoder.encode_page(PageHeader { pgno, flags: 0 }, &page)?; - } - } - encoder.close(0)?; - writer.flush().map_err(io)?; - Ok(Some(std::fs::metadata(out).map_err(io)?.len())) -} - -fn encode( - first: Txid, - last: Txid, - page_size: usize, - commit: u32, - pages: &[(u32, Vec)], -) -> Result> { - let header = Header { - version: VERSION, - flags: HEADER_FLAG_NO_CHECKSUM, - page_size: page_size as u32, - commit, - min_txid: TXID(first), - max_txid: TXID(last), - ..Header::default() +/// A database file's pages and size, in pages. +fn file_pages(image: &[u8]) -> crate::log::Pages<'_> { + let page_size = match image.get(16..18) { + Some(&[0, 1]) => 65536, + Some(size) => usize::from(u16::from_be_bytes([size[0], size[1]])), + None => return (0, BTreeMap::new()), }; - Ok(encode_file(&header, pages, 0)?) + let pages: BTreeMap<_, _> = (1..).zip(image.chunks_exact(page_size)).collect(); + (pages.len() as u32, pages) } diff --git a/src/workerd/server/cluster/frames.capnp b/src/workerd/server/cluster/frames.capnp new file mode 100644 index 00000000000..1af91fed60d --- /dev/null +++ b/src/workerd/server/cluster/frames.capnp @@ -0,0 +1,83 @@ +# Copyright (c) 2026 Cloudflare, Inc. +# Licensed under the Apache 2.0 license found in the LICENSE file or at: +# https://opensource.org/licenses/Apache-2.0 + +@0xf699edadaece33b7; +# The format of a Durable Object's log in the bucket. An object's state is +# a set of lanes, each a file; its log is a stream of frames, each a change +# to one lane, written by one session (an ownership epoch) in sequence. +# Compression and encryption are not part of it. + +using Cxx = import "/capnp/c++.capnp"; +$Cxx.namespace("workerd::server::cluster"); + +struct Frame { + # A batch of frames in the bucket starts and ends at a commit, and is + # applied whole. + + session @0 :UInt32; + # The ownership epoch that wrote the frame. + + sequence @1 :UInt32; + # Rises within a session. A frame that merges a run of commits takes the + # run's last sequence. + + lane @2 :UInt32; + # A lane is numbered on first use, with the lowest free number. + + union { + newLane @3 :LaneDescription; + deleteLane @4 :Void; + + pages :group { + # For a `sqlitePages` lane: the newest version of each page a commit, + # or a run of commits, wrote, and the database's size after, in + # pages. + commit @5 :UInt32; + list @6 :List(Page); + } + } +} + +struct Page { + number @0 :UInt32; + data @1 :Data; +} + +struct LaneDescription { + name @0 :Text; + # The lane's stable identifier; numbers differ between objects. + + type @1 :Type; + + enum Type { + sqlitePages @0; + # A SQLite database whose writes are pages, so a run of commits merges + # to one copy of each page. + } +} + +struct SnapshotHeader { + # The whole state of an object at a session and frame: this header, then + # each lane's content in lane order. The log is replayed from `frame` + # on. + + session @0 :UInt32; + frame @1 :UInt32; + + lanes @2 :List(Lane); + # Every lane, by number. + + struct Lane { + name @0 :Text; + + union { + deleted @1 :Void; + # A lane number freed while later ones are in use. + + sqlitePages :group { + databaseSize @2 :UInt64; + } + } + } +} diff --git a/src/workerd/server/cluster/lib.rs b/src/workerd/server/cluster/lib.rs index c0ecec6d31a..80d590c51cc 100644 --- a/src/workerd/server/cluster/lib.rs +++ b/src/workerd/server/cluster/lib.rs @@ -14,7 +14,7 @@ //! owner and an ownership epoch. A claim is a conditional write; taking an //! object over from an owner whose lease expired bumps the epoch, and //! everything the new owner writes is keyed by that epoch, under -//! `do///ltx/e/`, so a late write from the old +//! `do///log/e/`, so a late write from the old //! owner lands where nothing reads it. //! //! The clock is an argument everywhere (`now_ms`), so a test decides time. @@ -30,9 +30,9 @@ mod bucket; mod capture; pub mod cluster; mod http; -mod ltx; +mod log; #[cfg(test)] -mod ltx_tests; +mod log_tests; mod owner; mod ownership; mod registry; @@ -72,9 +72,13 @@ pub enum Error { /// holds its identity and must stop serving. #[error("node record {0} was rewritten by another writer")] Fenced(String), - /// Capturing, uploading or restoring a database failed. - #[error("replication: {0}")] - Replication(#[from] crate::ltx::Error), + #[error("io: {0}")] + Io(#[from] std::io::Error), + /// A log or snapshot does not decode. + #[error("log: {0}")] + Log(#[from] capnp::Error), + #[error("log: {0}")] + Schema(#[from] capnp::NotInSchema), } pub type Result = std::result::Result; diff --git a/src/workerd/server/cluster/log.rs b/src/workerd/server/cluster/log.rs new file mode 100644 index 00000000000..cc0807a4695 --- /dev/null +++ b/src/workerd/server/cluster/log.rs @@ -0,0 +1,458 @@ +// Copyright (c) 2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +//! An object's log in the bucket (`frames.capnp`): one lane, the `SQLite` +//! database, a `sqlitePages` lane whose frames are the pages each commit +//! wrote. +//! +//! An epoch's log is a chain of files under its prefix, each one LZ4 +//! frame: `0000/-.log`, the frames of commits `first..=last`, +//! and `0009/0000000000000001-.log`, a snapshot through `last`. A +//! restore is the newest snapshot and the files that extend it; a frame a +//! file before already applied is skipped by its sequence. + +use std::collections::BTreeMap; +use std::io::BufReader; +use std::io::Read; +use std::io::Seek; +use std::io::SeekFrom; +use std::io::Write; +use std::path::Path; +use std::path::PathBuf; + +use frames_capnp::frame; +use frames_capnp::snapshot_header; +use futures::StreamExt; +use futures::TryStreamExt; +use futures::stream; + +use crate::Bucket; +use crate::Error; +use crate::Result; +use crate::Txid; + +/// The level of snapshots, as litestream's. +pub const SNAPSHOT_LEVEL: i32 = 9; +/// The database's lane. +const LANE: &str = "db"; +/// Downloads in flight during a restore, to hide the store's round trips. +const RESTORE_CONCURRENCY: usize = 16; + +/// Where an epoch's next frame goes. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct Cursor { + pub session: u32, + pub sequence: u32, +} + +/// A commit's pages, by number, and the database's size after, in pages. +pub type Pages<'a> = (u32, BTreeMap); + +#[derive(Clone, Copy)] +pub enum Body<'a> { + NewLane, + DeleteLane, + Pages(&'a Pages<'a>), +} + +/// Appends a frame to `out` at `cursor`. +pub fn push(out: &mut Vec, cursor: &mut Cursor, body: Body) { + frame_at(out, cursor.session, cursor.sequence, body); + cursor.sequence += 1; +} + +fn frame_at(out: &mut Vec, session: u32, sequence: u32, body: Body) { + let mut message = capnp::message::Builder::new_default(); + let mut f = message.init_root::(); + f.set_session(session); + f.set_sequence(sequence); + match body { + Body::NewLane => { + let mut lane = f.init_new_lane(); + lane.set_name(LANE); + lane.set_type(frames_capnp::lane_description::Type::SqlitePages); + } + Body::DeleteLane => f.set_delete_lane(()), + Body::Pages((commit, pages)) => { + let mut group = f.init_pages(); + group.set_commit(*commit); + let mut list = group.init_list(pages.len() as u32); + for (i, (&number, data)) in pages.iter().enumerate() { + let mut page = list.reborrow().get(i as u32); + page.set_number(number); + page.set_data(data); + } + } + } + let _ = capnp::serialize::write_message(&mut *out, &message); +} + +/// Merges the frames of `entries`, consecutive commits, so each run of +/// page frames becomes one with the newest version of each page. +pub fn merge(entries: &[Vec]) -> Result> { + let mut messages = Vec::new(); + for entry in entries { + let mut r = entry.as_slice(); + while let Some(message) = capnp::serialize::try_read_message(&mut r, options())? { + messages.push(message); + } + } + let mut out = Vec::new(); + let mut run: Option<(u32, u32, Pages)> = None; + let flush = |out: &mut Vec, run: &mut Option<(u32, u32, Pages)>| { + if let Some((session, sequence, (commit, mut pages))) = run.take() { + pages.retain(|&number, _| number <= commit); + frame_at(out, session, sequence, Body::Pages(&(commit, pages))); + } + }; + for message in &messages { + let f = message.get_root::()?; + let (session, sequence) = (f.get_session(), f.get_sequence()); + if let frame::Pages(group) = f.which()? { + let r = run.get_or_insert_with(|| (session, sequence, (0, BTreeMap::new()))); + (r.1, r.2.0) = (sequence, group.get_commit()); + for page in group.get_list()? { + r.2.1.insert(page.get_number(), page.get_data()?); + } + } else { + flush(&mut out, &mut run); + let mut copy = capnp::message::Builder::new_default(); + copy.set_root(f)?; + capnp::serialize::write_message(&mut out, ©)?; + } + } + flush(&mut out, &mut run); + Ok(out) +} + +/// LZ4-frames `bytes`, as every file in the bucket is. +pub fn compress(bytes: &[u8]) -> Result> { + let mut encoder = encoder(Vec::new()); + encoder.write_all(bytes)?; + encoder.finish().map_err(|e| Error::Replica(e.to_string())) +} + +/// An LZ4 frame with a checksum of its content, which the decoder checks: +/// a corrupt copy fails to replay rather than replaying wrong. +fn encoder(w: W) -> lz4_flex::frame::FrameEncoder { + let info = lz4_flex::frame::FrameInfo::new().content_checksum(true); + lz4_flex::frame::FrameEncoder::with_frame_info(info, w) +} + +fn options() -> capnp::message::ReaderOptions { + *capnp::message::ReaderOptions::new().traversal_limit_in_words(None) +} + +/// The database lane, replayed into a file. +pub struct Replay { + db: std::fs::File, + live: bool, + /// The next frame, by session and sequence. + next: Option<(u32, u32)>, +} + +impl Replay { + pub fn new(db: &Path) -> Result { + Ok(Self { + db: std::fs::File::options() + .read(true) + .write(true) + .create(true) + .truncate(true) + .open(db)?, + live: false, + next: None, + }) + } + + /// Applies the file `path` of the chain at `level`. + pub fn apply(&mut self, level: i32, path: &Path) -> Result<()> { + let file = std::fs::File::open(path)?; + let mut r = BufReader::new(lz4_flex::frame::FrameDecoder::new(file)); + if level == SNAPSHOT_LEVEL { + return self.snapshot(&mut r); + } + while let Some(message) = capnp::serialize::try_read_message(&mut r, options())? { + let f = message.get_root::()?; + let at = (f.get_session(), f.get_sequence()); + if self.next.is_some_and(|next| at < next) { + continue; + } + self.next = Some((at.0, at.1 + 1)); + self.frame(f)?; + } + Ok(()) + } + + fn snapshot(&mut self, r: &mut impl Read) -> Result<()> { + let message = capnp::serialize::read_message(&mut *r, options())?; + let header = message.get_root::()?; + self.next = Some((header.get_session(), header.get_frame())); + self.live = false; + self.db.set_len(0)?; + self.db.seek(SeekFrom::Start(0))?; + let lanes = header.get_lanes()?; + if lanes.len() > 1 { + return Err(Error::Replica("a snapshot of more than one lane".into())); + } + for lane in lanes { + if let snapshot_header::lane::SqlitePages(lane) = lane.which()? { + std::io::copy(&mut r.take(lane.get_database_size()), &mut self.db)?; + self.live = true; + } + } + Ok(()) + } + + fn frame(&mut self, f: frame::Reader) -> Result<()> { + if f.get_lane() != 0 { + return Err(Error::Replica(format!("lane {}", f.get_lane()))); + } + match f.which()? { + frame::NewLane(_) | frame::DeleteLane(()) => { + self.db.set_len(0)?; + self.live = matches!(f.which()?, frame::NewLane(_)); + } + frame::Pages(group) => { + let mut page_size = 0; + for page in group.get_list()? { + let data = page.get_data()?; + page_size = data.len() as u64; + let at = u64::from(page.get_number() - 1) * page_size; + self.db.seek(SeekFrom::Start(at))?; + self.db.write_all(data)?; + } + if page_size > 0 { + self.db.set_len(u64::from(group.get_commit()) * page_size)?; + } + } + } + Ok(()) + } + + /// The replayed database, whole. + pub fn finish(self) -> Result<()> { + Ok(self.db.sync_data()?) + } + + /// Writes the replayed state as a snapshot to `out`. + pub fn write_snapshot(mut self, out: &Path) -> Result { + let (session, frame) = self.next.unwrap_or_default(); + let size = self.db.seek(SeekFrom::End(0))?; + self.db.seek(SeekFrom::Start(0))?; + let lane = self.live.then_some(size); + write_snapshot(out, session, frame, lane, |w| { + Ok(std::io::copy(&mut self.db, w).map(drop)?) + }) + } +} + +/// Writes a snapshot of the database lane, if it exists, of `size` bytes +/// that `payload` writes. +fn write_snapshot( + out: &Path, + session: u32, + frame: u32, + size: Option, + payload: impl FnOnce(&mut dyn Write) -> Result<()>, +) -> Result { + let mut message = capnp::message::Builder::new_default(); + let mut header = message.init_root::(); + header.set_session(session); + header.set_frame(frame); + let mut lanes = header.init_lanes(u32::from(size.is_some())); + if let Some(size) = size { + let mut lane = lanes.reborrow().get(0); + lane.set_name(LANE); + lane.init_sqlite_pages().set_database_size(size); + } + let tmp = with_suffix(out, ".tmp"); + let file = std::io::BufWriter::new(std::fs::File::create(&tmp)?); + let mut encoder = encoder(file); + capnp::serialize::write_message(&mut encoder, &message)?; + payload(&mut encoder)?; + let mut file = encoder + .finish() + .map_err(|e| Error::Replica(e.to_string()))?; + file.flush()?; + drop(file); + std::fs::rename(&tmp, out)?; + Ok(std::fs::metadata(out)?.len()) +} + +/// Writes the database at `path`, which no connection has open, as a +/// snapshot at the start of `session` into `out`: its size, or `None` for an +/// empty database. Closing the last connection checkpointed the WAL into +/// the file. +pub fn snapshot_file(path: &Path, out: &Path, session: u32) -> Result> { + let mut db = match std::fs::File::open(path) { + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(None), + db => db?, + }; + let size = db.metadata()?.len(); + if size < 100 { + return Ok(None); + } + let size = write_snapshot(out, session, 0, Some(size), |w| { + Ok(std::io::copy(&mut db, w).map(drop)?) + })?; + Ok(Some(size)) +} + +/// Replays the chain's files, local copies, into a snapshot at `out`. +pub fn compact(files: &[(File, PathBuf)], out: &Path) -> Result { + let db = with_suffix(out, ".db"); + let mut replay = Replay::new(&db)?; + for (f, path) in files { + replay.apply(f.level, path)?; + } + let size = replay.write_snapshot(out); + let _ = std::fs::remove_file(&db); + size +} + +/// One file of an epoch's chain: commits `first..=last` at level 0, or a +/// snapshot through `last` at `SNAPSHOT_LEVEL`. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct File { + pub level: i32, + pub first: Txid, + pub last: Txid, + pub size: u64, +} + +impl File { + /// This node's copy's name. + pub fn name(&self) -> String { + format!( + "{:04x}-{:016x}-{:016x}.log", + self.level, self.first, self.last + ) + } + + fn key(&self) -> String { + format!( + "{:04x}/{:016x}-{:016x}.log", + self.level, self.first, self.last + ) + } +} + +/// The files a restore reads, in order: from txid 1 on, each file that +/// extends what came before, through the first gap. A file inside the range +/// already reached is skipped: tiering and bucket-acknowledged commits +/// upload concurrently, so a narrower range can land beside a wider one. +pub fn chain(files: &[File]) -> Vec { + let mut files = files.to_vec(); + files.sort_by_key(|f| (f.first, std::cmp::Reverse(f.last))); + let mut reached = 0; + let mut chain = Vec::new(); + for f in files { + if f.first > reached + 1 { + break; + } + if f.last > reached { + reached = f.last; + chain.push(f); + } + } + chain +} + +/// An epoch's log in the bucket. +#[derive(Clone)] +pub struct Log { + bucket: Bucket, +} + +impl Log { + pub fn new(bucket: Bucket) -> Self { + Self { bucket } + } + + pub async fn files(&self) -> Result> { + let mut files = Vec::new(); + for level in [0, SNAPSHOT_LEVEL] { + for entry in self.bucket.list(&format!("{level:04x}/")).await? { + let name = entry.key.rsplit('/').next().unwrap_or_default(); + let range = name.strip_suffix(".log").and_then(|r| r.split_once('-')); + let Some((first, last)) = range else { + continue; + }; + let (Ok(first), Ok(last)) = ( + u64::from_str_radix(first, 16), + u64::from_str_radix(last, 16), + ) else { + continue; + }; + files.push(File { + level, + first, + last, + size: 0, + }); + } + } + Ok(files) + } + + pub async fn get(&self, f: &File, to: &Path) -> Result<()> { + if self.bucket.get_file(&f.key(), to).await? { + Ok(()) + } else { + Err(Error::Replica(format!("{} is missing", f.key()))) + } + } + + pub async fn put(&self, f: &File, from: &Path) -> Result<()> { + self.bucket.put_file(&f.key(), from, &[]).await + } + + pub async fn delete(&self, f: &File) -> Result<()> { + self.bucket.delete(&f.key()).await + } +} + +/// Restores the log into `output`, which must not exist: the chain's +/// files spooled to `.restore/`, then replayed on the blocking +/// pool. The txid restored through, 0 for an empty log, which leaves no +/// file. +pub async fn restore(log: &Log, output: &Path) -> Result { + let chain = chain(&log.files().await?); + let Some(end) = chain.last().map(|f| f.last) else { + return Ok(0); + }; + let spool = with_suffix(output, ".restore"); + std::fs::create_dir_all(&spool)?; + let files: Vec<_> = (0..chain.len()) + .map(|i| (chain[i], spool.join(i.to_string()))) + .collect(); + let downloaded = stream::iter(&files) + .map(|(f, path)| log.get(f, path)) + .buffer_unordered(RESTORE_CONCURRENCY) + .try_collect::<()>() + .await; + let output = output.to_owned(); + let result = match downloaded { + Ok(()) => { + crate::owner::blocking(move || { + let mut replay = Replay::new(&output)?; + for (f, path) in &files { + replay.apply(f.level, path)?; + } + replay.finish() + }) + .await + } + Err(e) => Err(e), + }; + let _ = std::fs::remove_dir_all(&spool); + result.map(|()| end) +} + +pub fn with_suffix(path: &Path, suffix: &str) -> PathBuf { + let mut path = path.as_os_str().to_owned(); + path.push(suffix); + path.into() +} diff --git a/src/workerd/server/cluster/ltx_tests.rs b/src/workerd/server/cluster/log_tests.rs similarity index 85% rename from src/workerd/server/cluster/ltx_tests.rs rename to src/workerd/server/cluster/log_tests.rs index 2b6875ac7df..e64c3a1bfa8 100644 --- a/src/workerd/server/cluster/ltx_tests.rs +++ b/src/workerd/server/cluster/log_tests.rs @@ -19,10 +19,10 @@ use std::sync::OnceLock; use crate::Bucket; use crate::Error; use crate::capture::Capture; -use crate::ltx::BucketClient; -use crate::ltx::TXID; -use crate::ltx::client::ReplicaClient; -use crate::ltx::restore; +use crate::log::File as LogFile; +use crate::log::Log; +use crate::log::SNAPSHOT_LEVEL; +use crate::log::restore; use crate::owner::Owners; use crate::ownership::Claim; use crate::ownership::Ownership; @@ -287,22 +287,41 @@ fn upload(store: &Bucket, first: u64, last: u64) { upload_at(store, 0, first, last); } -fn upload_at(store: &Bucket, level: i32, first: u64, last: u64) { +/// The captured entries `first..=last` as one log file in `dir`. +fn entries(dir: &Path, first: u64, last: u64) -> (LogFile, PathBuf) { let files: Vec> = CAPTURE.with_borrow(|c| { let c = c.as_ref().unwrap(); (first..=last).map(|t| c.entry(t).unwrap()).collect() }); - let from = scratch(&format!("upload-{level}-{first}-{last}")).join("ltx"); - std::fs::write(&from, crate::ltx::ltx::merge_files(&files).unwrap()).unwrap(); - let client = BucketClient::new(store.at("cell")); - runtime() - .block_on(client.write_ltx_file(level, TXID(first), TXID(last), &from)) - .unwrap(); + let from = dir.join(format!("{first}-{last}")); + let merged = crate::log::merge(&files).unwrap(); + std::fs::write(&from, crate::log::compress(&merged).unwrap()).unwrap(); + let file = LogFile { + level: 0, + first, + last, + size: 0, + }; + (file, from) +} + +fn upload_at(store: &Bucket, level: i32, first: u64, last: u64) { + let dir = scratch(&format!("upload-{level}-{first}-{last}")); + let (file, mut from) = entries(&dir, first, last); + if level == SNAPSHOT_LEVEL { + // A compaction of the entries. + let out = dir.join("snapshot"); + crate::log::compact(&[(file, from)], &out).unwrap(); + from = out; + } + let file = LogFile { level, ..file }; + let log = Log::new(store.at("cell")); + runtime().block_on(log.put(&file, &from)).unwrap(); } fn restored_rows(store: &Bucket, out: &Path) -> (i64, i64, i64) { - let client = BucketClient::new(store.at("cell")); - runtime().block_on(restore(&client, out, TXID(0))).unwrap(); + let log = Log::new(store.at("cell")); + runtime().block_on(restore(&log, out)).unwrap(); rows(&Connection::open(out.to_str().unwrap())) } @@ -330,28 +349,58 @@ fn restore_reads_the_newest_snapshot_and_what_extends_it() { insert(&app, 0, 100); let a = last(); upload(&store, 1, a); - upload_at(&store, crate::ltx::SNAPSHOT_LEVEL, 1, a); + upload_at(&store, SNAPSHOT_LEVEL, 1, a); insert(&app, 100, 200); let b = last(); upload(&store, a + 1, b); - upload_at(&store, crate::ltx::SNAPSHOT_LEVEL, 1, b); + upload_at(&store, SNAPSHOT_LEVEL, 1, b); app.execute("DELETE FROM t WHERE id % 3 = 0"); upload(&store, b + 1, last()); - let client = BucketClient::new(store.at("cell")); - let plan = runtime() - .block_on(crate::ltx::plan(&client, TXID(0))) - .unwrap(); - let plan: Vec<_> = plan + let log = Log::new(store.at("cell")); + let files = runtime().block_on(log.files()).unwrap(); + let plan: Vec<_> = crate::log::chain(&files) .iter() - .map(|f| (f.level, f.min_txid.0, f.max_txid.0)) + .map(|f| (f.level, f.first, f.last)) .collect(); - assert_eq!( - plan, - [(crate::ltx::SNAPSHOT_LEVEL, 1, b), (0, b + 1, last())] - ); + assert_eq!(plan, [(SNAPSHOT_LEVEL, 1, b), (0, b + 1, last())]); + assert_eq!(restored_rows(&store, &dir.join("restored.db")), rows(&app)); +} + +#[test] +fn a_snapshot_inside_a_wal_carries_it_across_checkpoints() { + let dir = scratch("snapshot-wal"); + let app = open_app(&dir.join("app.db")); + let store = Bucket::memory(""); + app.execute("PRAGMA wal_autocheckpoint=8"); + for from in (0..400).step_by(10) { + insert(&app, from, from + 10); + } + let a = last(); + upload_at(&store, SNAPSHOT_LEVEL, 1, a); + for from in (400..800).step_by(10) { + insert(&app, from, from + 10); + } + upload(&store, a + 1, last()); assert_eq!(restored_rows(&store, &dir.join("restored.db")), rows(&app)); } +#[test] +fn a_corrupt_file_fails_to_restore() { + let dir = scratch("corrupt"); + let app = open_app(&dir.join("app.db")); + let store = Bucket::memory(""); + insert(&app, 0, 100); + let (file, path) = entries(&dir, 1, last()); + let mut bytes = std::fs::read(&path).unwrap(); + let mid = bytes.len() / 2; + bytes[mid] ^= 1; + std::fs::write(&path, bytes).unwrap(); + let log = Log::new(store.at("cell")); + runtime().block_on(log.put(&file, &path)).unwrap(); + let restored = runtime().block_on(restore(&log, &dir.join("restored.db"))); + assert!(restored.is_err(), "{restored:?}"); +} + #[test] fn a_rolled_back_transaction_leaves_nothing() { let dir = scratch("rollback"); @@ -389,8 +438,9 @@ fn checkpoints_and_a_restart_keep_the_chain_whole() { upload(&store, 1, last()); assert_eq!(restored_rows(&store, &dir.join("one.db")), rows(&app)); - // A new chain from a snapshot of the live database (reclaim), after - // commits still in the WAL, then more commits, shrinking at the end. + // A new chain from the live database, then more commits, shrinking at + // the end. + app.execute("PRAGMA wal_checkpoint(TRUNCATE)"); CAPTURE.with_borrow_mut(|c| c.as_mut().unwrap().restart().unwrap()); insert(&app, 650, 700); app.execute("DELETE FROM t WHERE id > 20; VACUUM;"); @@ -457,9 +507,9 @@ fn a_deleted_database_restores_empty_then_as_recreated() { let deleted = last(); let store = Bucket::memory(""); upload(&store, 1, deleted); - let client = BucketClient::new(store.at("cell")); + let log = Log::new(store.at("cell")); let out = dir.join("empty.db"); - runtime().block_on(restore(&client, &out, TXID(0))).unwrap(); + runtime().block_on(restore(&log, &out)).unwrap(); assert_eq!(std::fs::metadata(&out).unwrap().len(), 0); let app = Connection::open(&format!("file:{}?vfs=capture-test", path.display())); diff --git a/src/workerd/server/cluster/ltx/client/bucket.rs b/src/workerd/server/cluster/ltx/client/bucket.rs deleted file mode 100644 index 60fbcabede0..00000000000 --- a/src/workerd/server/cluster/ltx/client/bucket.rs +++ /dev/null @@ -1,136 +0,0 @@ -//! client::bucket — the `ReplicaClient` over the cluster's `Bucket`. -//! -//! Ported (behavior only) from litestream@v0.5.11 `s3/replica_client.go` by -//! way of celld's `celld-ltx`, keeping the invariants from the upstream -//! conformance suite: -//! -//! * key scheme `{path}/{level:04x}/{min}-{max}.ltx` -//! (s3/replica_client.go:629, 677, 1040-1042); -//! * list + seek-skip on `min_txid < seek`, ascending TXID order -//! (s3/replica_client.go:1530-1533); -//! * `NoSuchKey` → `os.ErrNotExist` error mapping -//! (s3/replica_client.go:647-649, 1662-1668). -//! -//! Every file is one PUT. S3 takes up to 5 GiB that way, and this port has no -//! compactor to produce oversized files, so the multipart path stays in celld. - -use std::io::Read; -use std::path::Path; - -use crate::Bucket; -use crate::ltx::TXID; -use crate::ltx::client::ReplicaClient; -use crate::ltx::error::Error; -use crate::ltx::error::Result; -use crate::ltx::ltx::FileInfo; -use crate::ltx::ltx::{self}; -use crate::s3::format_rfc3339_nano; - -/// The standard Litestream S3 metadata key for an LTX header timestamp. -const METADATA_KEY_TIMESTAMP: &str = "litestream-timestamp"; - -/// A replica rooted at the bucket's prefix. -#[derive(Clone)] -pub struct BucketClient { - bucket: Bucket, -} - -impl BucketClient { - pub fn new(bucket: Bucket) -> Self { - Self { bucket } - } - - /// Delete a file a compaction replaced. - pub async fn delete_ltx_file(&self, level: i32, min_txid: TXID, max_txid: TXID) -> Result<()> { - self.bucket - .delete(&Self::ltx_key(level, min_txid, max_txid)) - .await - .map_err(other) - } - - /// `{level:04x}/{min}-{max}.ltx` (s3/replica_client.go:629, 677, 1040-1042). - fn ltx_key(level: i32, min_txid: TXID, max_txid: TXID) -> String { - format!("{level:04x}/{}", ltx::format_filename(min_txid, max_txid)) - } -} - -fn other(error: crate::Error) -> Error { - Error::Other(error.to_string().into()) -} - -fn not_found(key: &str) -> Error { - Error::Io(std::io::Error::new( - std::io::ErrorKind::NotFound, - format!("replica: get object {key}: not found"), - )) -} - -#[async_trait::async_trait(?Send)] -impl ReplicaClient for BucketClient { - async fn ltx_files(&self, level: i32, seek: TXID) -> Result> { - let prefix = format!("{level:04x}/"); - let listed = self.bucket.list(&prefix).await.map_err(other)?; - let mut infos: Vec = listed - .into_iter() - .filter_map(|entry| { - let name = entry.key.rsplit('/').next()?; - let (min_txid, max_txid) = ltx::parse_filename(name).ok()?; - (min_txid >= seek).then(|| FileInfo { - level, - min_txid, - max_txid, - ..Default::default() - }) - }) - .collect(); - infos.sort_by_key(|info| (info.level, info.min_txid.0, info.max_txid.0)); - Ok(infos) - } - - async fn open_ltx_file( - &self, - level: i32, - min_txid: TXID, - max_txid: TXID, - to: &Path, - ) -> Result<()> { - let key = Self::ltx_key(level, min_txid, max_txid); - match self.bucket.get_file(&key, to).await.map_err(other)? { - true => Ok(()), - false => Err(not_found(&key)), - } - } - - async fn write_ltx_file( - &self, - level: i32, - min_txid: TXID, - max_txid: TXID, - from: &Path, - ) -> Result { - // The LTX header timestamp goes in the standard Litestream metadata, - // so Litestream can restore by time without reading every file. - let mut file = std::fs::File::open(from)?; - let mut bytes = [0; ltx::HEADER_SIZE]; - file.read_exact(&mut bytes)?; - let header = ltx::Header::parse(&bytes)?; - let size = file.metadata()?.len(); - let key = Self::ltx_key(level, min_txid, max_txid); - let stamp = format_rfc3339_nano(header.timestamp); - self.bucket - .put_file(&key, from, &[(METADATA_KEY_TIMESTAMP, stamp)]) - .await - .map_err(|e| Error::Other(format!("replica: upload to {key}: {e}").into()))?; - Ok(FileInfo { - level, - min_txid, - max_txid, - size: size as i64, - created_at: Some( - std::time::UNIX_EPOCH - + std::time::Duration::from_millis(header.timestamp.max(0) as u64), - ), - ..Default::default() - }) - } -} diff --git a/src/workerd/server/cluster/ltx/client/mod.rs b/src/workerd/server/cluster/ltx/client/mod.rs deleted file mode 100644 index f4eba5c368c..00000000000 --- a/src/workerd/server/cluster/ltx/client/mod.rs +++ /dev/null @@ -1,48 +0,0 @@ -//! client — the `ReplicaClient` trait. -//! -//! Ported from litestream@v0.5.11 `replica_client.go`. The trait is the storage -//! abstraction that every backend implements. -//! -//! # I/O -//! Go's `ReplicaClient` uses `io.Reader`/`io.ReadCloser`. LTX files go -//! between the store and files on disk, so no file is held in memory. - -use std::path::Path; - -use async_trait::async_trait; - -use crate::ltx::TXID; -use crate::ltx::error::Result; -use crate::ltx::ltx::FileInfo; - -pub mod bucket; - -/// Client for reading and writing LTX files on a replica backend. -/// -/// Ported from the `ReplicaClient` interface (replica_client.go:19-51). Methods -/// take a compaction `level`: 0 for uploads, `SNAPSHOT_LEVEL` for snapshots. -#[async_trait(?Send)] -pub trait ReplicaClient { - /// Returns all LTX files for `level`, sorted ascending by `min_txid`, that - /// start at or after `seek`. - async fn ltx_files(&self, level: i32, seek: TXID) -> Result>; - - /// Writes an LTX file to the file `to`. Returns an - /// `io::ErrorKind::NotFound` error (wrapped) if it does not exist. - async fn open_ltx_file( - &self, - level: i32, - min_txid: TXID, - max_txid: TXID, - to: &Path, - ) -> Result<()>; - - /// Writes the LTX file `from` to the replica and returns its metadata. - async fn write_ltx_file( - &self, - level: i32, - min_txid: TXID, - max_txid: TXID, - from: &Path, - ) -> Result; -} diff --git a/src/workerd/server/cluster/ltx/codec.rs b/src/workerd/server/cluster/ltx/codec.rs deleted file mode 100644 index 1119ca9bd4a..00000000000 --- a/src/workerd/server/cluster/ltx/codec.rs +++ /dev/null @@ -1,354 +0,0 @@ -//! Streaming LTX encoder and decoder. - -use std::collections::BTreeMap; -use std::io::Read; -use std::io::Write; - -use crate::ltx::CHECKSUM_FLAG; -use crate::ltx::error::Error; -use crate::ltx::error::Result; -use crate::ltx::ltx::CHECKSUM_SIZE; -use crate::ltx::ltx::Crc64; -use crate::ltx::ltx::HEADER_SIZE; -use crate::ltx::ltx::Header; -use crate::ltx::ltx::PAGE_HEADER_FLAG_SIZE; -use crate::ltx::ltx::PAGE_HEADER_SIZE; -use crate::ltx::ltx::PageHeader; -use crate::ltx::ltx::TRAILER_SIZE; -use crate::ltx::ltx::Trailer; -use crate::ltx::ltx::checksum_page; -use crate::ltx::ltx::lock_pgno; - -#[derive(Debug, Clone, Copy, PartialEq, Eq)] -enum DecoderState { - Header, - Pages, - Close, - Closed, -} - -pub(crate) struct Decoder { - reader: R, - state: DecoderState, - pub(crate) header: Header, - pub(crate) trailer: Trailer, - hash: Crc64, - rolling_checksum: u64, -} - -impl Decoder { - pub(crate) fn new(reader: R) -> Self { - Self { - reader, - state: DecoderState::Header, - header: Header::default(), - trailer: Trailer::default(), - hash: Crc64::new(), - rolling_checksum: 0, - } - } - - pub(crate) fn decode_header(&mut self) -> Result<()> { - if self.state != DecoderState::Header { - return Err(Error::LTXCorrupted); - } - let mut bytes = [0; HEADER_SIZE]; - self.reader.read_exact(&mut bytes)?; - self.header = Header::parse(&bytes)?; - self.header.validate()?; - self.hash.update(&bytes); - if !self.header.no_checksum() { - self.rolling_checksum = CHECKSUM_FLAG; - } - self.state = DecoderState::Pages; - Ok(()) - } - - pub(crate) fn decode_page(&mut self, data: &mut [u8]) -> Result> { - if self.state == DecoderState::Close { - return Ok(None); - } - if self.state != DecoderState::Pages || data.len() != self.header.page_size as usize { - return Err(Error::LTXCorrupted); - } - - let mut header_bytes = [0; PAGE_HEADER_SIZE]; - self.reader.read_exact(&mut header_bytes)?; - let page = PageHeader::parse(&header_bytes)?; - self.hash.update(&header_bytes); - if page.is_zero() { - self.state = DecoderState::Close; - return Ok(None); - } - page.validate()?; - - if page.flags & PAGE_HEADER_FLAG_SIZE != 0 { - let mut size_bytes = [0; 4]; - self.reader.read_exact(&mut size_bytes)?; - self.hash.update(&size_bytes); - let compressed_size = u32::from_be_bytes(size_bytes) as usize; - if compressed_size > lz4_flex::block::get_maximum_output_size(data.len()) { - return Err(Error::LTXCorrupted); - } - let mut compressed = vec![0; compressed_size]; - self.reader.read_exact(&mut compressed)?; - let n = lz4_flex::block::decompress_into(&compressed, data) - .map_err(|_| Error::LTXCorrupted)?; - if n != data.len() { - return Err(Error::LTXCorrupted); - } - } else { - let mut decoder = lz4_flex::frame::FrameDecoder::new(&mut self.reader); - decoder.read_exact(data).map_err(|_| Error::LTXCorrupted)?; - let mut extra = [0; 1]; - if decoder.read(&mut extra).map_err(|_| Error::LTXCorrupted)? != 0 { - return Err(Error::LTXCorrupted); - } - } - - self.hash.update(data); - if self.header.is_snapshot() - && !self.header.no_checksum() - && page.pgno != lock_pgno(self.header.page_size) - { - self.rolling_checksum = - CHECKSUM_FLAG | (self.rolling_checksum ^ checksum_page(page.pgno, data)); - } - Ok(Some(page)) - } - - pub(crate) fn close(&mut self) -> Result<()> { - if self.state == DecoderState::Closed { - return Ok(()); - } - if self.state != DecoderState::Close { - return Err(Error::LTXCorrupted); - } - - let mut remaining = Vec::new(); - self.reader.read_to_end(&mut remaining)?; - if remaining.len() < 8 + TRAILER_SIZE { - return Err(Error::LTXCorrupted); - } - - let trailer_offset = remaining.len() - TRAILER_SIZE; - let size_offset = trailer_offset - 8; - let index_size = u64::from_be_bytes( - remaining[size_offset..trailer_offset] - .try_into() - .map_err(|_| Error::LTXCorrupted)?, - ) as usize; - if index_size != size_offset { - return Err(Error::LTXCorrupted); - } - verify_page_index(&remaining[..size_offset])?; - - self.trailer = Trailer::parse(&remaining[trailer_offset..])?; - self.trailer.validate(self.header)?; - self.hash - .update(&remaining[..remaining.len() - CHECKSUM_SIZE]); - if CHECKSUM_FLAG | self.hash.sum64() != self.trailer.file_checksum { - return Err(Error::ChecksumMismatch); - } - if self.header.is_snapshot() - && !self.header.no_checksum() - && self.rolling_checksum != self.trailer.post_apply_checksum - { - return Err(Error::ChecksumMismatch); - } - - self.state = DecoderState::Closed; - Ok(()) - } -} - -pub(crate) struct Encoder { - pub(crate) writer: W, - pub(crate) header: Header, - pub(crate) trailer: Trailer, - hash: Crc64, - index: BTreeMap, - bytes_written: u64, - previous_page_number: u32, - header_written: bool, - closed: bool, -} - -impl Encoder { - pub(crate) fn new(writer: W) -> Self { - Self { - writer, - header: Header::default(), - trailer: Trailer::default(), - hash: Crc64::new(), - index: BTreeMap::new(), - bytes_written: 0, - previous_page_number: 0, - header_written: false, - closed: false, - } - } - - pub(crate) fn encode_header(&mut self, header: Header) -> Result<()> { - if self.header_written || self.closed { - return Err(Error::LTXCorrupted); - } - header.validate()?; - self.header = header; - let bytes = header.marshal(); - self.write_hashed(&bytes)?; - self.header_written = true; - Ok(()) - } - - pub(crate) fn encode_page(&mut self, page: PageHeader, data: &[u8]) -> Result<()> { - if !self.header_written - || self.closed - || page.pgno > self.header.commit - || data.len() != self.header.page_size as usize - { - return Err(Error::LTXCorrupted); - } - page.validate()?; - let lock_page = lock_pgno(self.header.page_size); - if page.pgno == lock_page { - return Err(Error::LTXCorrupted); - } - - if self.header.is_snapshot() { - if self.previous_page_number == 0 && page.pgno != 1 { - return Err(Error::LTXCorrupted); - } - let expected = if self.previous_page_number == lock_page - 1 { - self.previous_page_number + 2 - } else { - self.previous_page_number + 1 - }; - if self.previous_page_number != 0 && page.pgno != expected { - return Err(Error::LTXCorrupted); - } - } else if self.previous_page_number >= page.pgno { - return Err(Error::LTXCorrupted); - } - - let offset = self.bytes_written; - if page.flags != 0 { - return Err(Error::LTXCorrupted); - } - self.write_hashed(&page.marshal())?; - // Each page is an LZ4 frame of independent 64 KiB blocks with a - // content checksum, as the pre-v0.5.2 Go encoder writes; the decoder - // also reads v0.5.2's sized blocks. - let frame_info = lz4_flex::frame::FrameInfo::new() - .block_size(lz4_flex::frame::BlockSize::Max64KB) - .block_mode(lz4_flex::frame::BlockMode::Independent) - .content_checksum(true); - let mut encoder = lz4_flex::frame::FrameEncoder::with_frame_info(frame_info, Vec::new()); - encoder.write_all(data)?; - let compressed = encoder.finish().map_err(|_| Error::LTXCorrupted)?; - self.writer.write_all(&compressed)?; - self.bytes_written += compressed.len() as u64; - self.hash.update(data); - - self.previous_page_number = page.pgno; - self.index - .insert(page.pgno, (offset, self.bytes_written - offset)); - Ok(()) - } - - pub(crate) fn close(&mut self, post_apply_checksum: u64) -> Result<()> { - if !self.header_written || self.closed { - return Err(Error::LTXCorrupted); - } - - self.write_hashed(&[0; PAGE_HEADER_SIZE])?; - let index_offset = self.bytes_written; - let mut index_bytes = Vec::new(); - for (&page_number, &(offset, size)) in &self.index { - write_uvarint(&mut index_bytes, page_number as u64); - write_uvarint(&mut index_bytes, offset); - write_uvarint(&mut index_bytes, size); - } - write_uvarint(&mut index_bytes, 0); - self.write_hashed(&index_bytes)?; - self.write_hashed(&(self.bytes_written - index_offset).to_be_bytes())?; - - self.trailer.post_apply_checksum = post_apply_checksum; - self.hash.update(&post_apply_checksum.to_be_bytes()); - self.trailer.file_checksum = CHECKSUM_FLAG | self.hash.sum64(); - self.trailer.validate(self.header)?; - // An empty database's checksum, unless the file carries none. - if self.header.commit == 0 - && !self.header.no_checksum() - && post_apply_checksum != CHECKSUM_FLAG - { - return Err(Error::LTXCorrupted); - } - self.writer.write_all(&self.trailer.marshal())?; - self.bytes_written += TRAILER_SIZE as u64; - self.closed = true; - Ok(()) - } - - fn write_hashed(&mut self, bytes: &[u8]) -> Result<()> { - self.writer.write_all(bytes)?; - self.hash.update(bytes); - self.bytes_written += bytes.len() as u64; - Ok(()) - } -} - -fn verify_page_index(bytes: &[u8]) -> Result<()> { - decode_page_index(bytes).map(|_| ()) -} - -/// Decodes a page index into its `(pgno, frame_offset, frame_size)` entries. -/// The index is varint triples terminated by a `pgno == 0` marker; each entry -/// locates one page's whole frame (header plus LZ4 body) at `frame_offset` in -/// the file, `frame_size` bytes long. Paged restore builds its page map from -/// these, so it can range-read one frame without the object. -pub(crate) fn decode_page_index(bytes: &[u8]) -> Result> { - let mut position = 0; - let mut entries = Vec::new(); - loop { - let page_number = read_uvarint(bytes, &mut position)?; - if page_number == 0 { - break; - } - let offset = read_uvarint(bytes, &mut position)?; - let size = read_uvarint(bytes, &mut position)?; - entries.push((page_number as u32, offset, size)); - } - if position != bytes.len() { - return Err(Error::LTXCorrupted); - } - Ok(entries) -} - -fn read_uvarint(bytes: &[u8], position: &mut usize) -> Result { - let mut value = 0u64; - let mut shift = 0u32; - loop { - let byte = *bytes.get(*position).ok_or(Error::LTXCorrupted)?; - *position += 1; - if byte < 0x80 { - if shift >= 64 || (shift == 63 && byte > 1) { - return Err(Error::LTXCorrupted); - } - return Ok(value | (u64::from(byte) << shift)); - } - value |= u64::from(byte & 0x7f) << shift; - shift += 7; - if shift >= 70 { - return Err(Error::LTXCorrupted); - } - } -} - -fn write_uvarint(bytes: &mut Vec, mut value: u64) { - while value >= 0x80 { - bytes.push(value as u8 | 0x80); - value >>= 7; - } - bytes.push(value as u8); -} diff --git a/src/workerd/server/cluster/ltx/error.rs b/src/workerd/server/cluster/ltx/error.rs deleted file mode 100644 index 8ba743ed720..00000000000 --- a/src/workerd/server/cluster/ltx/error.rs +++ /dev/null @@ -1,22 +0,0 @@ -//! The LTX error type, after Litestream's sentinel errors (litestream.go:32-37). - -use thiserror::Error; - -pub type Result = std::result::Result; - -#[derive(Debug, Error)] -pub enum Error { - /// The replica's rolling checksum does not match what was expected. - #[error("invalid replica, checksum mismatch")] - ChecksumMismatch, - /// An LTX file's contents are corrupt (bad magic, truncated, bad CRC). - #[error("ltx file corrupted")] - LTXCorrupted, - /// No backup chain reaches the requested transaction. - #[error("transaction not available")] - TxNotAvailable, - #[error("io error: {0}")] - Io(#[from] std::io::Error), - #[error("{0}")] - Other(#[from] Box), -} diff --git a/src/workerd/server/cluster/ltx/ltx.rs b/src/workerd/server/cluster/ltx/ltx.rs deleted file mode 100644 index 94cd854823f..00000000000 --- a/src/workerd/server/cluster/ltx/ltx.rs +++ /dev/null @@ -1,542 +0,0 @@ -//! ltx.rs — LTX (Lite Transaction) file reader/writer + CRC64-ISO checksums. -//! -//! Ported from ltx@v0.5.2 `ltx.go`, `checksum.go`, `encoder.go`, `decoder.go` -//! and litestream@v0.5.11 `v3.go`. `reference/ltx-format.md` describes the -//! authoritative byte layout. -//! -//! The reader decodes a complete LTX file with either v0.5.2 LZ4 blocks or the -//! older LZ4 frames. It verifies the CRC64-ISO file checksum and the rolling -//! snapshot checksum. The v0.5.2 writer emits exact upstream bytes, while the -//! default writer preserves the legacy layout during the staged rollout. - -use std::time::SystemTime; - -use crate::ltx::CHECKSUM_FLAG; -use crate::ltx::Checksum; -use crate::ltx::TXID; -use crate::ltx::error::Error; -use crate::ltx::error::Result; - -// ── Constants (ltx@v0.5.2 ltx.go:18-55) ────────────────────────────────────── - -/// First 4 bytes of every LTX file. -pub const MAGIC: &[u8; 4] = b"LTX1"; -/// Current LTX file format version. -pub const VERSION: i32 = 3; -pub const HEADER_SIZE: usize = 100; -pub const PAGE_HEADER_SIZE: usize = 6; -pub const TRAILER_SIZE: usize = 16; -pub const CHECKSUM_SIZE: usize = 8; - -/// Header flag: checksums are not tracked for this file. -pub const HEADER_FLAG_NO_CHECKSUM: u32 = 1 << 1; -pub const HEADER_FLAG_MASK: u32 = HEADER_FLAG_NO_CHECKSUM; - -/// A four-byte compressed-size field follows the page header, and the page -/// uses raw LZ4 block compression. Files written before ltx v0.5.2 omit this -/// flag and contain one LZ4 frame per page. -pub const PAGE_HEADER_FLAG_SIZE: u16 = 1 << 0; -pub const PAGE_HEADER_FLAG_MASK: u16 = PAGE_HEADER_FLAG_SIZE; - -/// SQLite PENDING_BYTE offset; the lock page derives from it. -pub const PENDING_BYTE: i64 = 0x4000_0000; - -fn corrupt(msg: impl Into) -> Error { - // Wrap a format error as LTXCorrupted, matching litestream's classification - // of malformed LTX content (litestream.go ErrLTXCorrupted). - let _ = msg; - Error::LTXCorrupted -} - -/// Returns the lock page number for a given page size (ltx.go:494). -/// -/// `page_size` is expected to be a validated SQLite page size (a power of two in -/// `[512, 65536]`); for any such value the result is identical to Go's -/// `LockPgno` (`PENDING_BYTE / page_size + 1`). A `page_size` of `0` — which -/// only reaches here via an unvalidated/adversarial header — would make the -/// underlying integer divide panic in both Go and Rust, so we guard it and -/// return `0` (never a real page number) instead of dividing. All in-crate -/// callers validate the header first, mirroring Go's `DecodeHeader`-before- -/// `LockPgno` ordering, so this guard is reached only by a direct external call. -pub fn lock_pgno(page_size: u32) -> u32 { - if page_size == 0 { - return 0; - } - (PENDING_BYTE / page_size as i64) as u32 + 1 -} - -// ── CRC64-ISO (checksum.go:177 `crc64.MakeTable(crc64.ISO)`) ────────────────── - -/// Streaming CRC64-ISO hasher matching Go's `hash/crc64` digest semantics -/// (CRC-64/GO-ISO). celld uses `crc-fast` for SIMD folding; this is Go's own -/// byte-at-a-time table walk (crc64.go `update`), which avoids a dependency. -#[derive(Clone, Default)] -pub struct Crc64 { - crc: u64, -} - -/// Go's `crc64.MakeTable(crc64.ISO)`: the reflected polynomial table. -const CRC64_ISO_TABLE: [u64; 256] = { - let mut table = [0u64; 256]; - let mut i = 0; - while i < 256 { - let mut crc = i as u64; - let mut bit = 0; - while bit < 8 { - crc = if crc & 1 == 1 { - (crc >> 1) ^ 0xD800_0000_0000_0000 - } else { - crc >> 1 - }; - bit += 1; - } - table[i] = crc; - i += 1; - } - table -}; - -impl Crc64 { - pub fn new() -> Self { - Self::default() - } - - pub fn update(&mut self, data: &[u8]) { - let mut crc = !self.crc; - for &b in data { - crc = CRC64_ISO_TABLE[(crc as u8 ^ b) as usize] ^ (crc >> 8); - } - self.crc = !crc; - } - - pub fn sum64(&self) -> u64 { - self.crc - } -} - -/// CRC64 checksum of a single page combined with its page number, with the -/// ChecksumFlag set (checksum.go:106-116). Input is `BE_u32(pgno) ++ data`. -pub fn checksum_page(pgno: u32, data: &[u8]) -> Checksum { - let mut h = Crc64::new(); - h.update(&pgno.to_be_bytes()); - h.update(data); - CHECKSUM_FLAG | h.sum64() -} - -// ── Header / PageHeader / Trailer ───────────────────────────────────────────── - -/// LTX file header (100 bytes). Ported from ltx@v0.5.1 ltx.go:179-326. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] -pub struct Header { - pub version: i32, - pub flags: u32, - pub page_size: u32, - pub commit: u32, - pub min_txid: TXID, - pub max_txid: TXID, - pub timestamp: i64, - pub pre_apply_checksum: Checksum, - pub wal_offset: i64, - pub wal_size: i64, - pub wal_salt1: u32, - pub wal_salt2: u32, - pub node_id: u64, -} - -impl Header { - /// True if this header begins a complete database snapshot (MinTXID == 1). - pub fn is_snapshot(&self) -> bool { - self.min_txid == TXID(1) - } - - /// True if checksum tracking is disabled for this file. - pub fn no_checksum(&self) -> bool { - self.flags & HEADER_FLAG_NO_CHECKSUM != 0 - } - - /// Decodes a header from a 100-byte slice (ltx.go:302-326). All big-endian. - pub fn parse(b: &[u8]) -> Result
{ - if b.len() < HEADER_SIZE { - return Err(corrupt("short header")); - } - if &b[0..4] != MAGIC { - return Err(corrupt("bad magic")); - } - Ok(Header { - version: VERSION, - flags: u32_be(&b[4..]), - page_size: u32_be(&b[8..]), - commit: u32_be(&b[12..]), - min_txid: TXID(u64_be(&b[16..])), - max_txid: TXID(u64_be(&b[24..])), - timestamp: u64_be(&b[32..]) as i64, - pre_apply_checksum: u64_be(&b[40..]), - wal_offset: u64_be(&b[48..]) as i64, - wal_size: u64_be(&b[56..]) as i64, - wal_salt1: u32_be(&b[64..]), - wal_salt2: u32_be(&b[68..]), - node_id: u64_be(&b[72..]), - }) - } - - /// Encodes the header to 100 bytes (ltx.go:283-299). - pub fn marshal(&self) -> [u8; HEADER_SIZE] { - let mut b = [0u8; HEADER_SIZE]; - b[0..4].copy_from_slice(MAGIC); - b[4..8].copy_from_slice(&self.flags.to_be_bytes()); - b[8..12].copy_from_slice(&self.page_size.to_be_bytes()); - b[12..16].copy_from_slice(&self.commit.to_be_bytes()); - b[16..24].copy_from_slice(&self.min_txid.0.to_be_bytes()); - b[24..32].copy_from_slice(&self.max_txid.0.to_be_bytes()); - b[32..40].copy_from_slice(&(self.timestamp as u64).to_be_bytes()); - b[40..48].copy_from_slice(&self.pre_apply_checksum.to_be_bytes()); - b[48..56].copy_from_slice(&(self.wal_offset as u64).to_be_bytes()); - b[56..64].copy_from_slice(&(self.wal_size as u64).to_be_bytes()); - b[64..68].copy_from_slice(&self.wal_salt1.to_be_bytes()); - b[68..72].copy_from_slice(&self.wal_salt2.to_be_bytes()); - b[72..80].copy_from_slice(&self.node_id.to_be_bytes()); - b - } - - /// Validates header invariants (ltx.go:208-267). - pub fn validate(&self) -> Result<()> { - if self.version != VERSION { - return Err(corrupt("invalid version")); - } - if self.flags != (self.flags & HEADER_FLAG_MASK) { - return Err(corrupt("invalid flags")); - } - if !is_valid_page_size(self.page_size) { - return Err(corrupt("invalid page size")); - } - if self.min_txid == TXID(0) { - return Err(corrupt("minimum transaction id required")); - } - if self.max_txid == TXID(0) { - return Err(corrupt("maximum transaction id required")); - } - if self.min_txid > self.max_txid { - return Err(corrupt("transaction ids out of order")); - } - if self.wal_offset < 0 { - return Err(corrupt("wal offset cannot be negative")); - } - if self.wal_size < 0 { - return Err(corrupt("wal size cannot be negative")); - } - if (self.wal_salt1 != 0 || self.wal_salt2 != 0) && self.wal_offset == 0 { - return Err(corrupt("wal offset required if salt exists")); - } - if self.wal_offset == 0 && self.wal_size != 0 { - return Err(corrupt("wal offset required if wal size exists")); - } - if self.is_snapshot() { - if self.pre_apply_checksum != 0 { - return Err(corrupt("pre-apply checksum must be zero on snapshots")); - } - } else if self.no_checksum() { - if self.pre_apply_checksum != 0 { - return Err(corrupt("pre-apply checksum not allowed")); - } - } else { - if self.pre_apply_checksum == 0 { - return Err(corrupt("pre-apply checksum required on non-snapshot files")); - } - if self.pre_apply_checksum & CHECKSUM_FLAG == 0 { - return Err(corrupt("invalid pre-apply checksum format")); - } - } - Ok(()) - } -} - -/// Per-page header (6 bytes). Ported from ltx.go:408-447. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] -pub struct PageHeader { - pub pgno: u32, - pub flags: u16, -} - -impl PageHeader { - pub fn is_zero(&self) -> bool { - self.pgno == 0 && self.flags == 0 - } - - pub fn parse(b: &[u8]) -> Result { - if b.len() < PAGE_HEADER_SIZE { - return Err(corrupt("short page header")); - } - Ok(PageHeader { - pgno: u32_be(&b[0..]), - flags: u16_be(&b[4..]), - }) - } - - pub fn marshal(&self) -> [u8; PAGE_HEADER_SIZE] { - let mut b = [0u8; PAGE_HEADER_SIZE]; - b[0..4].copy_from_slice(&self.pgno.to_be_bytes()); - b[4..6].copy_from_slice(&self.flags.to_be_bytes()); - b - } - - pub fn validate(&self) -> Result<()> { - if self.pgno == 0 { - return Err(corrupt("page number required")); - } - if self.flags != (self.flags & PAGE_HEADER_FLAG_MASK) { - return Err(corrupt("invalid page header flags")); - } - Ok(()) - } -} - -/// File trailer (16 bytes). Ported from ltx.go:348-393. -#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] -pub struct Trailer { - pub post_apply_checksum: Checksum, - pub file_checksum: Checksum, -} - -impl Trailer { - /// Validates the checksum fields against the header checksum mode. - pub fn validate(&self, header: Header) -> Result<()> { - if header.no_checksum() { - if self.post_apply_checksum != 0 { - return Err(corrupt("post-apply checksum not allowed")); - } - } else if self.post_apply_checksum == 0 || self.post_apply_checksum & CHECKSUM_FLAG == 0 { - return Err(corrupt("invalid post-apply checksum")); - } - - if self.file_checksum == 0 || self.file_checksum & CHECKSUM_FLAG == 0 { - return Err(corrupt("invalid file checksum")); - } - Ok(()) - } - - pub fn parse(b: &[u8]) -> Result { - if b.len() < TRAILER_SIZE { - return Err(corrupt("short trailer")); - } - Ok(Trailer { - post_apply_checksum: u64_be(&b[0..]), - file_checksum: u64_be(&b[8..]), - }) - } - - pub fn marshal(&self) -> [u8; TRAILER_SIZE] { - let mut b = [0u8; TRAILER_SIZE]; - b[0..8].copy_from_slice(&self.post_apply_checksum.to_be_bytes()); - b[8..16].copy_from_slice(&self.file_checksum.to_be_bytes()); - b - } -} - -/// True if `sz` is a power of two in [512, 65536] (ltx.go:399-406). -pub fn is_valid_page_size(sz: u32) -> bool { - let mut i = 512u32; - while i <= 65536 { - if sz == i { - return true; - } - i *= 2; - } - false -} - -/// Formats an LTX filename for a transaction range (ltx.go:487-489). -pub fn format_filename(min_txid: TXID, max_txid: TXID) -> String { - format!("{}-{}.ltx", min_txid, max_txid) -} - -/// Parses a `-.ltx` filename (ltx.go:450-459). -pub fn parse_filename(name: &str) -> Result<(TXID, TXID)> { - let stem = name - .strip_suffix(".ltx") - .ok_or_else(|| corrupt("invalid ltx filename"))?; - let (a, b) = stem - .split_once('-') - .ok_or_else(|| corrupt("invalid ltx filename"))?; - if a.len() != 16 || b.len() != 16 { - return Err(corrupt("invalid ltx filename")); - } - let min = u64::from_str_radix(a, 16).map_err(|_| corrupt("invalid ltx filename"))?; - let max = u64::from_str_radix(b, 16).map_err(|_| corrupt("invalid ltx filename"))?; - Ok((TXID(min), TXID(max))) -} - -// ── FileInfo ────────────────────────────────────────────────────────────────── - -/// Metadata about an LTX file on a replica. Ported from ltx@v0.5.1 ltx.go:571-596. -/// -/// `pre_apply_checksum`/`post_apply_checksum` are populated when known (e.g. by -/// decoding) and are zero when a file is discovered by a bare directory/bucket -/// listing. Listings leave `size` and `created_at` unset, and write results use -/// the LTX header timestamp. -#[derive(Debug, Clone, PartialEq, Eq, Default)] -pub struct FileInfo { - pub level: i32, - pub min_txid: TXID, - pub max_txid: TXID, - pub pre_apply_checksum: Checksum, - pub post_apply_checksum: Checksum, - pub size: i64, - pub created_at: Option, -} - -// ── Decoder ────────────────────────────────────────────────────────────────── - -// ── Encoder (round-trip; byte-fidelity vs the real binary is D1's job) ──────── - -/// Encodes a complete LTX file with the legacy LZ4 frame representation, -/// which the decoder reads alongside the v0.5.2 block -/// representation. -pub fn encode_file( - header: &Header, - pages: &[(u32, Vec)], - post_apply_checksum: Checksum, -) -> Result> { - let mut encoder = crate::ltx::codec::Encoder::new(Vec::new()); - encoder.encode_header(*header)?; - for (page_number, data) in pages { - encoder.encode_page( - PageHeader { - pgno: *page_number, - flags: 0, - }, - data, - )?; - } - encoder.close(post_apply_checksum)?; - Ok(encoder.writer) -} - -/// Merges consecutive LTX files into one covering all their txids, as -/// ltx's compactor does. -pub fn merge_files(files: &[Vec]) -> Result> { - let mut out = Vec::new(); - merge_into(files.iter().map(Vec::as_slice).collect(), &mut out)?; - Ok(out) -} - -/// `merge_files`, from readers to a writer, a page at a time. -pub fn merge_into(inputs: Vec, out: impl std::io::Write) -> Result<()> { - let mut merge = Merge::new(inputs)?; - let mut encoder = crate::ltx::codec::Encoder::new(out); - encoder.encode_header(merge.header)?; - while let Some((pgno, data)) = merge.next()? { - encoder.encode_page(PageHeader { pgno, flags: 0 }, data)?; - } - encoder.close(merge.finish()?) -} - -/// Consecutive LTX files, oldest first, read as one: the latest version of -/// each page, through the last file's commit. Every file keeps its pages in -/// ascending order, so this holds a page per file, never a database. -pub struct Merge { - /// The merged file's header. - pub header: Header, - inputs: Vec>, - page: Vec, -} - -struct Input { - decoder: crate::ltx::codec::Decoder, - /// The page `page` holds; `None` once the file's pages are read. - pgno: Option, - page: Vec, -} - -impl Input { - fn advance(&mut self) -> Result<()> { - self.pgno = self.decoder.decode_page(&mut self.page)?.map(|p| p.pgno); - Ok(()) - } -} - -impl Merge { - pub fn new(readers: Vec) -> Result { - let mut inputs = Vec::new(); - for reader in readers { - let mut decoder = crate::ltx::codec::Decoder::new(reader); - decoder.decode_header()?; - let page = vec![0; decoder.header.page_size as usize]; - inputs.push(Input { - decoder, - pgno: None, - page, - }); - } - let (Some(first), Some(last)) = (inputs.first(), inputs.last()) else { - return Err(corrupt("nothing to merge")); - }; - let (first, last) = (first.decoder.header, last.decoder.header); - if inputs - .iter() - .any(|i| i.decoder.header.page_size != last.page_size) - { - return Err(corrupt("mismatched page sizes")); - } - let header = Header { - flags: first.flags, - min_txid: first.min_txid, - pre_apply_checksum: first.pre_apply_checksum, - ..last - }; - for input in &mut inputs { - input.advance()?; - } - let page = vec![0; header.page_size as usize]; - Ok(Self { - header, - inputs, - page, - }) - } - - /// The next page, in ascending order; `None` after the last. - pub fn next(&mut self) -> Result> { - loop { - let Some(pgno) = self.inputs.iter().filter_map(|i| i.pgno).min() else { - return Ok(None); - }; - let newest = self.inputs.iter_mut().rev().find(|i| i.pgno == Some(pgno)); - if let Some(newest) = newest { - std::mem::swap(&mut self.page, &mut newest.page); - } - for input in &mut self.inputs { - if input.pgno == Some(pgno) { - input.advance()?; - } - } - if pgno <= self.header.commit { - return Ok(Some((pgno, &self.page))); - } - } - } - - /// Checks every file's trailer: the last file's post-apply checksum. - pub fn finish(mut self) -> Result { - for input in &mut self.inputs { - input.decoder.close()?; - } - Ok(self - .inputs - .last() - .map_or(0, |i| i.decoder.trailer.post_apply_checksum)) - } -} - -// ── small byte / varint helpers ────────────────────────────────────────────── - -fn u16_be(b: &[u8]) -> u16 { - u16::from_be_bytes([b[0], b[1]]) -} -fn u32_be(b: &[u8]) -> u32 { - u32::from_be_bytes([b[0], b[1], b[2], b[3]]) -} -fn u64_be(b: &[u8]) -> u64 { - u64::from_be_bytes([b[0], b[1], b[2], b[3], b[4], b[5], b[6], b[7]]) -} - -// ── Tests ───────────────────────────────────────────────────────────────────── diff --git a/src/workerd/server/cluster/ltx/mod.rs b/src/workerd/server/cluster/ltx/mod.rs deleted file mode 100644 index 3661ee9ebcd..00000000000 --- a/src/workerd/server/cluster/ltx/mod.rs +++ /dev/null @@ -1,92 +0,0 @@ -// Ported from celld's `celld-ltx` crate (Apache License 2.0), which was -// seeded from rustyriver and carries: -// -// - rustyriver (https://github.com/mikenomitch/rustyriver): Copyright 2026 The -// rustyriver authors, Apache License 2.0. -// - Litestream (https://github.com/benbjohnson/litestream), tag v0.5.11: -// Copyright (c) Ben Johnson and the Litestream authors, Apache License 2.0. -// - LTX (https://github.com/superfly/ltx), tag v0.5.2: Copyright (c) -// Superfly, Inc., Apache License 2.0. - -//! LTX replication for a `SQLite` database: WAL capture into LTX segments, -//! upload to an object store, and restore. -//! -//! Ported from celld's `celld-ltx` crate (Apache-2.0), itself a from-scratch -//! Rust reimplementation of Litestream v0.5.11 (Apache-2.0, Ben Johnson) with -//! the block format of LTX v0.5.2 (Apache-2.0), by way of rustyriver. Only -//! the LTX format, the replica client and restore are ported; capture is -//! `capture.rs` and compaction is the owner's (`owner.rs`). Paged restore -//! and bundles stay in celld. -//! -//! # Core position and identity helpers -//! -//! Ported from litestream@v0.5.11 litestream.go:18-203 and -//! ltx@v0.5.2 ltx.go:66-145. - -// The modules below keep celld's source nearly line for line so a later -// diff against celld stays readable. They are therefore exempt from the -// pedantic and nursery style groups; the crate's deny lints still apply. -#[expect( - clippy::pedantic, - clippy::nursery, - reason = "ported from celld; kept diffable" -)] -pub mod client; -#[expect( - clippy::pedantic, - clippy::nursery, - reason = "ported from celld; kept diffable" -)] -pub(crate) mod codec; -pub mod error; -#[expect( - clippy::pedantic, - clippy::nursery, - clippy::module_inception, - reason = "ported from celld; kept diffable" -)] -pub mod ltx; -#[expect(clippy::nursery, reason = "ported from celld; kept diffable")] -pub mod replica; - -pub use client::bucket::BucketClient; -pub use error::Error; -pub use replica::SNAPSHOT_LEVEL; -pub use replica::plan; -pub use replica::restore; - -// ── TXID ────────────────────────────────────────────────────────────────────── - -/// A monotonically increasing transaction identifier. -/// -/// Represented as a `u64` and formatted as a zero-padded 16-character -/// lowercase hex string (e.g. `"0000000000000001"`). This matches the -/// on-disk LTX filename convention and `ltx.TXID.String()` in -/// ltx@v0.5.1 ltx.go:142-144. -#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Default)] -#[expect(clippy::upper_case_acronyms, reason = "Litestream's name for it")] -pub struct TXID(pub u64); - -impl std::fmt::Display for TXID { - /// Formats as a 16-digit lowercase hex string, zero-padded. - /// - /// Ported from `TXID.String()` in ltx@v0.5.1 ltx.go:142. - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{:016x}", self.0) - } -} - -// ── Checksum ────────────────────────────────────────────────────────────────── - -/// A 64-bit LTX rolling / file checksum. -/// -/// The high bit is the `ChecksumFlag` sentinel used by the LTX format to -/// distinguish a real checksum from an absent one. -/// -/// Ported from ltx@v0.5.1 ltx.go (type alias `Checksum uint64`). -pub type Checksum = u64; - -/// Flag bit set on every valid non-zero LTX checksum. -/// -/// Ported from `ChecksumFlag` in ltx@v0.5.1 ltx.go:55. -pub const CHECKSUM_FLAG: Checksum = 1 << 63; diff --git a/src/workerd/server/cluster/ltx/replica.rs b/src/workerd/server/cluster/ltx/replica.rs deleted file mode 100644 index d2597d85a9c..00000000000 --- a/src/workerd/server/cluster/ltx/replica.rs +++ /dev/null @@ -1,157 +0,0 @@ -//! Restore from a replica, ported from litestream@v0.5.11 `replica.go`. -//! Uploading is the owner's (`owner.rs`). -//! -//! Uploads live at L0; a compaction writes a snapshot from TXID 1 at -//! `SNAPSHOT_LEVEL` (litestream's L9). A restore is the newest snapshot and -//! the L0 files that extend it, merged the way `ltx.NewCompactor` and -//! `Decoder.DecodeDatabaseTo` do (ltx@v0.5.1). - -use std::fs::File; -use std::io::BufReader; -use std::io::BufWriter; -use std::io::Write; -use std::path::Path; -use std::path::PathBuf; - -use futures::StreamExt; -use futures::TryStreamExt; -use futures::stream; - -use crate::ltx::TXID; -use crate::ltx::client::ReplicaClient; -use crate::ltx::error::Error; -use crate::ltx::error::Result; -use crate::ltx::ltx::{self}; - -/// Downloads in flight during a restore, to hide the store's round trips. -const RESTORE_DOWNLOAD_CONCURRENCY: usize = 16; - -/// The level of snapshots, as litestream's `SnapshotLevel`. -pub const SNAPSHOT_LEVEL: i32 = 9; - -/// The files a restore through `txid` (`TXID(0)`: the latest) reads, in -/// order: the newest snapshot, then each L0 file that extends what came -/// before. A file inside the range already reached is skipped: tiering and -/// bucket-acknowledged commits upload merged ranges concurrently, so an -/// older, narrower range can land beside a newer, wider one, and applying it -/// after would bring back old pages (lost writes on the fleet). A file that -/// extends the range carries the newest version of every page it holds. Not -/// checked for gaps. -pub async fn plan(client: &C, txid: TXID) -> Result> { - let mut infos = client.ltx_files(SNAPSHOT_LEVEL, TXID(0)).await?; - infos.extend(client.ltx_files(0, TXID(0)).await?); - if txid != TXID(0) { - infos.retain(|info| info.max_txid <= txid); - } - infos.sort_by_key(|info| (info.min_txid, std::cmp::Reverse(info.max_txid))); - let mut reached = TXID(0); - infos.retain(|info| { - let extends = info.max_txid > reached; - reached = reached.max(info.max_txid); - extends - }); - Ok(infos) -} - -/// Restores the replica into `output_path`, which must not exist, up to -/// `txid` (`TXID(0)`: the latest). The plan (`plan`) must be contiguous -/// from TXID 1 (replica.go:1454-1526). The files are spooled to -/// `.restore/` and merged a page at a time into `.tmp`, -/// which is renamed into place (replica.go:657-694), on the blocking pool. -pub async fn restore(client: &C, output_path: &Path, txid: TXID) -> Result<()> { - if output_path.exists() { - return Err(Error::Other( - format!( - "cannot restore, output path already exists: {}", - output_path.display() - ) - .into(), - )); - } - let infos = plan(client, txid).await?; - let mut reached = TXID(0); - for info in &infos { - if info.min_txid.0 > reached.0 + 1 { - return Err(Error::Other( - format!( - "non-contiguous ltx files: have up to {reached} but next file starts at {}", - info.min_txid - ) - .into(), - )); - } - reached = reached.max(info.max_txid); - } - if infos.is_empty() || (txid != TXID(0) && reached < txid) { - return Err(Error::TxNotAvailable); - } - let spool = with_suffix(output_path, ".restore"); - std::fs::create_dir_all(&spool)?; - let files: Vec<_> = (0..infos.len()) - .map(|i| spool.join(i.to_string())) - .collect(); - let downloaded = stream::iter(infos.iter().zip(&files)) - .map(|(info, file)| client.open_ltx_file(info.level, info.min_txid, info.max_txid, file)) - .buffer_unordered(RESTORE_DOWNLOAD_CONCURRENCY) - .try_collect::<()>() - .await; - if let Err(e) = downloaded { - let _ = std::fs::remove_dir_all(&spool); - return Err(e); - } - let output = output_path.to_owned(); - let result = blocking(move || write_image(&files, &output)).await; - let _ = std::fs::remove_dir_all(&spool); - result -} - -/// Merges the LTX `files` (a snapshot, then what extends it) into the -/// database at `output`: pages `1..=commit`, the lock page zero-filled -/// (decoder.go:236-254). A page missing from the merge means the chain is -/// incomplete. There is no fsync: a restored image is never read after a -/// crash, since the takeover that follows one deletes it. -fn write_image(files: &[PathBuf], output: &Path) -> Result<()> { - let readers = files - .iter() - .map(|f| Ok(BufReader::new(File::open(f)?))) - .collect::>>()?; - let mut merge = ltx::Merge::new(readers)?; - let (page_size, commit) = (merge.header.page_size, merge.header.commit); - let lock = ltx::lock_pgno(page_size); - let zero = vec![0; page_size as usize]; - let tmp = with_suffix(output, ".tmp"); - let mut out = BufWriter::new(File::create(&tmp)?); - let mut next = 1; - let fill = |out: &mut BufWriter, next: &mut u32, to: u32| -> Result<()> { - for pgno in *next..to { - if pgno != lock { - return Err(Error::Other(format!("restore lacks page {pgno}").into())); - } - out.write_all(&zero)?; - } - *next = to.max(*next); - Ok(()) - }; - while let Some((pgno, data)) = merge.next()? { - fill(&mut out, &mut next, pgno)?; - out.write_all(data)?; - next = pgno + 1; - } - fill(&mut out, &mut next, commit + 1)?; - merge.finish()?; - out.flush()?; - drop(out); - Ok(std::fs::rename(&tmp, output)?) -} - -fn with_suffix(path: &Path, suffix: &str) -> PathBuf { - let mut path = path.as_os_str().to_owned(); - path.push(suffix); - path.into() -} - -async fn blocking(f: impl FnOnce() -> Result + Send + 'static) -> Result { - tokio::task::spawn_blocking(f) - .await - .map_err(|e| Error::Other(format!("blocking task: {e}").into()))? -} diff --git a/src/workerd/server/cluster/owner.rs b/src/workerd/server/cluster/owner.rs index 6c493d4d2f8..25cad5aa20d 100644 --- a/src/workerd/server/cluster/owner.rs +++ b/src/workerd/server/cluster/owner.rs @@ -7,7 +7,7 @@ //! //! A commit is acknowledged once the bucket holds it and own.json still //! names the epoch after the upload. Each epoch's commits form a chain of -//! LTX files under `do///ltx/e`, starting with a snapshot +//! log files under `do///log/e`, starting with a snapshot //! of the state the epoch inherited; a takeover restores the last opened //! epoch's chain. @@ -28,10 +28,11 @@ use crate::Ownership; use crate::Result; use crate::Txid; use crate::capture::Capture; -use crate::ltx::BucketClient; -use crate::ltx::SNAPSHOT_LEVEL; -use crate::ltx::TXID; -use crate::ltx::client::ReplicaClient; +use crate::log::Cursor; +use crate::log::File; +use crate::log::Log; +use crate::log::SNAPSHOT_LEVEL; +use crate::log::chain; use crate::ownership::Owned; /// Objects compacting at once, and deletes in flight for one compaction. @@ -100,6 +101,8 @@ struct Object { replica: Replica, /// How far the epoch's chain in the bucket is contiguous. tiered: Txid, + /// Where the capture was at eviction. + cursor: Cursor, /// The files of this epoch's chain. Only this owner writes the epoch's /// prefix, so compaction deletes from here rather than listing it. files: Vec, @@ -137,9 +140,9 @@ impl Owners { } } - fn client(&self, ns: &str, actor: &str, epoch: Epoch) -> BucketClient { - let path = format!("{}do/{ns}/{actor}/ltx/e{epoch}", self.bucket.prefix()); - BucketClient::new(self.bucket.at(&path)) + fn client(&self, ns: &str, actor: &str, epoch: Epoch) -> Log { + let path = format!("{}do/{ns}/{actor}/log/e{epoch}", self.bucket.prefix()); + Log::new(self.bucket.at(&path)) } /// Take the object over at `owned.epoch` and open the epoch, leaving its @@ -167,7 +170,7 @@ impl Owners { "{key}: not open here, nor dormant" ))); }; - let capture = Capture::resume(path, object.tiered); + let capture = Capture::resume(path, object.tiered, object.cursor); tracing::info!( "cluster: {ns}/{actor} epoch {} resumes at {}", object.epoch, @@ -188,35 +191,36 @@ impl Owners { if !local { remove_database(path)?; if let Some(recover) = recover { - let from = self.client(ns, actor, recover); - let end = end_of(&from).await?; + // Nothing in the bucket: an object that committed nothing. + let end = crate::log::restore(&self.client(ns, actor, recover), path).await?; tracing::info!( "cluster: {ns}/{actor} epoch {} recovers {recover}: bucket through {end}", owned.epoch ); - // Nothing in the bucket: an object that committed nothing. - if end > 0 { - crate::ltx::restore(&from, path, TXID(end)).await?; - } } } - // The state the epoch inherits is its snapshot at txid 1, encoded + // The state the epoch inherits is its snapshot at txid 1, written // from the file on the blocking pool; a new object has none. let cache = open_cache(path, owned.epoch)?; let snapshot = File { - level: 0, + level: SNAPSHOT_LEVEL, first: 1, last: 1, size: 0, }; + let session = owned.epoch as u32; let size = if self.opened.borrow().contains_key(&key) { None } else { let (path, out) = (path.to_owned(), cache.join(snapshot.name())); - blocking(move || crate::capture::snapshot_file(&path, &out)).await? + blocking(move || crate::log::snapshot_file(&path, &out, session)).await? }; let snapshot = size.map(|size| File { size, ..snapshot }); - let capture = Capture::resume(path, snapshot.map_or(0, |f| f.last)); + let cursor = Cursor { + session, + ..Cursor::default() + }; + let capture = Capture::resume(path, snapshot.map_or(0, |f| f.last), cursor); let object = self .open_epoch(ownership, ns, actor, owned, cache, snapshot) .await?; @@ -270,6 +274,7 @@ impl Owners { cache, }, tiered: 0, + cursor: Cursor::default(), files: Vec::new(), }; if self @@ -390,9 +395,10 @@ impl Owners { async fn compact(&self, slot: &Slot) { let (replica, epoch, chain) = { let guard = slot.object.lock().await; - let Some(chain) = chain(&guard.files) else { + let chain = chain(&guard.files); + if chain.is_empty() { return; - }; + } (guard.replica.clone(), guard.epoch, chain) }; let Ok(snapshot) = merged(&replica, &chain).await else { @@ -418,10 +424,7 @@ impl Owners { stream::iter(replaced) .for_each_concurrent(TIER_CONCURRENCY, |f| async move { let _ = std::fs::remove_file(replica.cache.join(f.name())); - let _ = replica - .client - .delete_ltx_file(f.level, TXID(f.first), TXID(f.last)) - .await; + let _ = replica.client.delete(&f).await; }) .await; } @@ -452,8 +455,9 @@ impl Owners { return; }; self.tier_one(&slot).await; - let guard = slot.object.lock().await; + let mut guard = slot.object.lock().await; let last = slot.capture.borrow().last(); + guard.cursor = slot.capture.borrow().cursor(); if guard.epoch == epoch && last.is_ok_and(|last| guard.tiered >= last) { // Bounded: a forgotten entry costs its object a restore. let mut map = self.dormant.borrow_mut(); @@ -493,28 +497,14 @@ impl Owners { } } -/// Where an epoch's chain in the bucket is contiguous from its start -/// through. -async fn end_of(client: &BucketClient) -> Result { - let mut end = 0; - for info in crate::ltx::plan(client, TXID(0)).await? { - if info.min_txid.0 > end + 1 { - break; - } - end = end.max(info.max_txid.0); - } - Ok(end) -} - -/// One LTX file covering txids `first..=last`, ready to upload. +/// One log file covering txids `first..=last`, ready to upload. struct Staged { first: Txid, last: Txid, bytes: Vec, } -/// The capture's entries after `from` through `through`, merged into one -/// LTX file, as litestream compacts. +/// The capture's entries after `from` through `through`, as one log file. fn stage( entry: impl Fn(Txid) -> Result>, from: Txid, @@ -529,42 +519,20 @@ fn stage( Ok(Some(Staged { first: from + 1, last: through, - bytes: crate::ltx::ltx::merge_files(&files)?, + bytes: crate::log::compress(&crate::log::merge(&files)?)?, })) } -/// The files a restore reads, in order, as `ltx::plan` picks them; `None` -/// unless they run unbroken from txid 1. -fn chain(files: &[File]) -> Option> { - let mut files = files.to_vec(); - files.sort_by_key(|f| (f.first, std::cmp::Reverse(f.last))); - let mut reached = 0; - let mut chain = Vec::new(); - for f in files { - if f.first > reached + 1 { - return None; - } - if f.last > reached { - reached = f.last; - chain.push(f); - } - } - (!chain.is_empty()).then_some(chain) -} - -/// `chain`'s files merged into this node's copy of a snapshot covering -/// txid 1 through the chain's end, a page at a time on the blocking pool. -/// A missing copy is fetched from the bucket first. +/// `chain`'s files replayed into this node's copy of a snapshot covering +/// txid 1 through the chain's end, on the blocking pool. A missing copy is +/// fetched from the bucket first. async fn merged(replica: &Replica, chain: &[File]) -> Result { let paths: Vec<_> = chain.iter().map(|f| replica.cache.join(f.name())).collect(); for (f, path) in chain.iter().zip(&paths) { if !path.exists() { - let (first, last, tmp) = (TXID(f.first), TXID(f.last), tmp_path(path)); - replica - .client - .open_ltx_file(f.level, first, last, &tmp) - .await?; - std::fs::rename(&tmp, path).map_err(crate::ltx::Error::from)?; + let tmp = tmp_path(path); + replica.client.get(f, &tmp).await?; + std::fs::rename(&tmp, path)?; } } let snapshot = File { @@ -574,21 +542,8 @@ async fn merged(replica: &Replica, chain: &[File]) -> Result { size: 0, }; let out = replica.cache.join(snapshot.name()); - let size = blocking(move || { - let io = |e| crate::Error::from(crate::ltx::Error::from(e)); - let files = paths.iter().map(std::fs::File::open); - let readers: Vec<_> = files - .map(|f| f.map(std::io::BufReader::new)) - .collect::>() - .map_err(io)?; - let tmp = tmp_path(&out); - let mut writer = std::io::BufWriter::new(std::fs::File::create(&tmp).map_err(io)?); - crate::ltx::ltx::merge_into(readers, &mut writer)?; - std::io::Write::flush(&mut writer).map_err(io)?; - std::fs::rename(&tmp, &out).map_err(io)?; - Ok(std::fs::metadata(&out).map_err(io)?.len()) - }) - .await?; + let files: Vec<_> = chain.iter().copied().zip(paths).collect(); + let size = blocking(move || crate::log::compact(&files, &out)).await?; Ok(File { size, ..snapshot }) } @@ -610,7 +565,9 @@ fn tmp_path(path: &Path) -> PathBuf { tmp.into() } -async fn blocking(f: impl FnOnce() -> Result + Send + 'static) -> Result { +pub async fn blocking( + f: impl FnOnce() -> Result + Send + 'static, +) -> Result { tokio::task::spawn_blocking(f) .await .map_err(|e| Error::Replica(format!("blocking task: {e}")))? @@ -619,8 +576,8 @@ async fn blocking(f: impl FnOnce() -> Result + Send + 'sta /// Where an epoch's chain is: the bucket, and this node's copy of it. #[derive(Clone)] struct Replica { - client: BucketClient, - /// `.ltx/e`. + client: Log, + /// `.log/e`. cache: PathBuf, } @@ -631,43 +588,23 @@ fn open_cache(path: &Path, epoch: Epoch) -> Result { let root = cache_root(path); remove_dir(&root)?; let cache = root.join(format!("e{epoch}")); - std::fs::create_dir_all(&cache).map_err(crate::ltx::Error::from)?; + std::fs::create_dir_all(&cache)?; Ok(cache) } fn cache_root(path: &Path) -> PathBuf { let mut root = path.as_os_str().to_owned(); - root.push(".ltx"); + root.push(".log"); root.into() } fn remove_dir(path: &Path) -> Result<()> { match std::fs::remove_dir_all(path) { - Err(e) if e.kind() != std::io::ErrorKind::NotFound => { - Err(crate::ltx::Error::from(e).into()) - } + Err(e) if e.kind() != std::io::ErrorKind::NotFound => Err(crate::Error::from(e)), _ => Ok(()), } } -/// One LTX file in an epoch's chain. -#[derive(Clone, Copy, Debug, PartialEq, Eq)] -struct File { - level: i32, - first: Txid, - last: Txid, - size: u64, -} - -impl File { - fn name(&self) -> String { - format!( - "{:04x}-{:016x}-{:016x}.ltx", - self.level, self.first, self.last - ) - } -} - /// L0 files a snapshot waits for, so a small object does not snapshot on /// nearly every commit. const SNAPSHOT_MIN_FILES: usize = 32; @@ -688,12 +625,8 @@ fn snapshot_due(files: &[File]) -> bool { /// Uploads this node's copy of `file`. async fn upload(replica: &Replica, file: File) -> Result { - let (first, last) = (TXID(file.first), TXID(file.last)); let path = replica.cache.join(file.name()); - replica - .client - .write_ltx_file(file.level, first, last, &path) - .await?; + replica.client.put(&file, &path).await?; Ok(file) } @@ -709,15 +642,13 @@ async fn put(replica: &Replica, level: i32, staged: Option) -> Result Result<()> { match std::fs::remove_file(path) { - Err(e) if e.kind() != std::io::ErrorKind::NotFound => { - Err(crate::ltx::Error::from(e).into()) - } + Err(e) if e.kind() != std::io::ErrorKind::NotFound => Err(crate::Error::from(e)), _ => Ok(()), } } @@ -760,65 +691,10 @@ mod tests { let snapshot = span(SNAPSHOT_LEVEL, 1, 4); let (narrow, wide) = (span(0, 5, 6), span(0, 5, 8)); let chain = chain(&[span(0, 9, 9), narrow, snapshot, wide]); - assert_eq!(chain, Some(vec![snapshot, wide, span(0, 9, 9)])); - assert_eq!(super::chain(&[snapshot, span(0, 6, 6)]), None, "a gap"); - assert_eq!(super::chain(&[]), None); - } - - #[test] - fn a_compaction_merges_the_chain() { - let ltx = |first: Txid, last: Txid, commit: u32, pages: &[(u32, u8)]| { - let header = crate::ltx::ltx::Header { - version: crate::ltx::ltx::VERSION, - flags: crate::ltx::ltx::HEADER_FLAG_NO_CHECKSUM, - page_size: 512, - commit, - min_txid: TXID(first), - max_txid: TXID(last), - ..Default::default() - }; - let pages: Vec<_> = pages.iter().map(|&(p, b)| (p, vec![b; 512])).collect(); - crate::ltx::ltx::encode_file(&header, &pages, 0).unwrap() - }; - let dir = std::env::temp_dir().join(format!("owner-test-{}", std::process::id())); - let replica = Replica { - client: BucketClient::new(Bucket::memory("e1")), - cache: open_cache(&dir.join("a.sqlite"), 1).unwrap(), - }; - let files = [ - (span(0, 1, 1), ltx(1, 1, 3, &[(1, 1), (2, 1), (3, 1)])), - (span(0, 2, 3), ltx(2, 3, 3, &[(2, 2)])), - (span(0, 4, 4), ltx(4, 4, 2, &[(1, 4)])), - ]; - let runtime = tokio::runtime::Builder::new_current_thread() - .build() - .unwrap(); - let snapshot = runtime.block_on(async { - // The first two only here, the last only in the bucket. - for (f, bytes) in &files[..2] { - std::fs::write(replica.cache.join(f.name()), bytes).unwrap(); - } - let (f, bytes) = &files[2]; - let (first, last, from) = (TXID(f.first), TXID(f.last), dir.join("up")); - std::fs::write(&from, bytes).unwrap(); - replica - .client - .write_ltx_file(f.level, first, last, &from) - .await - .unwrap(); - let chain: Vec<_> = files.iter().map(|(f, _)| *f).collect(); - merged(&replica, &chain).await.unwrap() - }); - let bytes = std::fs::read(replica.cache.join(snapshot.name())).unwrap(); - std::fs::remove_dir_all(&dir).unwrap(); - assert_eq!((snapshot.first, snapshot.last), (1, 4)); - let mut merge = crate::ltx::ltx::Merge::new(vec![bytes.as_slice()]).unwrap(); - assert_eq!(merge.header.commit, 2, "truncated by the last commit"); - let mut firsts = Vec::new(); - while let Some((pgno, page)) = merge.next().unwrap() { - firsts.push((pgno, page[0])); - } - assert_eq!(firsts, [(1, 4), (2, 2)]); + assert_eq!(chain, vec![snapshot, wide, span(0, 9, 9)]); + let gap = super::chain(&[snapshot, span(0, 6, 6)]); + assert_eq!(gap, vec![snapshot], "through the gap"); + assert_eq!(super::chain(&[]), vec![]); } #[test] From 2a39d16adcd8a579f40e59ba70d97ba1e53d5493 Mon Sep 17 00:00:00 2001 From: Ryan Dahl Date: Mon, 26 Oct 2026 21:42:53 -0700 Subject: [PATCH 07/38] cluster: start every log file with a FileHeader A file says whether it holds frames or a snapshot, so replay no longer needs the level from its key, and a later field or kind of file has a place to go. --- src/workerd/server/cluster/frames.capnp | 13 +++++++++ src/workerd/server/cluster/log.rs | 39 +++++++++++++++---------- 2 files changed, 36 insertions(+), 16 deletions(-) diff --git a/src/workerd/server/cluster/frames.capnp b/src/workerd/server/cluster/frames.capnp index 1af91fed60d..33ef129fe98 100644 --- a/src/workerd/server/cluster/frames.capnp +++ b/src/workerd/server/cluster/frames.capnp @@ -11,6 +11,19 @@ using Cxx = import "/capnp/c++.capnp"; $Cxx.namespace("workerd::server::cluster"); +struct FileHeader { + # The first message of every file in the bucket. A file's level and + # range are in its key, not here. + + union { + log @0 :Void; + # Frames follow. + + snapshot @1 :SnapshotHeader; + # Each lane's content follows. + } +} + struct Frame { # A batch of frames in the bucket starts and ends at a commit, and is # applied whole. diff --git a/src/workerd/server/cluster/log.rs b/src/workerd/server/cluster/log.rs index cc0807a4695..908bbc2950e 100644 --- a/src/workerd/server/cluster/log.rs +++ b/src/workerd/server/cluster/log.rs @@ -7,7 +7,7 @@ //! wrote. //! //! An epoch's log is a chain of files under its prefix, each one LZ4 -//! frame: `0000/-.log`, the frames of commits `first..=last`, +//! frame that starts with a `FileHeader`: `0000/-.log`, the frames of commits `first..=last`, //! and `0009/0000000000000001-.log`, a snapshot through `last`. A //! restore is the newest snapshot and the files that extend it; a frame a //! file before already applied is skipped by its sequence. @@ -21,6 +21,7 @@ use std::io::Write; use std::path::Path; use std::path::PathBuf; +use frames_capnp::file_header; use frames_capnp::frame; use frames_capnp::snapshot_header; use futures::StreamExt; @@ -88,8 +89,8 @@ fn frame_at(out: &mut Vec, session: u32, sequence: u32, body: Body) { let _ = capnp::serialize::write_message(&mut *out, &message); } -/// Merges the frames of `entries`, consecutive commits, so each run of -/// page frames becomes one with the newest version of each page. +/// The log file of `entries`, consecutive commits: each run of page frames +/// becomes one with the newest version of each page. pub fn merge(entries: &[Vec]) -> Result> { let mut messages = Vec::new(); for entry in entries { @@ -99,6 +100,9 @@ pub fn merge(entries: &[Vec]) -> Result> { } } let mut out = Vec::new(); + let mut header = capnp::message::Builder::new_default(); + header.init_root::().set_log(()); + capnp::serialize::write_message(&mut out, &header)?; let mut run: Option<(u32, u32, Pages)> = None; let flush = |out: &mut Vec, run: &mut Option<(u32, u32, Pages)>| { if let Some((session, sequence, (commit, mut pages))) = run.take() { @@ -166,14 +170,19 @@ impl Replay { }) } - /// Applies the file `path` of the chain at `level`. - pub fn apply(&mut self, level: i32, path: &Path) -> Result<()> { + /// Applies the file `path` of the chain. + pub fn apply(&mut self, path: &Path) -> Result<()> { let file = std::fs::File::open(path)?; let mut r = BufReader::new(lz4_flex::frame::FrameDecoder::new(file)); - if level == SNAPSHOT_LEVEL { - return self.snapshot(&mut r); + let message = capnp::serialize::read_message(&mut r, options())?; + match message.get_root::()?.which()? { + file_header::Log(()) => self.log(&mut r), + file_header::Snapshot(header) => self.snapshot(header?, &mut r), } - while let Some(message) = capnp::serialize::try_read_message(&mut r, options())? { + } + + fn log(&mut self, r: &mut impl Read) -> Result<()> { + while let Some(message) = capnp::serialize::try_read_message(&mut *r, options())? { let f = message.get_root::()?; let at = (f.get_session(), f.get_sequence()); if self.next.is_some_and(|next| at < next) { @@ -185,9 +194,7 @@ impl Replay { Ok(()) } - fn snapshot(&mut self, r: &mut impl Read) -> Result<()> { - let message = capnp::serialize::read_message(&mut *r, options())?; - let header = message.get_root::()?; + fn snapshot(&mut self, header: snapshot_header::Reader, r: &mut impl Read) -> Result<()> { self.next = Some((header.get_session(), header.get_frame())); self.live = false; self.db.set_len(0)?; @@ -258,7 +265,7 @@ fn write_snapshot( payload: impl FnOnce(&mut dyn Write) -> Result<()>, ) -> Result { let mut message = capnp::message::Builder::new_default(); - let mut header = message.init_root::(); + let mut header = message.init_root::().init_snapshot(); header.set_session(session); header.set_frame(frame); let mut lanes = header.init_lanes(u32::from(size.is_some())); @@ -304,8 +311,8 @@ pub fn snapshot_file(path: &Path, out: &Path, session: u32) -> Result Result { let db = with_suffix(out, ".db"); let mut replay = Replay::new(&db)?; - for (f, path) in files { - replay.apply(f.level, path)?; + for (_, path) in files { + replay.apply(path)?; } let size = replay.write_snapshot(out); let _ = std::fs::remove_file(&db); @@ -438,8 +445,8 @@ pub async fn restore(log: &Log, output: &Path) -> Result { Ok(()) => { crate::owner::blocking(move || { let mut replay = Replay::new(&output)?; - for (f, path) in &files { - replay.apply(f.level, path)?; + for (_, path) in &files { + replay.apply(path)?; } replay.finish() }) From 5eaa0b2f26a92fd075e622efb7b74a3eeeb46a62 Mon Sep 17 00:00:00 2001 From: Ryan Dahl Date: Mon, 26 Oct 2026 21:52:38 -0700 Subject: [PATCH 08/38] cluster: never route to this address's previous instance A process that restarts at the same address before its predecessor's lease runs out finds the predecessor's record live and routed to it, which is itself: the call came back in and routed again, with no hop limit. Refuse instead, until the lease ends and the object can be claimed. --- src/workerd/server/cluster/ownership.rs | 29 +++++++++++++++++++++++++ src/workerd/server/cluster/registry.rs | 5 +++++ 2 files changed, 34 insertions(+) diff --git a/src/workerd/server/cluster/ownership.rs b/src/workerd/server/cluster/ownership.rs index c1b7cc41161..2143d93c68b 100644 --- a/src/workerd/server/cluster/ownership.rs +++ b/src/workerd/server/cluster/ownership.rs @@ -173,6 +173,14 @@ impl Ownership { if !record.node.is_empty() && let Some(peer) = self.registry.live(&record.node, now_ms).await? { + // An earlier process at our address, restarted + // before its lease ran out: a route there loops back + // here. Its lease, not a route, ends this. + if peer.address == self.registry.advertise() { + return Err(Error::Replica(format!( + "{key}: owned by this address's previous instance" + ))); + } return Ok(self.route(key, peer, now_ms)); } (record, Guard::Match(version)) @@ -405,6 +413,27 @@ mod tests { }); } + #[test] + fn a_restart_never_routes_to_its_previous_instance() { + block_on(async { + let store = Bucket::memory(""); + let (a, a_own) = node(&store, "a"); + a.renew(0).await.unwrap(); + open_x(&a_own).await; + // The process restarts at a's address with a fresh node id while + // a's record is still live: routing there would reach itself. + let bucket = store.at("cluster"); + let a_ = Rc::new(Registry::new(bucket.clone(), "a_", "a:1", TTL)); + a_.renew(1_000).await.unwrap(); + let a_own_ = Ownership::new(bucket, a_); + let claim = a_own_.acquire_or_route("ns", "x", 1_000).await; + assert!(matches!(claim, Err(Error::Replica(_))), "{claim:?}"); + // Once a's lease ends, the restart takes it over. + let claim = a_own_.acquire_or_route("ns", "x", 11_000).await; + assert_eq!(claim.unwrap(), owned(2)); + }); + } + #[test] fn a_rewritten_node_record_fences_its_instance() { block_on(async { diff --git a/src/workerd/server/cluster/registry.rs b/src/workerd/server/cluster/registry.rs index dacdca3e20b..0f52f9e616e 100644 --- a/src/workerd/server/cluster/registry.rs +++ b/src/workerd/server/cluster/registry.rs @@ -69,6 +69,11 @@ impl Registry { &self.node } + /// Where this instance listens. + pub fn advertise(&self) -> &str { + &self.address + } + /// Where peer `node` listens, as of the last listing. pub fn address(&self, node: &str) -> Option { self.listed.borrow().get(node).map(|r| r.address.clone()) From 1f38de499f65b206b7a9b7c08163f26d0e2fe552 Mon Sep 17 00:00:00 2001 From: Ryan Dahl Date: Mon, 26 Oct 2026 21:52:38 -0700 Subject: [PATCH 09/38] cluster: say why a compaction did not happen Every failure returned silently, and a gap in the chain stranded the files past it while the prefix kept compacting. On a soak, one epoch grew 1,454 L0 files after its last snapshot with nothing in the log. --- src/workerd/server/cluster/owner.rs | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/src/workerd/server/cluster/owner.rs b/src/workerd/server/cluster/owner.rs index 25cad5aa20d..29a1b0689ef 100644 --- a/src/workerd/server/cluster/owner.rs +++ b/src/workerd/server/cluster/owner.rs @@ -393,20 +393,29 @@ impl Owners { /// upload. The snapshot merges the chain's files, never the live /// database, so the merge runs off the event loop. async fn compact(&self, slot: &Slot) { - let (replica, epoch, chain) = { + let (replica, epoch, chain, key) = { let guard = slot.object.lock().await; let chain = chain(&guard.files); if chain.is_empty() { return; } - (guard.replica.clone(), guard.epoch, chain) + // A gap strands every file past it: say so, since the chain's + // prefix still compacts and nothing else would show it. + let end = guard.files.iter().map(|f| f.last).max(); + let key = format!("{}/{}", guard.ns, guard.actor); + if end > chain.last().map(|f| f.last) { + tracing::warn!("cluster: {key} chain has a gap before {end:?}"); + } + (guard.replica.clone(), guard.epoch, chain, key) }; - let Ok(snapshot) = merged(&replica, &chain).await else { - return; + let snapshot = match merged(&replica, &chain).await { + Ok(snapshot) => snapshot, + Err(e) => return tracing::warn!("cluster: {key} compaction failed: {e}"), }; let last = snapshot.last; - let Ok(snapshot) = upload(&replica, snapshot).await else { - return; + let snapshot = match upload(&replica, snapshot).await { + Ok(snapshot) => snapshot, + Err(e) => return tracing::warn!("cluster: {key} snapshot upload failed: {e}"), }; let replaced: Vec = { let mut guard = slot.object.lock().await; From 9248ea1d477655b7d466c26eaa64ee7127ad3698 Mon Sep 17 00:00:00 2001 From: Ryan Dahl Date: Tue, 27 Oct 2026 11:19:33 -0700 Subject: [PATCH 10/38] cluster: repin crates for lz4_flex and ring Main now checks in crate_universe's lockfile, so a Cargo.toml that adds crates must repin it (CARGO_BAZEL_REPIN=1). The repin also catches up main's Cargo.lock bumps that the lockfile had not followed. --- deps/rust/Cargo.toml | 2 +- deps/rust/cargo-bazel-lock.json | 944 +++++++++++--------------------- 2 files changed, 329 insertions(+), 617 deletions(-) diff --git a/deps/rust/Cargo.toml b/deps/rust/Cargo.toml index e66c65d9cda..fb0f6e4cc1d 100644 --- a/deps/rust/Cargo.toml +++ b/deps/rust/Cargo.toml @@ -67,7 +67,7 @@ libc = "0" libm = "0" lol_html_c_api = { git = "https://github.com/cloudflare/lol-html" } hashlink = "0" -# cluster's LTX port (src/workerd/server/cluster/ltx): LTX page compression. +# cluster: the LZ4 frame of every log file in the bucket. lz4_flex = { version = "0.11", default-features = false, features = ["frame", "safe-decode", "safe-encode"] } # kj-rs-io: `fs` for fcntl (KJ's fd flags), `signal` for ignoring SIGPIPE once per process like # kj::UnixEventPort does. diff --git a/deps/rust/cargo-bazel-lock.json b/deps/rust/cargo-bazel-lock.json index 282eee23c7b..8401d459518 100644 --- a/deps/rust/cargo-bazel-lock.json +++ b/deps/rust/cargo-bazel-lock.json @@ -2,7 +2,7 @@ "binary_crates": [ "capnpc 0.27.0" ], - "checksum": "3ef4c2d4c2a480f444a657336ab4e9dd7cf5874bc6365c7919303353139fe15c", + "checksum": "61cebc4aa64c396cca8187baadd5b83d7a8f8d63c4d8a5dbbe2f03ae35bde8d6", "conditions": { "aarch64-apple-darwin": [ "aarch64-apple-darwin" @@ -138,7 +138,6 @@ "cfg(windows)": [ "x86_64-pc-windows-msvc" ], - "cfg(windows_raw_dylib)": [], "i686-pc-windows-gnullvm": [], "x86_64-apple-darwin": [ "x86_64-apple-darwin" @@ -156,7 +155,7 @@ "build_script_attrs": { "deps": [ { - "id": "cc 1.9.0", + "id": "cc 1.10.0", "target": "cc" }, { @@ -806,6 +805,17 @@ }, "bitflags 2.13.2": { "common_attrs": { + "crate_features": { + "common": [], + "selects": { + "aarch64-apple-darwin": [ + "std" + ], + "x86_64-apple-darwin": [ + "std" + ] + } + }, "edition": "2021" }, "library_target_name": "bitflags", @@ -1186,7 +1196,7 @@ ], "version": "0.27.0" }, - "capnp-rpc 0.27.1": { + "capnp-rpc 0.27.2": { "common_attrs": { "deps": [ { @@ -1214,8 +1224,8 @@ "package_url": "https://github.com/capnproto/capnproto-rust", "repository": { "Http": { - "sha256": "fe88941ca04ec396b38ce5304fe9e40257ac7373e84b2a4fc6c2edd0b58fd56f", - "url": "https://static.crates.io/crates/capnp-rpc/0.27.1/download" + "sha256": "6fe464cf9820b078dfa044d6b352e9ec685a797c5fb4b265db332e5acd4093da", + "url": "https://static.crates.io/crates/capnp-rpc/0.27.2/download" } }, "targets": [ @@ -1226,7 +1236,7 @@ } } ], - "version": "0.27.1" + "version": "0.27.2" }, "capnpc 0.27.0": { "common_attrs": { @@ -1307,7 +1317,7 @@ ], "version": "0.2.4" }, - "cc 1.9.0": { + "cc 1.10.0": { "common_attrs": { "crate_features": [ "parallel" @@ -1367,8 +1377,8 @@ "package_url": "https://github.com/rust-lang/cc-rs", "repository": { "Http": { - "sha256": "877bf67ac8e52449eb2f9e98a344ea163beb70dd1f9eebe3cc5357dcfeaee4df", - "url": "https://static.crates.io/crates/cc/1.9.0/download" + "sha256": "86e055fa6e3e40ad6187ba25b6855eeb635d5271f8e0f0e1e9a5826b8d75bea0", + "url": "https://static.crates.io/crates/cc/1.10.0/download" } }, "targets": [ @@ -1379,7 +1389,7 @@ } } ], - "version": "1.9.0" + "version": "1.10.0" }, "cfg-if 1.0.5": { "common_attrs": { @@ -2233,7 +2243,7 @@ "target": "phf" }, { - "id": "smallvec 1.16.3", + "id": "smallvec 1.16.4", "target": "smallvec" } ], @@ -2351,7 +2361,7 @@ "target": "serde" }, { - "id": "uuid 1.28.0", + "id": "uuid 1.29.0", "target": "uuid" } ], @@ -2676,7 +2686,7 @@ "target": "capnp" }, { - "id": "capnp-rpc 0.27.1", + "id": "capnp-rpc 0.27.2", "target": "capnp_rpc" }, { @@ -2684,7 +2694,7 @@ "target": "capnpc" }, { - "id": "cc 1.9.0", + "id": "cc 1.10.0", "target": "cc" }, { @@ -2771,12 +2781,16 @@ "id": "lol_html_c_api 1.4.0", "target": "lolhtml" }, + { + "id": "lz4_flex 0.11.6", + "target": "lz4_flex" + }, { "id": "nix 0.31.3", "target": "nix" }, { - "id": "notify 8.2.0", + "id": "notify 8.2.1", "target": "notify" }, { @@ -2799,6 +2813,10 @@ "id": "rand 0.9.5", "target": "rand" }, + { + "id": "ring 0.17.14", + "target": "ring" + }, { "id": "ruff_python_ast 0.0.0", "target": "ruff_python_ast" @@ -2860,7 +2878,7 @@ "target": "thiserror" }, { - "id": "tokio 1.54.0", + "id": "tokio 1.54.1", "target": "tokio" }, { @@ -3598,40 +3616,6 @@ ], "version": "3.0.0" }, - "fsevent-sys 4.1.0": { - "common_attrs": { - "deps": [ - { - "id": "libc 0.2.191", - "target": "libc" - } - ], - "edition": "2018" - }, - "library_target_name": "fsevent_sys", - "license": "MIT", - "license_file": "LICENSE", - "license_ids": [ - "MIT" - ], - "name": "fsevent-sys", - "package_url": "https://github.com/octplane/fsevent-rust/tree/master/fsevent-sys", - "repository": { - "Http": { - "sha256": "76ee7a02da4d231650c7cea31349b889be2f45ddb3ef3032d2ec8185f6313fd2", - "url": "https://static.crates.io/crates/fsevent-sys/4.1.0/download" - } - }, - "targets": [ - { - "Library": { - "crate_name": "fsevent_sys", - "crate_root": "src/lib.rs" - } - } - ], - "version": "4.1.0" - }, "funty 2.0.0": { "common_attrs": { "edition": "2018" @@ -4964,11 +4948,11 @@ "target": "pin_project_lite" }, { - "id": "smallvec 1.16.3", + "id": "smallvec 1.16.4", "target": "smallvec" }, { - "id": "tokio 1.54.0", + "id": "tokio 1.54.1", "target": "tokio" }, { @@ -5061,7 +5045,7 @@ "target": "socket2" }, { - "id": "tokio 1.54.0", + "id": "tokio 1.54.1", "target": "tokio" }, { @@ -5099,7 +5083,7 @@ ], "version": "0.1.21" }, - "icu_collections 2.3.0": { + "icu_collections 2.4.0": { "common_attrs": { "deps": [ { @@ -5111,7 +5095,7 @@ "target": "utf8_iter" }, { - "id": "yoke 0.8.3", + "id": "yoke 0.8.4", "target": "yoke" }, { @@ -5119,7 +5103,7 @@ "target": "zerofrom" }, { - "id": "zerovec 0.11.8", + "id": "zerovec 0.11.9", "target": "zerovec" } ], @@ -5141,8 +5125,8 @@ "package_url": "https://github.com/unicode-org/icu4x", "repository": { "Http": { - "sha256": "fa68d21081c4a05d5a901a1c62add574c77048b6a1c67be3b50ce0b60d4ca513", - "url": "https://static.crates.io/crates/icu_collections/2.3.0/download" + "sha256": "fe090a42a0d3140fed8c63263ed1fecc02422cb2f4bfe55cd8de2de1b8e105dd", + "url": "https://static.crates.io/crates/icu_collections/2.4.0/download" } }, "targets": [ @@ -5153,9 +5137,9 @@ } } ], - "version": "2.3.0" + "version": "2.4.0" }, - "icu_locale_core 2.3.0": { + "icu_locale_core 2.4.0": { "common_attrs": { "crate_features": [ "zerovec" @@ -5166,7 +5150,7 @@ "target": "litemap" }, { - "id": "tinystr 0.8.4", + "id": "tinystr 0.8.5", "target": "tinystr" }, { @@ -5174,7 +5158,7 @@ "target": "writeable" }, { - "id": "zerovec 0.11.8", + "id": "zerovec 0.11.9", "target": "zerovec" } ], @@ -5196,8 +5180,8 @@ "package_url": "https://github.com/unicode-org/icu4x", "repository": { "Http": { - "sha256": "d56e28588da92eee5c3201a6eff33fabdd49b62269c8938d4ff050ce4d900deb", - "url": "https://static.crates.io/crates/icu_locale_core/2.3.0/download" + "sha256": "25d29b57c17c6528d0dc4c2141cfa2d08200465acb537a3909c8b9b8a4a984c0", + "url": "https://static.crates.io/crates/icu_locale_core/2.4.0/download" } }, "targets": [ @@ -5208,32 +5192,32 @@ } } ], - "version": "2.3.0" + "version": "2.4.0" }, - "icu_normalizer 2.3.0": { + "icu_normalizer 2.4.0": { "common_attrs": { "crate_features": [ "compiled_data" ], "deps": [ { - "id": "icu_collections 2.3.0", + "id": "icu_collections 2.4.0", "target": "icu_collections" }, { - "id": "icu_normalizer_data 2.3.0", + "id": "icu_normalizer_data 2.4.0", "target": "icu_normalizer_data" }, { - "id": "icu_provider 2.3.1", + "id": "icu_provider 2.4.0", "target": "icu_provider" }, { - "id": "smallvec 1.16.3", + "id": "smallvec 1.16.4", "target": "smallvec" }, { - "id": "zerovec 0.11.8", + "id": "zerovec 0.11.9", "target": "zerovec" } ], @@ -5249,8 +5233,8 @@ "package_url": "https://github.com/unicode-org/icu4x", "repository": { "Http": { - "sha256": "12f9cf5f235641ed274641dd81c3f28d870e276763d0797aeeab72317b1c646f", - "url": "https://static.crates.io/crates/icu_normalizer/2.3.0/download" + "sha256": "404a3b8c486f205c87fe7f23ab6894d84a4ad76d6e4671ae1981ba7def87c599", + "url": "https://static.crates.io/crates/icu_normalizer/2.4.0/download" } }, "targets": [ @@ -5261,9 +5245,9 @@ } } ], - "version": "2.3.0" + "version": "2.4.0" }, - "icu_normalizer_data 2.3.0": { + "icu_normalizer_data 2.4.0": { "build_script_attrs": {}, "common_attrs": { "edition": "2024", @@ -5281,8 +5265,8 @@ "package_url": "https://github.com/unicode-org/icu4x", "repository": { "Http": { - "sha256": "1563da1ed3e0b3bf3d74c9b85917ac9c56464d2f57242270c09c9e752f8021a0", - "url": "https://static.crates.io/crates/icu_normalizer_data/2.3.0/download" + "sha256": "20f0088bfb7441f7a37fa0c9e20cb1e9cba1cac6b9ce8cd2a6dc44294f8e8cbc", + "url": "https://static.crates.io/crates/icu_normalizer_data/2.4.0/download" } }, "targets": [ @@ -5299,28 +5283,28 @@ } } ], - "version": "2.3.0" + "version": "2.4.0" }, - "icu_properties 2.3.0": { + "icu_properties 2.4.0": { "common_attrs": { "crate_features": [ "compiled_data" ], "deps": [ { - "id": "icu_collections 2.3.0", + "id": "icu_collections 2.4.0", "target": "icu_collections" }, { - "id": "icu_locale_core 2.3.0", + "id": "icu_locale_core 2.4.0", "target": "icu_locale_core" }, { - "id": "icu_properties_data 2.3.0", + "id": "icu_properties_data 2.4.0", "target": "icu_properties_data" }, { - "id": "icu_provider 2.3.1", + "id": "icu_provider 2.4.0", "target": "icu_provider" }, { @@ -5328,7 +5312,7 @@ "target": "zerotrie" }, { - "id": "zerovec 0.11.8", + "id": "zerovec 0.11.9", "target": "zerovec" } ], @@ -5350,8 +5334,8 @@ "package_url": "https://github.com/unicode-org/icu4x", "repository": { "Http": { - "sha256": "7e7ca276ad3145661a65914e6daf131ca5120cd3dcee8f8f3214b8875184a148", - "url": "https://static.crates.io/crates/icu_properties/2.3.0/download" + "sha256": "2ea83d991c56158f7cf898bf86436c1dfe7c07ee82fe88951f2695c9f37a379f", + "url": "https://static.crates.io/crates/icu_properties/2.4.0/download" } }, "targets": [ @@ -5362,9 +5346,9 @@ } } ], - "version": "2.3.0" + "version": "2.4.0" }, - "icu_properties_data 2.3.0": { + "icu_properties_data 2.4.0": { "build_script_attrs": {}, "common_attrs": { "edition": "2024", @@ -5382,8 +5366,8 @@ "package_url": "https://github.com/unicode-org/icu4x", "repository": { "Http": { - "sha256": "e590f038c1464a96894fd6d10127e90a8be4509f56ff7ecef851b15cee0b7caa", - "url": "https://static.crates.io/crates/icu_properties_data/2.3.0/download" + "sha256": "b98f29af82d02b163fe50161388350e743bb3107cc8e8c2f7314cb505faddfcf", + "url": "https://static.crates.io/crates/icu_properties_data/2.4.0/download" } }, "targets": [ @@ -5400,16 +5384,16 @@ } } ], - "version": "2.3.0" + "version": "2.4.0" }, - "icu_provider 2.3.1": { + "icu_provider 2.4.0": { "common_attrs": { "crate_features": [ "baked" ], "deps": [ { - "id": "icu_locale_core 2.3.0", + "id": "icu_locale_core 2.4.0", "target": "icu_locale_core" }, { @@ -5417,7 +5401,7 @@ "target": "writeable" }, { - "id": "yoke 0.8.3", + "id": "yoke 0.8.4", "target": "yoke" }, { @@ -5429,7 +5413,7 @@ "target": "zerotrie" }, { - "id": "zerovec 0.11.8", + "id": "zerovec 0.11.9", "target": "zerovec" } ], @@ -5451,8 +5435,8 @@ "package_url": "https://github.com/unicode-org/icu4x", "repository": { "Http": { - "sha256": "d27bbb9d3abbefac45d55f647c9de1d44aafcd1186eb91879afef17c396c3e73", - "url": "https://static.crates.io/crates/icu_provider/2.3.1/download" + "sha256": "413099315636c588ee8f011190b329b971a09a8fae9d157e01dcae8379468514", + "url": "https://static.crates.io/crates/icu_provider/2.4.0/download" } }, "targets": [ @@ -5463,7 +5447,7 @@ } } ], - "version": "2.3.1" + "version": "2.4.0" }, "idna 1.1.0": { "common_attrs": { @@ -5478,7 +5462,7 @@ "target": "idna_adapter" }, { - "id": "smallvec 1.16.3", + "id": "smallvec 1.16.4", "target": "smallvec" }, { @@ -5520,11 +5504,11 @@ ], "deps": [ { - "id": "icu_normalizer 2.3.0", + "id": "icu_normalizer 2.4.0", "target": "icu_normalizer" }, { - "id": "icu_properties 2.3.0", + "id": "icu_properties 2.4.0", "target": "icu_properties" } ], @@ -6484,7 +6468,7 @@ ], "version": "0.4.34" }, - "lol_html 3.0.1": { + "lol_html 3.0.2": { "common_attrs": { "deps": [ { @@ -6545,7 +6529,7 @@ "repository": { "Git": { "commitish": { - "Rev": "608cc4a66b7ab4fcbe1bbdeb25df8f265572b11c" + "Rev": "c2cfb7b086ccaca10308365a3ed7205cba98608b" }, "remote": "https://github.com/cloudflare/lol-html" } @@ -6558,7 +6542,7 @@ } } ], - "version": "3.0.1" + "version": "3.0.2" }, "lol_html_c_api 1.4.0": { "build_script_attrs": { @@ -6579,7 +6563,7 @@ "target": "libc" }, { - "id": "lol_html 3.0.1", + "id": "lol_html 3.0.2", "target": "lol_html" }, { @@ -6600,7 +6584,7 @@ "repository": { "Git": { "commitish": { - "Rev": "608cc4a66b7ab4fcbe1bbdeb25df8f265572b11c" + "Rev": "c2cfb7b086ccaca10308365a3ed7205cba98608b" }, "remote": "https://github.com/cloudflare/lol-html", "strip_prefix": "c-api" @@ -6622,6 +6606,46 @@ ], "version": "1.4.0" }, + "lz4_flex 0.11.6": { + "common_attrs": { + "crate_features": [ + "frame", + "safe-decode", + "safe-encode", + "std" + ], + "deps": [ + { + "id": "twox-hash 2.1.5", + "target": "twox_hash" + } + ], + "edition": "2021" + }, + "library_target_name": "lz4_flex", + "license": "MIT", + "license_file": "LICENSE", + "license_ids": [ + "MIT" + ], + "name": "lz4_flex", + "package_url": "https://github.com/pseitz/lz4_flex", + "repository": { + "Http": { + "sha256": "373f5eceeeab7925e0c1098212f2fbc4d416adec9d35051a6ab251e824c1854a", + "url": "https://static.crates.io/crates/lz4_flex/0.11.6/download" + } + }, + "targets": [ + { + "Library": { + "crate_name": "lz4_flex", + "crate_root": "src/lib.rs" + } + } + ], + "version": "0.11.6" + }, "memchr 2.8.3": { "common_attrs": { "crate_features": [ @@ -6933,12 +6957,14 @@ ], "version": "0.31.3" }, - "notify 8.2.0": { + "notify 8.2.1": { "common_attrs": { "crate_features": [ "default", "fsevent-sys", - "macos_fsevent" + "macos_fsevent", + "objc2-core-foundation", + "objc2-core-services" ], "deps": { "common": [ @@ -6962,8 +6988,12 @@ "selects": { "aarch64-apple-darwin": [ { - "id": "fsevent-sys 4.1.0", - "target": "fsevent_sys" + "id": "objc2-core-foundation 0.3.2", + "target": "objc2_core_foundation" + }, + { + "id": "objc2-core-services 0.3.2", + "target": "objc2_core_services" } ], "cfg(any(target_os = \"freebsd\", target_os = \"openbsd\", target_os = \"netbsd\", target_os = \"dragonflybsd\", target_os = \"ios\"))": [ @@ -6994,14 +7024,18 @@ ], "cfg(windows)": [ { - "id": "windows-sys 0.60.2", + "id": "windows-sys 0.61.2", "target": "windows_sys" } ], "x86_64-apple-darwin": [ { - "id": "fsevent-sys 4.1.0", - "target": "fsevent_sys" + "id": "objc2-core-foundation 0.3.2", + "target": "objc2_core_foundation" + }, + { + "id": "objc2-core-services 0.3.2", + "target": "objc2_core_services" } ] } @@ -7018,8 +7052,8 @@ "package_url": "https://github.com/notify-rs/notify.git", "repository": { "Http": { - "sha256": "4d3d07927151ff8575b7087f245456e549fea62edf0ec4e565a5ee50c8402bc3", - "url": "https://static.crates.io/crates/notify/8.2.0/download" + "sha256": "82f812702fbc75d35416d3f884e04925be7e9498f4eceea2e24ba34559284674", + "url": "https://static.crates.io/crates/notify/8.2.1/download" } }, "targets": [ @@ -7030,7 +7064,7 @@ } } ], - "version": "8.2.0" + "version": "8.2.1" }, "notify-types 2.1.0": { "common_attrs": { @@ -7250,6 +7284,98 @@ ], "version": "1.17.0" }, + "objc2-core-foundation 0.3.2": { + "common_attrs": { + "crate_features": [ + "CFArray", + "CFDate", + "CFError", + "CFRunLoop", + "CFString", + "CFURL", + "CFUUID", + "alloc", + "bitflags", + "std" + ], + "deps": [ + { + "id": "bitflags 2.13.2", + "target": "bitflags" + } + ], + "edition": "2021" + }, + "library_target_name": "objc2_core_foundation", + "license": "Zlib OR Apache-2.0 OR MIT", + "license_ids": [ + "Apache-2.0", + "MIT", + "Zlib" + ], + "name": "objc2-core-foundation", + "package_url": "https://github.com/madsmtm/objc2", + "repository": { + "Http": { + "sha256": "2a180dd8642fa45cdb7dd721cd4c11b1cadd4929ce112ebd8b9f5803cc79d536", + "url": "https://static.crates.io/crates/objc2-core-foundation/0.3.2/download" + } + }, + "targets": [ + { + "Library": { + "crate_name": "objc2_core_foundation", + "crate_root": "src/lib.rs" + } + } + ], + "version": "0.3.2" + }, + "objc2-core-services 0.3.2": { + "common_attrs": { + "crate_features": [ + "FSEvents", + "alloc", + "libc", + "std" + ], + "deps": [ + { + "id": "libc 0.2.191", + "target": "libc" + }, + { + "id": "objc2-core-foundation 0.3.2", + "target": "objc2_core_foundation" + } + ], + "edition": "2021" + }, + "library_target_name": "objc2_core_services", + "license": "Zlib OR Apache-2.0 OR MIT", + "license_ids": [ + "Apache-2.0", + "MIT", + "Zlib" + ], + "name": "objc2-core-services", + "package_url": "https://github.com/madsmtm/objc2", + "repository": { + "Http": { + "sha256": "583300ad934cba24ff5292aee751ecc070f7ca6b39a574cc21b7b5e588e06a0b", + "url": "https://static.crates.io/crates/objc2-core-services/0.3.2/download" + } + }, + "targets": [ + { + "Library": { + "crate_name": "objc2_core_services", + "crate_root": "src/lib.rs" + } + } + ], + "version": "0.3.2" + }, "once_cell 1.21.4": { "common_attrs": { "crate_features": [ @@ -7285,7 +7411,7 @@ ], "version": "1.21.4" }, - "openssl-probe 0.2.1": { + "openssl-probe 0.2.2": { "common_attrs": { "edition": "2021" }, @@ -7300,8 +7426,8 @@ "package_url": "https://github.com/rustls/openssl-probe", "repository": { "Http": { - "sha256": "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe", - "url": "https://static.crates.io/crates/openssl-probe/0.2.1/download" + "sha256": "7e036b411754132fd6b74e94b74e9a13c5cf606fa93d1885c35860d3060820b1", + "url": "https://static.crates.io/crates/openssl-probe/0.2.2/download" } }, "targets": [ @@ -7312,7 +7438,7 @@ } } ], - "version": "0.2.1" + "version": "0.2.2" }, "outref 0.5.2": { "common_attrs": { @@ -7935,7 +8061,7 @@ ], "deps": [ { - "id": "zerovec 0.11.8", + "id": "zerovec 0.11.9", "target": "zerovec" } ], @@ -8692,7 +8818,7 @@ "build_script_attrs": { "deps": [ { - "id": "cc 1.9.0", + "id": "cc 1.10.0", "target": "cc" } ], @@ -9328,7 +9454,7 @@ "selects": { "cfg(all(unix, not(target_os = \"macos\")))": [ { - "id": "openssl-probe 0.2.1", + "id": "openssl-probe 0.2.2", "target": "openssl_probe" } ], @@ -9462,7 +9588,7 @@ "target": "webpki" }, { - "id": "webpki-root-certs 1.0.10", + "id": "webpki-root-certs 1.0.11", "target": "webpki_root_certs" } ], @@ -9943,7 +10069,7 @@ "target": "servo_arc" }, { - "id": "smallvec 1.16.3", + "id": "smallvec 1.16.4", "target": "smallvec" } ], @@ -10686,7 +10812,7 @@ ], "version": "0.4.12" }, - "smallvec 1.16.3": { + "smallvec 1.16.4": { "common_attrs": { "crate_features": [ "const_generics", @@ -10705,8 +10831,8 @@ "package_url": "https://github.com/servo/rust-smallvec", "repository": { "Http": { - "sha256": "5b3dc8af474f516a851ff4bd12db780f948b9250ad37211e4eec0bccea54e01b", - "url": "https://static.crates.io/crates/smallvec/1.16.3/download" + "sha256": "d0d5d4b1981b8bbbb7aed204cae5726d683d5d6ac7e74ec54122ea28f315e7c8", + "url": "https://static.crates.io/crates/smallvec/1.16.4/download" } }, "targets": [ @@ -10717,7 +10843,7 @@ } } ], - "version": "1.16.3" + "version": "1.16.4" }, "socket2 0.6.5": { "common_attrs": { @@ -12553,14 +12679,14 @@ ], "version": "2.0.21" }, - "tinystr 0.8.4": { + "tinystr 0.8.5": { "common_attrs": { "crate_features": [ "zerovec" ], "deps": [ { - "id": "zerovec 0.11.8", + "id": "zerovec 0.11.9", "target": "zerovec" } ], @@ -12582,8 +12708,8 @@ "package_url": "https://github.com/unicode-org/icu4x", "repository": { "Http": { - "sha256": "b1e27c91459209c2986af3dcf603a5a74a4368754ce37414f59acc971167f643", - "url": "https://static.crates.io/crates/tinystr/0.8.4/download" + "sha256": "abc840c238154d58f9e86b2792261edfde565a17696e1cda1a449de27979801a", + "url": "https://static.crates.io/crates/tinystr/0.8.5/download" } }, "targets": [ @@ -12594,7 +12720,7 @@ } } ], - "version": "0.8.4" + "version": "0.8.5" }, "tinyvec 1.13.3": { "common_attrs": { @@ -12630,7 +12756,7 @@ ], "version": "1.13.3" }, - "tokio 1.54.0": { + "tokio 1.54.1": { "common_attrs": { "crate_features": { "common": [ @@ -12734,8 +12860,8 @@ "package_url": "https://github.com/tokio-rs/tokio", "repository": { "Http": { - "sha256": "a0c168a474b95bfecfe908a65d4bdffc523144b649a74c09c286afb35d53e0bc", - "url": "https://static.crates.io/crates/tokio/1.54.0/download" + "sha256": "b52fd3255258d26fac7264ab5a76b53c13367136dc011d3d6ad4f8ed070396a3", + "url": "https://static.crates.io/crates/tokio/1.54.1/download" } }, "targets": [ @@ -12746,7 +12872,7 @@ } } ], - "version": "1.54.0" + "version": "1.54.1" }, "tokio-rustls 0.26.6": { "common_attrs": { @@ -12760,7 +12886,7 @@ "target": "rustls" }, { - "id": "tokio 1.54.0", + "id": "tokio 1.54.1", "target": "tokio" } ], @@ -13022,6 +13148,37 @@ ], "version": "0.2.5" }, + "twox-hash 2.1.5": { + "common_attrs": { + "crate_features": [ + "xxhash32" + ], + "edition": "2021" + }, + "library_target_name": "twox_hash", + "license": "MIT", + "license_file": "LICENSE.txt", + "license_ids": [ + "MIT" + ], + "name": "twox-hash", + "package_url": "https://github.com/shepmaster/twox-hash", + "repository": { + "Http": { + "sha256": "86a801b3cea342a06d468c8710662aa29e5e05e4f5c0d62f00bbb7f2ad7941c2", + "url": "https://static.crates.io/crates/twox-hash/2.1.5/download" + } + }, + "targets": [ + { + "Library": { + "crate_name": "twox_hash", + "crate_root": "src/lib.rs" + } + } + ], + "version": "2.1.5" + }, "typenum 1.20.1": { "common_attrs": { "crate_features": [ @@ -13397,7 +13554,7 @@ ], "version": "1.0.4" }, - "uuid 1.28.0": { + "uuid 1.29.0": { "common_attrs": { "crate_features": [ "default", @@ -13416,8 +13573,8 @@ "package_url": "https://github.com/uuid-rs/uuid", "repository": { "Http": { - "sha256": "7cc1186384beb7dd8eedea376413fd654937285ea6c9cfbb928dc3043ea4b606", - "url": "https://static.crates.io/crates/uuid/1.28.0/download" + "sha256": "4a0e2d11620c50695fe12bbbfde96b3477fe606e7284c22d505c12ce51214514", + "url": "https://static.crates.io/crates/uuid/1.29.0/download" } }, "targets": [ @@ -13428,7 +13585,7 @@ } } ], - "version": "1.28.0" + "version": "1.29.0" }, "version_check 0.9.5": { "common_attrs": { @@ -13844,7 +14001,7 @@ ], "version": "0.2.129" }, - "webpki-root-certs 1.0.10": { + "webpki-root-certs 1.0.11": { "common_attrs": { "deps": [ { @@ -13865,8 +14022,8 @@ "package_url": "https://github.com/rustls/webpki-roots", "repository": { "Http": { - "sha256": "58ec80e73fd8baf57bf0d9615554c75348bda990d6d5bbeab964dd068319fecb", - "url": "https://static.crates.io/crates/webpki-root-certs/1.0.10/download" + "sha256": "e49d0e65abc00c142e8f9428da991da9f1a250c39be97459901eb12b9b49f62a", + "url": "https://static.crates.io/crates/webpki-root-certs/1.0.11/download" } }, "targets": [ @@ -13877,7 +14034,7 @@ } } ], - "version": "1.0.10" + "version": "1.0.11" }, "winapi-util 0.1.11": { "common_attrs": { @@ -13983,53 +14140,6 @@ ], "version": "0.52.0" }, - "windows-sys 0.60.2": { - "common_attrs": { - "crate_features": [ - "Win32", - "Win32_Foundation", - "Win32_Security", - "Win32_Storage", - "Win32_Storage_FileSystem", - "Win32_System", - "Win32_System_IO", - "Win32_System_Threading", - "Win32_System_WindowsProgramming", - "default" - ], - "deps": [ - { - "id": "windows-targets 0.53.5", - "target": "windows_targets" - } - ], - "edition": "2021" - }, - "library_target_name": "windows_sys", - "license": "MIT OR Apache-2.0", - "license_file": "license-apache-2.0", - "license_ids": [ - "Apache-2.0", - "MIT" - ], - "name": "windows-sys", - "package_url": "https://github.com/microsoft/windows-rs", - "repository": { - "Http": { - "sha256": "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb", - "url": "https://static.crates.io/crates/windows-sys/0.60.2/download" - } - }, - "targets": [ - { - "Library": { - "crate_name": "windows_sys", - "crate_root": "src/lib.rs" - } - } - ], - "version": "0.60.2" - }, "windows-sys 0.61.2": { "common_attrs": { "crate_features": { @@ -14178,96 +14288,8 @@ ], "version": "0.52.6" }, - "windows-targets 0.53.5": { - "common_attrs": { - "deps": { - "common": [], - "selects": { - "aarch64-pc-windows-gnullvm": [ - { - "id": "windows_aarch64_gnullvm 0.53.1", - "target": "windows_aarch64_gnullvm" - } - ], - "cfg(all(any(target_arch = \"x86_64\", target_arch = \"arm64ec\"), target_env = \"msvc\", not(windows_raw_dylib)))": [ - { - "id": "windows_x86_64_msvc 0.53.1", - "target": "windows_x86_64_msvc" - } - ], - "cfg(all(target_arch = \"aarch64\", target_env = \"msvc\", not(windows_raw_dylib)))": [ - { - "id": "windows_aarch64_msvc 0.53.1", - "target": "windows_aarch64_msvc" - } - ], - "cfg(all(target_arch = \"x86\", target_env = \"gnu\", not(target_abi = \"llvm\"), not(windows_raw_dylib)))": [ - { - "id": "windows_i686_gnu 0.53.1", - "target": "windows_i686_gnu" - } - ], - "cfg(all(target_arch = \"x86\", target_env = \"msvc\", not(windows_raw_dylib)))": [ - { - "id": "windows_i686_msvc 0.53.1", - "target": "windows_i686_msvc" - } - ], - "cfg(all(target_arch = \"x86_64\", target_env = \"gnu\", not(target_abi = \"llvm\"), not(windows_raw_dylib)))": [ - { - "id": "windows_x86_64_gnu 0.53.1", - "target": "windows_x86_64_gnu" - } - ], - "cfg(windows_raw_dylib)": [ - { - "id": "windows-link 0.2.1", - "target": "windows_link" - } - ], - "i686-pc-windows-gnullvm": [ - { - "id": "windows_i686_gnullvm 0.53.1", - "target": "windows_i686_gnullvm" - } - ], - "x86_64-pc-windows-gnullvm": [ - { - "id": "windows_x86_64_gnullvm 0.53.1", - "target": "windows_x86_64_gnullvm" - } - ] - } - }, - "edition": "2021" - }, - "library_target_name": "windows_targets", - "license": "MIT OR Apache-2.0", - "license_file": "license-apache-2.0", - "license_ids": [ - "Apache-2.0", - "MIT" - ], - "name": "windows-targets", - "package_url": "https://github.com/microsoft/windows-rs", - "repository": { - "Http": { - "sha256": "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3", - "url": "https://static.crates.io/crates/windows-targets/0.53.5/download" - } - }, - "targets": [ - { - "Library": { - "crate_name": "windows_targets", - "crate_root": "src/lib.rs" - } - } - ], - "version": "0.53.5" - }, - "windows_aarch64_gnullvm 0.52.6": { - "build_script_attrs": {}, + "windows_aarch64_gnullvm 0.52.6": { + "build_script_attrs": {}, "common_attrs": { "edition": "2021", "extra_deps": [ @@ -14305,45 +14327,6 @@ ], "version": "0.52.6" }, - "windows_aarch64_gnullvm 0.53.1": { - "build_script_attrs": {}, - "common_attrs": { - "edition": "2021", - "extra_deps": [ - ":build_script_build" - ] - }, - "library_target_name": "windows_aarch64_gnullvm", - "license": "MIT OR Apache-2.0", - "license_file": "license-apache-2.0", - "license_ids": [ - "Apache-2.0", - "MIT" - ], - "name": "windows_aarch64_gnullvm", - "package_url": "https://github.com/microsoft/windows-rs", - "repository": { - "Http": { - "sha256": "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53", - "url": "https://static.crates.io/crates/windows_aarch64_gnullvm/0.53.1/download" - } - }, - "targets": [ - { - "Library": { - "crate_name": "windows_aarch64_gnullvm", - "crate_root": "src/lib.rs" - } - }, - { - "BuildScript": { - "crate_name": "build_script_build", - "crate_root": "build.rs" - } - } - ], - "version": "0.53.1" - }, "windows_aarch64_msvc 0.52.6": { "build_script_attrs": {}, "common_attrs": { @@ -14383,45 +14366,6 @@ ], "version": "0.52.6" }, - "windows_aarch64_msvc 0.53.1": { - "build_script_attrs": {}, - "common_attrs": { - "edition": "2021", - "extra_deps": [ - ":build_script_build" - ] - }, - "library_target_name": "windows_aarch64_msvc", - "license": "MIT OR Apache-2.0", - "license_file": "license-apache-2.0", - "license_ids": [ - "Apache-2.0", - "MIT" - ], - "name": "windows_aarch64_msvc", - "package_url": "https://github.com/microsoft/windows-rs", - "repository": { - "Http": { - "sha256": "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006", - "url": "https://static.crates.io/crates/windows_aarch64_msvc/0.53.1/download" - } - }, - "targets": [ - { - "Library": { - "crate_name": "windows_aarch64_msvc", - "crate_root": "src/lib.rs" - } - }, - { - "BuildScript": { - "crate_name": "build_script_build", - "crate_root": "build.rs" - } - } - ], - "version": "0.53.1" - }, "windows_i686_gnu 0.52.6": { "build_script_attrs": {}, "common_attrs": { @@ -14461,45 +14405,6 @@ ], "version": "0.52.6" }, - "windows_i686_gnu 0.53.1": { - "build_script_attrs": {}, - "common_attrs": { - "edition": "2021", - "extra_deps": [ - ":build_script_build" - ] - }, - "library_target_name": "windows_i686_gnu", - "license": "MIT OR Apache-2.0", - "license_file": "license-apache-2.0", - "license_ids": [ - "Apache-2.0", - "MIT" - ], - "name": "windows_i686_gnu", - "package_url": "https://github.com/microsoft/windows-rs", - "repository": { - "Http": { - "sha256": "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3", - "url": "https://static.crates.io/crates/windows_i686_gnu/0.53.1/download" - } - }, - "targets": [ - { - "Library": { - "crate_name": "windows_i686_gnu", - "crate_root": "src/lib.rs" - } - }, - { - "BuildScript": { - "crate_name": "build_script_build", - "crate_root": "build.rs" - } - } - ], - "version": "0.53.1" - }, "windows_i686_gnullvm 0.52.6": { "build_script_attrs": {}, "common_attrs": { @@ -14539,45 +14444,6 @@ ], "version": "0.52.6" }, - "windows_i686_gnullvm 0.53.1": { - "build_script_attrs": {}, - "common_attrs": { - "edition": "2021", - "extra_deps": [ - ":build_script_build" - ] - }, - "library_target_name": "windows_i686_gnullvm", - "license": "MIT OR Apache-2.0", - "license_file": "license-apache-2.0", - "license_ids": [ - "Apache-2.0", - "MIT" - ], - "name": "windows_i686_gnullvm", - "package_url": "https://github.com/microsoft/windows-rs", - "repository": { - "Http": { - "sha256": "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c", - "url": "https://static.crates.io/crates/windows_i686_gnullvm/0.53.1/download" - } - }, - "targets": [ - { - "Library": { - "crate_name": "windows_i686_gnullvm", - "crate_root": "src/lib.rs" - } - }, - { - "BuildScript": { - "crate_name": "build_script_build", - "crate_root": "build.rs" - } - } - ], - "version": "0.53.1" - }, "windows_i686_msvc 0.52.6": { "build_script_attrs": {}, "common_attrs": { @@ -14617,45 +14483,6 @@ ], "version": "0.52.6" }, - "windows_i686_msvc 0.53.1": { - "build_script_attrs": {}, - "common_attrs": { - "edition": "2021", - "extra_deps": [ - ":build_script_build" - ] - }, - "library_target_name": "windows_i686_msvc", - "license": "MIT OR Apache-2.0", - "license_file": "license-apache-2.0", - "license_ids": [ - "Apache-2.0", - "MIT" - ], - "name": "windows_i686_msvc", - "package_url": "https://github.com/microsoft/windows-rs", - "repository": { - "Http": { - "sha256": "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2", - "url": "https://static.crates.io/crates/windows_i686_msvc/0.53.1/download" - } - }, - "targets": [ - { - "Library": { - "crate_name": "windows_i686_msvc", - "crate_root": "src/lib.rs" - } - }, - { - "BuildScript": { - "crate_name": "build_script_build", - "crate_root": "build.rs" - } - } - ], - "version": "0.53.1" - }, "windows_x86_64_gnu 0.52.6": { "build_script_attrs": {}, "common_attrs": { @@ -14695,45 +14522,6 @@ ], "version": "0.52.6" }, - "windows_x86_64_gnu 0.53.1": { - "build_script_attrs": {}, - "common_attrs": { - "edition": "2021", - "extra_deps": [ - ":build_script_build" - ] - }, - "library_target_name": "windows_x86_64_gnu", - "license": "MIT OR Apache-2.0", - "license_file": "license-apache-2.0", - "license_ids": [ - "Apache-2.0", - "MIT" - ], - "name": "windows_x86_64_gnu", - "package_url": "https://github.com/microsoft/windows-rs", - "repository": { - "Http": { - "sha256": "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499", - "url": "https://static.crates.io/crates/windows_x86_64_gnu/0.53.1/download" - } - }, - "targets": [ - { - "Library": { - "crate_name": "windows_x86_64_gnu", - "crate_root": "src/lib.rs" - } - }, - { - "BuildScript": { - "crate_name": "build_script_build", - "crate_root": "build.rs" - } - } - ], - "version": "0.53.1" - }, "windows_x86_64_gnullvm 0.52.6": { "build_script_attrs": {}, "common_attrs": { @@ -14773,45 +14561,6 @@ ], "version": "0.52.6" }, - "windows_x86_64_gnullvm 0.53.1": { - "build_script_attrs": {}, - "common_attrs": { - "edition": "2021", - "extra_deps": [ - ":build_script_build" - ] - }, - "library_target_name": "windows_x86_64_gnullvm", - "license": "MIT OR Apache-2.0", - "license_file": "license-apache-2.0", - "license_ids": [ - "Apache-2.0", - "MIT" - ], - "name": "windows_x86_64_gnullvm", - "package_url": "https://github.com/microsoft/windows-rs", - "repository": { - "Http": { - "sha256": "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1", - "url": "https://static.crates.io/crates/windows_x86_64_gnullvm/0.53.1/download" - } - }, - "targets": [ - { - "Library": { - "crate_name": "windows_x86_64_gnullvm", - "crate_root": "src/lib.rs" - } - }, - { - "BuildScript": { - "crate_name": "build_script_build", - "crate_root": "build.rs" - } - } - ], - "version": "0.53.1" - }, "windows_x86_64_msvc 0.52.6": { "build_script_attrs": {}, "common_attrs": { @@ -14851,45 +14600,6 @@ ], "version": "0.52.6" }, - "windows_x86_64_msvc 0.53.1": { - "build_script_attrs": {}, - "common_attrs": { - "edition": "2021", - "extra_deps": [ - ":build_script_build" - ] - }, - "library_target_name": "windows_x86_64_msvc", - "license": "MIT OR Apache-2.0", - "license_file": "license-apache-2.0", - "license_ids": [ - "Apache-2.0", - "MIT" - ], - "name": "windows_x86_64_msvc", - "package_url": "https://github.com/microsoft/windows-rs", - "repository": { - "Http": { - "sha256": "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650", - "url": "https://static.crates.io/crates/windows_x86_64_msvc/0.53.1/download" - } - }, - "targets": [ - { - "Library": { - "crate_name": "windows_x86_64_msvc", - "crate_root": "src/lib.rs" - } - }, - { - "BuildScript": { - "crate_name": "build_script_build", - "crate_root": "build.rs" - } - } - ], - "version": "0.53.1" - }, "wit-bindgen 0.57.1": { "build_script_attrs": {}, "common_attrs": { @@ -15034,7 +14744,7 @@ ], "version": "0.2.5" }, - "yoke 0.8.3": { + "yoke 0.8.4": { "common_attrs": { "crate_features": [ "derive", @@ -15068,8 +14778,8 @@ "package_url": "https://github.com/unicode-org/icu4x", "repository": { "Http": { - "sha256": "709fe23a0424b6a435d82152b1bd3fdfb0833487d5fa90d05d42762a9891fef5", - "url": "https://static.crates.io/crates/yoke/0.8.3/download" + "sha256": "417ef3fcf6c300b7dbd25c85531a746931c02ad536f3cf4786ed1e88223340b3", + "url": "https://static.crates.io/crates/yoke/0.8.4/download" } }, "targets": [ @@ -15080,7 +14790,7 @@ } } ], - "version": "0.8.3" + "version": "0.8.4" }, "yoke-derive 0.8.4": { "common_attrs": { @@ -15350,7 +15060,7 @@ ], "deps": [ { - "id": "yoke 0.8.3", + "id": "yoke 0.8.4", "target": "yoke" }, { @@ -15390,7 +15100,7 @@ ], "version": "0.2.5" }, - "zerovec 0.11.8": { + "zerovec 0.11.9": { "common_attrs": { "crate_features": [ "derive", @@ -15398,7 +15108,7 @@ ], "deps": [ { - "id": "yoke 0.8.3", + "id": "yoke 0.8.4", "target": "yoke" }, { @@ -15409,7 +15119,7 @@ "edition": "2021", "proc_macro_deps": [ { - "id": "zerovec-derive 0.11.6", + "id": "zerovec-derive 0.11.7", "target": "zerovec_derive" } ] @@ -15424,8 +15134,8 @@ "package_url": "https://github.com/unicode-org/icu4x", "repository": { "Http": { - "sha256": "bb0464e17806c1d976d5cba29399c7f08e516e279e2ba493f63123b5fca67dd8", - "url": "https://static.crates.io/crates/zerovec/0.11.8/download" + "sha256": "f58a2799cfe9a1670c7ea8edbd52ef8edb93cb37a7f83bf31ba9e35d9dae6a40", + "url": "https://static.crates.io/crates/zerovec/0.11.9/download" } }, "targets": [ @@ -15436,9 +15146,9 @@ } } ], - "version": "0.11.8" + "version": "0.11.9" }, - "zerovec-derive 0.11.6": { + "zerovec-derive 0.11.7": { "common_attrs": { "deps": [ { @@ -15466,8 +15176,8 @@ "package_url": "https://github.com/unicode-org/icu4x", "repository": { "Http": { - "sha256": "34df6fc39dbd26ddc9c10e6a2984476e13acce22e64e4487636ef494369225da", - "url": "https://static.crates.io/crates/zerovec-derive/0.11.6/download" + "sha256": "c3743ddbf47738d929cd1ae2e199d40d0162979dfe31aef8ae646d2bf00bf97a", + "url": "https://static.crates.io/crates/zerovec-derive/0.11.7/download" } }, "targets": [ @@ -15478,7 +15188,7 @@ } } ], - "version": "0.11.6" + "version": "0.11.7" }, "zlib-rs 0.6.8": { "common_attrs": { @@ -15559,9 +15269,9 @@ "base64 0.23.1", "bytes 1.12.1", "capnp 0.27.2", - "capnp-rpc 0.27.1", + "capnp-rpc 0.27.2", "capnpc 0.27.0", - "cc 1.9.0", + "cc 1.10.0", "clang-ast 0.1.35", "clap 4.6.7", "codespan-reporting 0.13.1", @@ -15583,13 +15293,15 @@ "libm 0.2.16", "libz-rs-sys 0.6.8", "lol_html_c_api 1.4.0", + "lz4_flex 0.11.6", "nix 0.31.3", - "notify 8.2.0", + "notify 8.2.1", "percent-encoding 2.3.2", "pico-args 0.5.0", "proc-macro2 1.0.107", "quote 1.0.47", "rand 0.9.5", + "ring 0.17.14", "ruff_python_ast 0.0.0", "ruff_python_parser 0.0.0", "rust_icu_sys 5.8.0", @@ -15606,7 +15318,7 @@ "swc_ts_fast_strip 57.0.0", "syn 2.0.119", "thiserror 2.0.21", - "tokio 1.54.0", + "tokio 1.54.1", "tokio-rustls 0.26.6", "tower-service 0.3.3", "tracing 0.1.44", From 391ae4558b4a08956579c524d231d0e5710f123f Mon Sep 17 00:00:00 2001 From: Ryan Dahl Date: Tue, 27 Oct 2026 11:07:03 -0700 Subject: [PATCH 11/38] cluster: drop the unused facet index from own.json; document both records Nothing set or read `facets`, so own.json always carried `"facets":[]`. Facets come in their own change, which can add the field back. Each record's fields now say what they hold, and NodeRecord's `node` and `address` say what they are: a random id and `host:port`, not a public key and sockaddr bytes. --- src/workerd/server/cluster/ownership.rs | 58 +++---------------------- src/workerd/server/cluster/registry.rs | 4 +- 2 files changed, 8 insertions(+), 54 deletions(-) diff --git a/src/workerd/server/cluster/ownership.rs b/src/workerd/server/cluster/ownership.rs index 2143d93c68b..2bfc86491b6 100644 --- a/src/workerd/server/cluster/ownership.rs +++ b/src/workerd/server/cluster/ownership.rs @@ -19,22 +19,17 @@ use crate::registry::NodeRecord; /// One Durable Object's record in `do///own.json`, the only /// authority for who owns it and whose log holds its latest commits. -/// -/// `from` is the last epoch that opened. A claim writes the next `epoch` -/// and copies `from` from the record it replaced; the opening write sets -/// `from` to `epoch`. So an epoch is recovering while `from != epoch`, and -/// recovers `from`'s log (none when 0). -/// -/// `facets` is the object's facet tree index, `(parent, name)` by id from 1: -/// each facet's database is a log of its own under the object's epoch. -/// Every epoch carries the index forward, and an owner adds to it before a -/// new facet's log opens. #[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] struct OwnerRecord { + /// The owning instance (`NodeRecord::node`); empty for none. node: String, + /// The owner's epoch: its log is `log/e/`. epoch: u64, + /// The last epoch that opened. A claim writes the next `epoch` and + /// copies `from` from the record it replaced; the opening write sets it + /// to `epoch`. So an epoch is recovering while `from != epoch`, and + /// recovers `from`'s log (none when 0). from: u64, - facets: Vec<(u32, String)>, } impl OwnerRecord { @@ -45,7 +40,6 @@ impl OwnerRecord { node: node.to_owned(), epoch: self.epoch + 1, from: self.from, - facets: self.facets.clone(), } } @@ -53,7 +47,6 @@ impl OwnerRecord { Owned { recover: (self.from != self.epoch).then_some(self.from), epoch: self.epoch, - facets: self.facets, } } } @@ -65,8 +58,6 @@ pub struct Owned { /// The epoch whose log a new owner recovers (0: none). `None` when this /// instance already opened the epoch. pub recover: Option, - /// The facet tree index, as own.json holds it. - pub facets: Vec<(u32, String)>, } /// Who serves an object. @@ -296,43 +287,6 @@ impl Ownership { Ok(Some(record.owned())) } - /// Record a longer facet index for this instance's open epoch, before a - /// new facet's log opens: a takeover restores the facets own.json names. - /// `false` when the record no longer names this open epoch. - pub async fn set_facets( - &self, - namespace: &str, - actor: &str, - epoch: u64, - facets: &[(u32, String)], - ) -> Result { - let key = owner_key(namespace, actor); - for _ in 0..ATTEMPTS { - let Some((body, version)) = self.bucket.get(&key).await? else { - return Ok(false); - }; - let mut record: OwnerRecord = serde_json::from_slice(&body)?; - if record.node != self.registry.node() || record.epoch != epoch || record.from != epoch - { - return Ok(false); - } - if record.facets.len() >= facets.len() { - return Ok(true); - } - record.facets = facets.to_vec(); - let body = serde_json::to_vec(&record)?; - if self - .bucket - .put_if(&key, body, Guard::Match(version)) - .await? - .is_some() - { - return Ok(true); - } - } - Err(Error::Contended(key)) - } - /// The epoch own.json names now: the re-read after a bucket upload that /// makes the upload an acknowledgement (spec: `TierAck`). pub async fn current_epoch(&self, namespace: &str, actor: &str) -> Result> { diff --git a/src/workerd/server/cluster/registry.rs b/src/workerd/server/cluster/registry.rs index 0f52f9e616e..486dc7c72ed 100644 --- a/src/workerd/server/cluster/registry.rs +++ b/src/workerd/server/cluster/registry.rs @@ -17,9 +17,9 @@ use crate::Version; /// One instance's record in `nodes/.json`. #[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] pub struct NodeRecord { - /// The instance's identity: its cluster public key, hex. + /// The instance's identity: 16 random bytes, hex, new each start. pub node: String, - /// Where peers reach it: its sockaddr's bytes, hex. + /// Where peers reach it, `host:port`. pub address: String, /// The instance is live until this time and must renew before it. pub expires_ms: u64, From 52db46dc71647edecbceb9392fa50ced4bd2617e Mon Sep 17 00:00:00 2001 From: Ryan Dahl Date: Tue, 27 Oct 2026 11:35:56 -0700 Subject: [PATCH 12/38] cluster: skip the SIGTERM drain tests on Windows The cluster test runs on Windows since WAL capture moved to a directory, but Node's child.kill('SIGTERM') is TerminateProcess there: the node dies without draining, so the two drain tests read ECONNRESET. --- .../server/tests/cluster/cluster-test.mjs | 170 ++++++++++-------- 1 file changed, 96 insertions(+), 74 deletions(-) diff --git a/src/workerd/server/tests/cluster/cluster-test.mjs b/src/workerd/server/tests/cluster/cluster-test.mjs index ac803b6fb4b..ef2cf84d4a5 100644 --- a/src/workerd/server/tests/cluster/cluster-test.mjs +++ b/src/workerd/server/tests/cluster/cluster-test.mjs @@ -45,6 +45,11 @@ assert( const CONTROL_FD = 3; +// Node's SIGTERM is TerminateProcess on Windows, so a node cannot drain there. +const SIGTERM_DRAINS = { + skip: process.platform === 'win32' && 'SIGTERM does not drain on Windows', +}; + // A minimal harness specialized for the cluster test. Unlike server-harness.mjs // it: // - allows overriding bindings via process environment (NODE_ID), @@ -502,86 +507,103 @@ test('cluster: killing the owner allows another node to take over', async () => }); }); -test('cluster: draining node still routes in-flight requests to a remote owner', async () => { - await withCluster(2, async ({ nodes }) => { - // Prime the DO so that exactly one node owns it. - const initial = await fetchJson(nodes[0].httpPort, '/increment?name=drain'); - assert.strictEqual(initial.status, 200); - assert.strictEqual(initial.body.count, 1); - const owner = nodes.find((n) => n.nodeId === initial.body.nodeId); - const nonOwner = nodes.find((n) => n.nodeId !== initial.body.nodeId); - assert(owner !== undefined && nonOwner !== undefined); - - // Put a request in flight on the non-owner whose stateless handler waits - // before calling the DO, then SIGTERM the non-owner while it is waiting. - // Draining lets in-flight requests finish, so this request must still be - // able to route to the owner over cluster RPC after draining began. - const requestPromise = fetchJson( - nonOwner.httpPort, - '/increment?name=drain&delay=1000', - { timeoutMs: 10_000 } - ); - await sleep(250); - const exitPromise = nonOwner.stop({ signal: 'SIGTERM', timeoutMs: 10_000 }); +test( + 'cluster: draining node still routes in-flight requests to a remote owner', + SIGTERM_DRAINS, + async () => { + await withCluster(2, async ({ nodes }) => { + // Prime the DO so that exactly one node owns it. + const initial = await fetchJson( + nodes[0].httpPort, + '/increment?name=drain' + ); + assert.strictEqual(initial.status, 200); + assert.strictEqual(initial.body.count, 1); + const owner = nodes.find((n) => n.nodeId === initial.body.nodeId); + const nonOwner = nodes.find((n) => n.nodeId !== initial.body.nodeId); + assert(owner !== undefined && nonOwner !== undefined); + + // Put a request in flight on the non-owner whose stateless handler waits + // before calling the DO, then SIGTERM the non-owner while it is waiting. + // Draining lets in-flight requests finish, so this request must still be + // able to route to the owner over cluster RPC after draining began. + const requestPromise = fetchJson( + nonOwner.httpPort, + '/increment?name=drain&delay=1000', + { timeoutMs: 10_000 } + ); + await sleep(250); + const exitPromise = nonOwner.stop({ + signal: 'SIGTERM', + timeoutMs: 10_000, + }); - const response = await requestPromise; - assert.strictEqual( - response.status, - 200, - `in-flight request on draining node failed: ${JSON.stringify(response.body)}` - ); - assert.strictEqual(response.body.nodeId, owner.nodeId); - assert.strictEqual(response.body.count, 2); + const response = await requestPromise; + assert.strictEqual( + response.status, + 200, + `in-flight request on draining node failed: ${JSON.stringify(response.body)}` + ); + assert.strictEqual(response.body.nodeId, owner.nodeId); + assert.strictEqual(response.body.count, 2); - // The draining node must have exited cleanly once the request finished. - const exit = await exitPromise; - assert.deepStrictEqual(exit, { code: 0, signal: null }); - }); -}); + // The draining node must have exited cleanly once the request finished. + const exit = await exitPromise; + assert.deepStrictEqual(exit, { code: 0, signal: null }); + }); + } +); -test('cluster: draining node refuses to claim an unowned DO', async () => { - await withCluster(2, async ({ nodes, bucket, nodeKeys }) => { - // Put a request in flight on node 0 for a DO that nobody owns yet, then - // SIGTERM node 0 while its stateless handler is still waiting. By the time - // the handler calls the DO, node 0 is draining: its cluster listener no - // longer accepts connections, so claiming the DO there would leave it - // unreachable from the rest of the cluster until node 0 exits. The claim - // must be refused instead. - const requestPromise = fetchJson( - nodes[0].httpPort, - '/increment?name=lame-duck&delay=1000', - { timeoutMs: 10_000 } - ); - await sleep(250); - const exitPromise = nodes[0].stop({ signal: 'SIGTERM', timeoutMs: 10_000 }); +test( + 'cluster: draining node refuses to claim an unowned DO', + SIGTERM_DRAINS, + async () => { + await withCluster(2, async ({ nodes, bucket, nodeKeys }) => { + // Put a request in flight on node 0 for a DO that nobody owns yet, then + // SIGTERM node 0 while its stateless handler is still waiting. By the time + // the handler calls the DO, node 0 is draining: its cluster listener no + // longer accepts connections, so claiming the DO there would leave it + // unreachable from the rest of the cluster until node 0 exits. The claim + // must be refused instead. + const requestPromise = fetchJson( + nodes[0].httpPort, + '/increment?name=lame-duck&delay=1000', + { timeoutMs: 10_000 } + ); + await sleep(250); + const exitPromise = nodes[0].stop({ + signal: 'SIGTERM', + timeoutMs: 10_000, + }); - const response = await requestPromise; - assert.strictEqual( - response.status, - 500, - `draining node must not claim the DO: ${JSON.stringify(response.body)}` - ); - // The refusal is a DISCONNECTED exception, which JS sees as a generic - // connection-loss error. - assert.match(response.body.error, /Network connection lost/); + const response = await requestPromise; + assert.strictEqual( + response.status, + 500, + `draining node must not claim the DO: ${JSON.stringify(response.body)}` + ); + // The refusal is a DISCONNECTED exception, which JS sees as a generic + // connection-loss error. + assert.match(response.body.error, /Network connection lost/); - const exit = await exitPromise; - assert.deepStrictEqual(exit, { code: 0, signal: null }); + const exit = await exitPromise; + assert.deepStrictEqual(exit, { code: 0, signal: null }); - // The DO was never instantiated anywhere: the survivor claims it fresh. - const survivor = await fetchJson( - nodes[1].httpPort, - '/increment?name=lame-duck' - ); - assert.strictEqual(survivor.status, 200, JSON.stringify(survivor.body)); - assert.strictEqual(survivor.body.count, 1); - assert.strictEqual(survivor.body.nodeId, nodes[1].nodeId); - assert.strictEqual( - await bucket.owner(survivor.body.id), - nodeKeys.get(nodes[1].nodeId) - ); - }); -}); + // The DO was never instantiated anywhere: the survivor claims it fresh. + const survivor = await fetchJson( + nodes[1].httpPort, + '/increment?name=lame-duck' + ); + assert.strictEqual(survivor.status, 200, JSON.stringify(survivor.body)); + assert.strictEqual(survivor.body.count, 1); + assert.strictEqual(survivor.body.nodeId, nodes[1].nodeId); + assert.strictEqual( + await bucket.owner(survivor.body.id), + nodeKeys.get(nodes[1].nodeId) + ); + }); + } +); test('cluster: concurrent first requests to a DO read its owner record once per node', async () => { await withCluster(2, async ({ nodes, bucket }) => { From 577aec742ca78c57f3946cc761e90899493ca069 Mon Sep 17 00:00:00 2001 From: Ryan Dahl Date: Tue, 27 Oct 2026 15:47:36 -0700 Subject: [PATCH 13/38] cluster: log in SRS's sqlite lane format and key layout The log's frames are the storage relay's, field for field: each commit is a write of the bytes SQLite wrote to the WAL, at an offset that keeps growing across checkpoints, each marked by a consume, plus newLane, initialize and deleteLane. A batch starts and ends at a commit. A snapshot is a SnapshotHeader and then the database file, never a WAL: replay checkpoints committed WAL frames into the file first. Members workerd leaves unset are kept for wire compatibility and marked unused. Compaction follows SRS's rule: snapshot a session once the log since its last snapshot reaches the database's size, from that snapshot and the batches past it, then delete what the new one covers. Batches are never merged, so the page merge and the level-numbered chain are gone. Keys match SRS's layout, and none depends on a name's length: do///log// /snapshot// /own.json is base64url of the first 16 bytes of a labelled SHA-256 of the tenant (empty for now) and the namespace's uniqueKey; is the actor ID's hex. Numbers are SRS's length-prefixed hex, sessions and snapshot frames descending, so a listing comes newest first. The FileHeader is dropped: the key says what a file is. --- src/workerd/server/cluster/BUILD.bazel | 1 + src/workerd/server/cluster/capture.rs | 204 +++---- src/workerd/server/cluster/cluster.rs | 6 +- src/workerd/server/cluster/frames.capnp | 144 +++-- src/workerd/server/cluster/lib.rs | 33 +- src/workerd/server/cluster/log.rs | 558 +++++++++++------- src/workerd/server/cluster/log_tests.rs | 164 ++--- src/workerd/server/cluster/owner.rs | 272 ++++----- src/workerd/server/cluster/ownership.rs | 4 +- .../server/tests/cluster/cluster-test.mjs | 25 +- 10 files changed, 789 insertions(+), 622 deletions(-) diff --git a/src/workerd/server/cluster/BUILD.bazel b/src/workerd/server/cluster/BUILD.bazel index f30174c16fc..5c014a06ee6 100644 --- a/src/workerd/server/cluster/BUILD.bazel +++ b/src/workerd/server/cluster/BUILD.bazel @@ -15,6 +15,7 @@ wd_rust_crate( deps = [ ":frames_capnp_rust", "//src/workerd/server/cluster/logic:cluster-logic", + "@crates_vendor//:base64", "@crates_vendor//:bytes", "@crates_vendor//:capnp", "@crates_vendor//:futures", diff --git a/src/workerd/server/cluster/capture.rs b/src/workerd/server/cluster/capture.rs index 09f7563d2c1..ad6fb22e455 100644 --- a/src/workerd/server/cluster/capture.rs +++ b/src/workerd/server/cluster/capture.rs @@ -3,17 +3,19 @@ // https://opensource.org/licenses/Apache-2.0 //! A Durable Object's commits, cut from the writes `SQLite` makes to its WAL -//! as workerd's VFS reports them: one log entry, a frame of the pages it -//! wrote, per transaction. +//! as workerd's VFS reports them: per transaction, the frames of +//! `frames.capnp` that write its bytes to the database lane. //! //! `SQLite` writes a frame as its 24-byte header and then its page, at offsets //! that follow the 32-byte WAL header, and the header of a transaction's //! last frame carries the database's size, which marks the commit. A write //! at offset 0 starts a new WAL, after a checkpoint has copied every -//! earlier frame into the database file. So the frames since the last -//! commit are buffered, and the write that completes a commit frame -//! completes one transaction. A rolled-back transaction's frames start -//! where the next one's do, which overwrites them. +//! earlier frame into the database file. So the bytes since the last commit +//! are buffered, and the write that completes a commit frame completes one +//! transaction: a write of those bytes at the lane's next offset, which +//! keeps growing across checkpoints, each marked by a consume. A +//! rolled-back transaction's frames start where the next one's do, which +//! overwrites them. use std::collections::BTreeMap; use std::path::Path; @@ -21,96 +23,95 @@ use std::path::PathBuf; use crate::Error; use crate::Result; -use crate::Txid; +use crate::Sequence; use crate::log::Body; use crate::log::Cursor; use crate::log::push; const WAL_HEADER: u64 = 32; -const FRAME_HEADER: usize = 24; +const FRAME_HEADER: u64 = 24; pub struct Capture { /// The database file. path: PathBuf, - /// The chain's entries not yet in the bucket, by txid. - entries: BTreeMap>, - last: Txid, + /// The session's frames not yet in the bucket, by commit, keyed by the + /// frame after it. + entries: BTreeMap>, cursor: Cursor, - page_size: usize, - /// Where the next transaction's first frame goes: 0 before a WAL header. - start: u64, + page_size: u64, + /// Where in the WAL the next transaction's bytes start; `None` before a + /// WAL header. + start: Option, /// What was written from `start` on. - frames: Vec, - /// The lane does not exist: the next commit creates it with every page - /// of the file, since a new database's first page reaches the file, not - /// the WAL. - fresh: bool, + pending: Vec, + /// A checkpoint consumed the lane's WAL since the last commit. + consumed: bool, /// Why the capture can no longer be trusted, if it cannot. lost: Option, } impl Capture { - /// Capture the database at `path`, which no connection has open yet: - /// what it holds now is txid 1. + /// Capture the database at `path`, which no connection has open yet, + /// starting from what it holds now. #[cfg(test)] pub fn open(path: &Path) -> Result { - let mut capture = Self::resume(path, 0, Cursor::default()); + let mut capture = Self::resume(path, Cursor::default()); capture.restart()?; Ok(capture) } - /// Continue a chain through `last`, at `cursor`, over the database at - /// `path`, which holds exactly that chain and no connection has open - /// yet. A chain at 0 starts with the next commit. - pub fn resume(path: &Path, last: Txid, cursor: Cursor) -> Self { + /// Continue a session at `cursor` over the database at `path`, which + /// holds exactly the session's log so far and no connection has open + /// yet. Without a lane, the next commit creates it from the file, since + /// a new database's first page reaches the file, not the WAL. + pub fn resume(path: &Path, cursor: Cursor) -> Self { Self { path: path.to_owned(), entries: BTreeMap::new(), - last, cursor, page_size: 0, - start: 0, - frames: Vec::new(), - fresh: last == 0, + start: None, + pending: Vec::new(), + consumed: false, lost: None, } } - /// Start a new chain from the database file, which holds every commit: - /// no WAL, or one a checkpoint truncated. + /// Start the session over from the database file, which holds every + /// commit: no WAL, or one a checkpoint truncated. #[cfg(test)] pub fn restart(&mut self) -> Result<()> { self.entries.clear(); - (self.last, self.cursor.sequence) = (0, 0); + self.start = None; + self.cursor = Cursor { + session: self.cursor.session, + ..Cursor::default() + }; let image = match std::fs::read(&self.path) { Err(e) if e.kind() == std::io::ErrorKind::NotFound => Vec::new(), image => image?, }; - self.fresh = image.is_empty(); - if !self.fresh { - let commit = file_pages(&image); + if !image.is_empty() { let mut entry = Vec::new(); push(&mut entry, &mut self.cursor, Body::NewLane); - push(&mut entry, &mut self.cursor, Body::Pages(&commit)); - self.last = 1; - self.entries.insert(1, entry); + push(&mut entry, &mut self.cursor, Body::Initialize(&image)); + self.entries.insert(self.cursor.sequence, entry); } Ok(()) } - /// The database's files were deleted: the chain continues without the + /// The database's files were deleted: the session continues without the /// lane, and the next commit creates it. pub fn reset(&mut self) { - self.start = 0; - self.frames.clear(); - self.fresh = true; - if self.last == 0 { + self.start = None; + self.pending.clear(); + self.consumed = false; + if !self.cursor.lane { return; } let mut entry = Vec::new(); push(&mut entry, &mut self.cursor, Body::DeleteLane); - self.last += 1; - self.entries.insert(self.last, entry); + self.entries.insert(self.cursor.sequence, entry); } /// A write `SQLite` made to the WAL. @@ -118,66 +119,62 @@ impl Capture { if offset == 0 { // A new WAL: a checkpoint copied the last one into the file. if let Some(size) = data.get(8..12) { - self.page_size = u32::from_be_bytes([size[0], size[1], size[2], size[3]]) as usize; - self.start = WAL_HEADER; - self.frames.clear(); + self.page_size = + u64::from(u32::from_be_bytes([size[0], size[1], size[2], size[3]])); + self.consumed |= self.cursor.end > self.cursor.base; + self.start = Some(0); + self.pending.clear(); } - return; } - if self.start == 0 || offset < self.start { - self.lost = Some(format!("a WAL write at {offset}, before {}", self.start)); + let Some(start) = self.start.filter(|&s| offset >= s) else { + self.lost = Some(format!("a WAL write at {offset}, before {:?}", self.start)); return; - } - let at = usize::try_from(offset - self.start).unwrap_or(usize::MAX); + }; + let at = usize::try_from(offset - start).unwrap_or(usize::MAX); let end = at.saturating_add(data.len()); - if self.frames.len() < end { - self.frames.resize(end, 0); + if self.pending.len() < end { + self.pending.resize(end, 0); } - self.frames[at..end].copy_from_slice(data); + self.pending[at..end].copy_from_slice(data); let size = FRAME_HEADER + self.page_size; - if !end.is_multiple_of(size) { + let wal_end = start + end as u64; + if wal_end <= WAL_HEADER || !(wal_end - WAL_HEADER).is_multiple_of(size) { return; } - let header = &self.frames[end - size..end - size + FRAME_HEADER]; - let commit = u32::from_be_bytes([header[4], header[5], header[6], header[7]]); - if commit != 0 { - self.commit(end, commit); + let header = &self.pending[end - size as usize..]; + if u32::from_be_bytes([header[4], header[5], header[6], header[7]]) != 0 { + self.commit(start, end); } } - /// The frames before `end` are one committed transaction. - fn commit(&mut self, end: usize, commit: u32) { + /// The bytes before `end` are one committed transaction. + fn commit(&mut self, start: u64, end: usize) { let mut entry = Vec::new(); - let image = if self.fresh { - self.fresh = false; - std::fs::read(&self.path).unwrap_or_else(|e| { - self.lost = Some(format!("the new database: {e}")); - Vec::new() - }) - } else { - Vec::new() - }; - let (_, mut pages) = file_pages(&image); - if !image.is_empty() { - push(&mut entry, &mut self.cursor, Body::NewLane); + if !self.cursor.lane { + match std::fs::read(&self.path) { + Ok(image) => { + push(&mut entry, &mut self.cursor, Body::NewLane); + push(&mut entry, &mut self.cursor, Body::Initialize(&image)); + } + Err(e) => self.lost = Some(format!("the new database: {e}")), + } } - let size = FRAME_HEADER + self.page_size; - for frame in self.frames[..end].chunks_exact(size) { - let pgno = u32::from_be_bytes([frame[0], frame[1], frame[2], frame[3]]); - pages.insert(pgno, &frame[FRAME_HEADER..]); + if std::mem::take(&mut self.consumed) { + push(&mut entry, &mut self.cursor, Body::Consume); } - pages.retain(|&pgno, _| pgno <= commit); - push(&mut entry, &mut self.cursor, Body::Pages(&(commit, pages))); - self.start += end as u64; - self.frames.drain(..end); - self.last += 1; - self.entries.insert(self.last, entry); + let at = self.cursor.base + start; + let bytes = &self.pending[..end]; + push(&mut entry, &mut self.cursor, Body::Write(at, bytes)); + self.start = Some(start + end as u64); + self.pending.drain(..end); + self.entries.insert(self.cursor.sequence, entry); } - /// The last committed txid, or why the capture cannot be trusted. - pub fn last(&self) -> Result { + /// The frame after the last commit, or why the capture cannot be + /// trusted. + pub fn last(&self) -> Result { match &self.lost { - None => Ok(self.last), + None => Ok(self.cursor.sequence), Some(why) => Err(Error::Replica(format!("capture lost: {why}"))), } } @@ -186,26 +183,17 @@ impl Capture { self.cursor } - pub fn entry(&self, txid: Txid) -> Result> { - self.entries - .get(&txid) - .cloned() - .ok_or_else(|| Error::Replica(format!("no captured entry {txid}"))) + /// The frames from `from` up to `to`, both after a commit. + pub fn batch(&self, from: Sequence, to: Sequence) -> Result> { + let mut entries = self.entries.range(from + 1..=to).peekable(); + if entries.peek().is_none() { + return Err(Error::Replica(format!("no captured frames {from}..{to}"))); + } + Ok(entries.flat_map(|(_, e)| e.iter().copied()).collect()) } - /// The bucket holds everything through `txid`. - pub fn forget(&mut self, txid: Txid) { - self.entries.retain(|t, _| *t > txid); + /// The bucket holds everything up to `to`. + pub fn forget(&mut self, to: Sequence) { + self.entries.retain(|&end, _| end > to); } } - -/// A database file's pages and size, in pages. -fn file_pages(image: &[u8]) -> crate::log::Pages<'_> { - let page_size = match image.get(16..18) { - Some(&[0, 1]) => 65536, - Some(size) => usize::from(u16::from_be_bytes([size[0], size[1]])), - None => return (0, BTreeMap::new()), - }; - let pages: BTreeMap<_, _> = (1..).zip(image.chunks_exact(page_size)).collect(); - (pages.len() as u32, pages) -} diff --git a/src/workerd/server/cluster/cluster.rs b/src/workerd/server/cluster/cluster.rs index 237880d0c9f..fcc792aa838 100644 --- a/src/workerd/server/cluster/cluster.rs +++ b/src/workerd/server/cluster/cluster.rs @@ -217,7 +217,7 @@ impl Cluster { } } - /// Compact owned objects' chains in the bucket. + /// Compact owned objects' logs in the bucket. pub async fn tier(&self) { self.owners.tier().await; } @@ -278,7 +278,7 @@ impl Cluster { } } - /// Compacts owned objects' chains. Never returns. + /// Compacts owned objects' logs. Never returns. pub async fn tier_loop(&self) { loop { tokio::time::sleep(TIER_INTERVAL).await; @@ -308,7 +308,7 @@ impl Cluster { } } -/// How often owned objects' chains are checked for compaction. +/// How often owned objects' logs are checked for compaction. const TIER_INTERVAL: Duration = Duration::from_secs(2); const fn ms(ms: u64) -> Duration { diff --git a/src/workerd/server/cluster/frames.capnp b/src/workerd/server/cluster/frames.capnp index 33ef129fe98..9f3f4f6371e 100644 --- a/src/workerd/server/cluster/frames.capnp +++ b/src/workerd/server/cluster/frames.capnp @@ -3,77 +3,109 @@ # https://opensource.org/licenses/Apache-2.0 @0xf699edadaece33b7; -# The format of a Durable Object's log in the bucket. An object's state is -# a set of lanes, each a file; its log is a stream of frames, each a change -# to one lane, written by one session (an ownership epoch) in sequence. +# The format of a Durable Object's log in the bucket: the storage relay's +# (SRS's), field for field, so the two stay wire-compatible. Workerd writes +# one `sqlite` lane and leaves unset what it has no use for, noted below. # Compression and encryption are not part of it. +# +# A log batch is a run of frames, each one message, that starts and ends at +# a commit. A snapshot is a `SnapshotHeader` and then each lane's content in +# lane order. using Cxx = import "/capnp/c++.capnp"; $Cxx.namespace("workerd::server::cluster"); -struct FileHeader { - # The first message of every file in the bucket. A file's level and - # range are in its key, not here. - - union { - log @0 :Void; - # Frames follow. - - snapshot @1 :SnapshotHeader; - # Each lane's content follows. - } -} - struct Frame { - # A batch of frames in the bucket starts and ends at a commit, and is - # applied whole. + # One change to an object's state. session @0 :UInt32; # The ownership epoch that wrote the frame. sequence @1 :UInt32; - # Rises within a session. A frame that merges a run of commits takes the - # run's last sequence. + # Position within the session, from zero. A position between two frames + # is the sequence of the one after it. lane @2 :UInt32; # A lane is numbered on first use, with the lowest free number. + const unknownContinueFromSession :UInt32 = 0xffffffff; + + continueFromSession @10 :UInt32 = Frame.unknownContinueFromSession; + continueFromSequence @11 :UInt32; + # For a session's first frame: where the last non-empty session ended. + # Workerd leaves these unknown: own.json names the session a session + # recovered. + union { newLane @3 :LaneDescription; + + cloneLane :group { + # Unused by workerd. + name @13 :Text; + from @14 :UInt32; + } + deleteLane @4 :Void; - pages :group { - # For a `sqlitePages` lane: the newest version of each page a commit, - # or a run of commits, wrote, and the database's size after, in - # pages. - commit @5 :UInt32; - list @6 :List(Page); + write :group { + # For a `sqlite` lane, bytes SQLite wrote to the WAL, at an offset + # that keeps growing across checkpoints, though SQLite sees each WAL + # start over at zero. + offset @5 :UInt64; + data @6 :Data; + } + + obsolete7 @7 :Void; + + consume :group { + # For a `sqlite` lane, a checkpoint copied the WAL's committed pages + # into the database: data before `offset` is not needed again. + offset @8 :UInt64; } - } -} -struct Page { - number @0 :UInt32; - data @1 :Data; + initialize @9 :Data; + # For a `sqlite` lane, the whole database file, written before the WAL + # is in use. + + setReplicaInformation @12 :AnyPointer; + # Unused by workerd. + } } struct LaneDescription { name @0 :Text; # The lane's stable identifier; numbers differ between objects. - type @1 :Type; - - enum Type { - sqlitePages @0; - # A SQLite database whose writes are pages, so a run of commits merges - # to one copy of each page. + union { + sqlite @1 :Void; + journal @2 :Void; + # Unused by workerd. + replicaInformation @3 :Void; + # Unused by workerd. } } +enum SnapshotJustification { + # Workerd writes only `clientRequested` and `coldStartMitigation`. + + unknown @0; + clientRequested @1; + # The log written since the last snapshot outgrew the database. + pointInTimeRecovery @2; + snapshotFallback @3; + lostAndFoundRecovery @4; + coldStartMitigation @5; + # Workerd's snapshot of the state a session starts from, so the next + # restore reads one file and no earlier session's log. + cleanShutdown @6; + test @7; + srsTools @8; + localOnly @9; +} + struct SnapshotHeader { - # The whole state of an object at a session and frame: this header, then - # each lane's content in lane order. The log is replayed from `frame` - # on. + # The whole state of an object at a session and frame; the log continues + # from `frame`. session @0 :UInt32; frame @1 :UInt32; @@ -88,9 +120,39 @@ struct SnapshotHeader { deleted @1 :Void; # A lane number freed while later ones are in use. - sqlitePages :group { + sqlite :group { databaseSize @2 :UInt64; + walSize @3 :UInt64; + # The WAL bytes after the database's. Always 0 in the bucket: a + # snapshot's database has every committed page. + + walBaseOffset @6 :UInt64; + # Where the log's next write continues. Workerd snapshots from the + # log, not the live database, so that write can continue a WAL + # past its header, at a frame. + + walShiftOffset @7 :UInt64; + # Always 0: workerd keeps no lane files of its own to shift. + } + + journal :group { + # Unused by workerd, as is `replicaInformation`. + size @4 :UInt64; + offset @5 :UInt64; + } + + replicaInformation :group { + size @8 :UInt64; } } } + + logBytesSinceLastSnapshot @3 :UInt64; + # The log bytes in the bucket since the previous snapshot. + + continueFromSession @4 :UInt32 = Frame.unknownContinueFromSession; + continueFromSequence @5 :UInt32; + # Unknown, as in `Frame`. + + justification @6 :SnapshotJustification; } diff --git a/src/workerd/server/cluster/lib.rs b/src/workerd/server/cluster/lib.rs index 80d590c51cc..e8ca3a84d8d 100644 --- a/src/workerd/server/cluster/lib.rs +++ b/src/workerd/server/cluster/lib.rs @@ -6,16 +6,20 @@ //! (S3, R2, `MinIO`) instead of a shared filesystem. //! //! Everything lives under `v1/` in the configured prefix, so a later format -//! can tell an old bucket from its own. Two records: +//! can tell an old bucket from its own: //! //! - `nodes/.json`: one per running instance, with its address and a //! lease expiry the instance renews. A peer is live while its lease is. //! - `do///own.json`: one per Durable Object, naming its //! owner and an ownership epoch. A claim is a conditional write; taking an -//! object over from an owner whose lease expired bumps the epoch, and -//! everything the new owner writes is keyed by that epoch, under -//! `do///log/e/`, so a late write from the old -//! owner lands where nothing reads it. +//! object over from an owner whose lease expired bumps the epoch. +//! - `do///{log,snapshot}//..`: the object's log +//! (`log.rs`), a session per epoch. A takeover restores the last opened +//! epoch's session and starts the next from a snapshot of it, so a late +//! write from the old owner lands where nothing reads it. +//! +//! `` is a hash of the namespace's `uniqueKey`, and `` the +//! actor ID's hex, so no key's length depends on a name. //! //! The clock is an argument everywhere (`now_ms`), so a test decides time. @@ -85,8 +89,23 @@ pub type Result = std::result::Result; /// An ownership epoch: each takeover of an object bumps it. pub type Epoch = u64; -/// A transaction id within one epoch's chain, from 1. -pub type Txid = u64; +/// A position in an epoch's session of the log: the sequence of the frame +/// after it. +pub type Sequence = u32; + +/// Where an object's records are: `do//`, the namespace +/// as the first 16 bytes of a labelled SHA-256 of its tenant, empty for +/// now, and `uniqueKey`, in base64url. +pub(crate) fn object_prefix(ns: &str, actor: &str) -> String { + use base64::Engine; + let mut data = Vec::new(); + for part in ["workerd cluster namespace v1", "", ns] { + data.extend((part.len() as u64).to_be_bytes()); + data.extend(part.as_bytes()); + } + let hash = base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(&sha256(&data)[..16]); + format!("do/{hash}/{actor}") +} pub(crate) fn sha256(data: &[u8]) -> Vec { ring::digest::digest(&ring::digest::SHA256, data) diff --git a/src/workerd/server/cluster/log.rs b/src/workerd/server/cluster/log.rs index 908bbc2950e..5c0d0c66167 100644 --- a/src/workerd/server/cluster/log.rs +++ b/src/workerd/server/cluster/log.rs @@ -2,17 +2,22 @@ // Licensed under the Apache 2.0 license found in the LICENSE file or at: // https://opensource.org/licenses/Apache-2.0 -//! An object's log in the bucket (`frames.capnp`): one lane, the `SQLite` -//! database, a `sqlitePages` lane whose frames are the pages each commit -//! wrote. +//! An object's log in the bucket (`frames.capnp`), laid out as the storage +//! relay's: one `sqlite` lane, the database, whose frames are the bytes +//! each commit wrote to the WAL. //! -//! An epoch's log is a chain of files under its prefix, each one LZ4 -//! frame that starts with a `FileHeader`: `0000/-.log`, the frames of commits `first..=last`, -//! and `0009/0000000000000001-.log`, a snapshot through `last`. A -//! restore is the newest snapshot and the files that extend it; a frame a -//! file before already applied is skipped by its sequence. +//! Under `do///`, each file one LZ4 frame: +//! +//! - `log//`: a batch, the frames of a run of commits, up to +//! frame `end`. +//! - `snapshot//`: a snapshot, the log continuing from +//! `frame`. +//! +//! Numbers are length-prefixed hex, sessions and snapshot frames +//! descending, so a listing comes newest first. A session's state is its +//! newest snapshot and the batches after it; a frame a batch before +//! already applied is skipped by its sequence. -use std::collections::BTreeMap; use std::io::BufReader; use std::io::Read; use std::io::Seek; @@ -21,7 +26,7 @@ use std::io::Write; use std::path::Path; use std::path::PathBuf; -use frames_capnp::file_header; +use frames_capnp::SnapshotJustification; use frames_capnp::frame; use frames_capnp::snapshot_header; use futures::StreamExt; @@ -31,103 +36,69 @@ use futures::stream; use crate::Bucket; use crate::Error; use crate::Result; -use crate::Txid; +use crate::Sequence; -/// The level of snapshots, as litestream's. -pub const SNAPSHOT_LEVEL: i32 = 9; /// The database's lane. const LANE: &str = "db"; /// Downloads in flight during a restore, to hide the store's round trips. const RESTORE_CONCURRENCY: usize = 16; +const WAL_HEADER: usize = 32; +const FRAME_HEADER: usize = 24; -/// Where an epoch's next frame goes. +/// Where a session's next frame goes, and the database lane's state. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] pub struct Cursor { pub session: u32, - pub sequence: u32, + pub sequence: Sequence, + /// Whether the lane exists. + pub lane: bool, + /// Where the lane's next write goes. + pub end: u64, + /// Where the current WAL starts. + pub base: u64, } -/// A commit's pages, by number, and the database's size after, in pages. -pub type Pages<'a> = (u32, BTreeMap); - #[derive(Clone, Copy)] pub enum Body<'a> { NewLane, DeleteLane, - Pages(&'a Pages<'a>), + Initialize(&'a [u8]), + Write(u64, &'a [u8]), + /// A checkpoint consumed everything written so far. + Consume, } -/// Appends a frame to `out` at `cursor`. +/// Appends a frame to `out` at `cursor`, and moves it past. pub fn push(out: &mut Vec, cursor: &mut Cursor, body: Body) { - frame_at(out, cursor.session, cursor.sequence, body); - cursor.sequence += 1; -} - -fn frame_at(out: &mut Vec, session: u32, sequence: u32, body: Body) { let mut message = capnp::message::Builder::new_default(); let mut f = message.init_root::(); - f.set_session(session); - f.set_sequence(sequence); + f.set_session(cursor.session); + f.set_sequence(cursor.sequence); match body { Body::NewLane => { let mut lane = f.init_new_lane(); lane.set_name(LANE); - lane.set_type(frames_capnp::lane_description::Type::SqlitePages); - } - Body::DeleteLane => f.set_delete_lane(()), - Body::Pages((commit, pages)) => { - let mut group = f.init_pages(); - group.set_commit(*commit); - let mut list = group.init_list(pages.len() as u32); - for (i, (&number, data)) in pages.iter().enumerate() { - let mut page = list.reborrow().get(i as u32); - page.set_number(number); - page.set_data(data); - } + lane.set_sqlite(()); + cursor.lane = true; } - } - let _ = capnp::serialize::write_message(&mut *out, &message); -} - -/// The log file of `entries`, consecutive commits: each run of page frames -/// becomes one with the newest version of each page. -pub fn merge(entries: &[Vec]) -> Result> { - let mut messages = Vec::new(); - for entry in entries { - let mut r = entry.as_slice(); - while let Some(message) = capnp::serialize::try_read_message(&mut r, options())? { - messages.push(message); + Body::DeleteLane => { + f.set_delete_lane(()); + (cursor.lane, cursor.end, cursor.base) = (false, 0, 0); } - } - let mut out = Vec::new(); - let mut header = capnp::message::Builder::new_default(); - header.init_root::().set_log(()); - capnp::serialize::write_message(&mut out, &header)?; - let mut run: Option<(u32, u32, Pages)> = None; - let flush = |out: &mut Vec, run: &mut Option<(u32, u32, Pages)>| { - if let Some((session, sequence, (commit, mut pages))) = run.take() { - pages.retain(|&number, _| number <= commit); - frame_at(out, session, sequence, Body::Pages(&(commit, pages))); + Body::Initialize(image) => f.set_initialize(image), + Body::Write(offset, data) => { + let mut write = f.init_write(); + write.set_offset(offset); + write.set_data(data); + cursor.end = offset + data.len() as u64; } - }; - for message in &messages { - let f = message.get_root::()?; - let (session, sequence) = (f.get_session(), f.get_sequence()); - if let frame::Pages(group) = f.which()? { - let r = run.get_or_insert_with(|| (session, sequence, (0, BTreeMap::new()))); - (r.1, r.2.0) = (sequence, group.get_commit()); - for page in group.get_list()? { - r.2.1.insert(page.get_number(), page.get_data()?); - } - } else { - flush(&mut out, &mut run); - let mut copy = capnp::message::Builder::new_default(); - copy.set_root(f)?; - capnp::serialize::write_message(&mut out, ©)?; + Body::Consume => { + f.init_consume().set_offset(cursor.end); + cursor.base = cursor.end; } } - flush(&mut out, &mut run); - Ok(out) + let _ = capnp::serialize::write_message(&mut *out, &message); + cursor.sequence += 1; } /// LZ4-frames `bytes`, as every file in the bucket is. @@ -148,12 +119,23 @@ fn options() -> capnp::message::ReaderOptions { *capnp::message::ReaderOptions::new().traversal_limit_in_words(None) } +fn be32(b: &[u8]) -> u32 { + u32::from_be_bytes([b[0], b[1], b[2], b[3]]) +} + /// The database lane, replayed into a file. pub struct Replay { db: std::fs::File, live: bool, + /// The WAL bytes not yet checkpointed, from `base` on. + wal: Vec, + base: u64, + /// Whether `wal` starts with a WAL header, rather than a frame past a + /// snapshot. + header: bool, /// The next frame, by session and sequence. - next: Option<(u32, u32)>, + next: Option<(u32, Sequence)>, + log_bytes: u64, } impl Replay { @@ -166,37 +148,42 @@ impl Replay { .truncate(true) .open(db)?, live: false, + wal: Vec::new(), + base: 0, + header: true, next: None, + log_bytes: 0, }) } - /// Applies the file `path` of the chain. - pub fn apply(&mut self, path: &Path) -> Result<()> { + /// Applies `f`, the local copy at `path`. A batch that does not reach + /// back to the next frame is skipped: a later, wider one covers it. + pub fn apply(&mut self, f: &File, path: &Path) -> Result<()> { let file = std::fs::File::open(path)?; + let size = file.metadata()?.len(); let mut r = BufReader::new(lz4_flex::frame::FrameDecoder::new(file)); - let message = capnp::serialize::read_message(&mut r, options())?; - match message.get_root::()?.which()? { - file_header::Log(()) => self.log(&mut r), - file_header::Snapshot(header) => self.snapshot(header?, &mut r), + if f.snapshot { + return self.snapshot(&mut r); } - } - - fn log(&mut self, r: &mut impl Read) -> Result<()> { - while let Some(message) = capnp::serialize::try_read_message(&mut *r, options())? { + while let Some(message) = capnp::serialize::try_read_message(&mut r, options())? { let f = message.get_root::()?; let at = (f.get_session(), f.get_sequence()); - if self.next.is_some_and(|next| at < next) { - continue; + match self.next.unwrap_or((at.0, 0)) { + next if at < next => continue, + next if at > next => return Ok(()), + _ => self.next = Some((at.0, at.1 + 1)), } - self.next = Some((at.0, at.1 + 1)); self.frame(f)?; } + self.log_bytes += size; Ok(()) } - fn snapshot(&mut self, header: snapshot_header::Reader, r: &mut impl Read) -> Result<()> { + fn snapshot(&mut self, r: &mut impl Read) -> Result<()> { + let message = capnp::serialize::read_message(&mut *r, options())?; + let header = message.get_root::()?; self.next = Some((header.get_session(), header.get_frame())); - self.live = false; + (self.wal, self.base, self.live, self.log_bytes) = (Vec::new(), 0, false, 0); self.db.set_len(0)?; self.db.seek(SeekFrom::Start(0))?; let lanes = header.get_lanes()?; @@ -204,9 +191,11 @@ impl Replay { return Err(Error::Replica("a snapshot of more than one lane".into())); } for lane in lanes { - if let snapshot_header::lane::SqlitePages(lane) = lane.which()? { - std::io::copy(&mut r.take(lane.get_database_size()), &mut self.db)?; - self.live = true; + if let snapshot_header::lane::Sqlite(sqlite) = lane.which()? { + std::io::copy(&mut r.take(sqlite.get_database_size()), &mut self.db)?; + r.take(sqlite.get_wal_size()).read_to_end(&mut self.wal)?; + (self.base, self.live) = (sqlite.get_wal_base_offset(), true); + self.header = !self.wal.is_empty(); } } Ok(()) @@ -219,80 +208,142 @@ impl Replay { match f.which()? { frame::NewLane(_) | frame::DeleteLane(()) => { self.db.set_len(0)?; + (self.wal, self.base, self.header) = (Vec::new(), 0, true); self.live = matches!(f.which()?, frame::NewLane(_)); } - frame::Pages(group) => { - let mut page_size = 0; - for page in group.get_list()? { - let data = page.get_data()?; - page_size = data.len() as u64; - let at = u64::from(page.get_number() - 1) * page_size; - self.db.seek(SeekFrom::Start(at))?; - self.db.write_all(data)?; - } - if page_size > 0 { - self.db.set_len(u64::from(group.get_commit()) * page_size)?; + frame::Initialize(image) => { + self.db.set_len(0)?; + self.db.seek(SeekFrom::Start(0))?; + self.db.write_all(image?)?; + } + frame::Write(w) => { + if w.get_offset() != self.base + self.wal.len() as u64 { + return Err(Error::Replica(format!("a write at {}", w.get_offset()))); } + self.wal.extend_from_slice(w.get_data()?); } + frame::Consume(c) => { + self.checkpoint()?; + (self.base, self.header) = (c.get_offset(), true); + } + _ => return Err(Error::Replica("an unsupported frame".into())), } Ok(()) } - /// The replayed database, whole. - pub fn finish(self) -> Result<()> { - Ok(self.db.sync_data()?) + /// Copies the WAL's committed pages into the database. + fn checkpoint(&mut self) -> Result<()> { + let wal = std::mem::take(&mut self.wal); + self.base += wal.len() as u64; + let (page_size, frames) = if std::mem::replace(&mut self.header, false) { + let Some(size) = wal.get(8..12) else { + return Ok(()); + }; + (be32(size) as usize, &wal[WAL_HEADER..]) + } else if wal.is_empty() { + return Ok(()); + } else { + // Past a snapshot, the database's header has the page size. + let mut size = [0; 2]; + self.db.seek(SeekFrom::Start(16))?; + self.db.read_exact(&mut size)?; + let size = usize::from(u16::from_be_bytes(size)); + (if size == 1 { 1 << 16 } else { size }, &wal[..]) + }; + let frames: Vec<_> = frames.chunks_exact(FRAME_HEADER + page_size).collect(); + let Some(last) = frames.iter().rposition(|f| be32(&f[4..]) != 0) else { + return Ok(()); + }; + for f in &frames[..=last] { + let at = u64::from(be32(f) - 1) * page_size as u64; + self.db.seek(SeekFrom::Start(at))?; + self.db.write_all(&f[FRAME_HEADER..])?; + } + let pages = u64::from(be32(&frames[last][4..])); + Ok(self.db.set_len(pages * page_size as u64)?) + } + + /// The replayed database, whole, and the frame the log continues from. + pub fn finish(mut self) -> Result { + self.checkpoint()?; + self.db.sync_data()?; + Ok(self.next.map_or(0, |n| n.1)) } - /// Writes the replayed state as a snapshot to `out`. - pub fn write_snapshot(mut self, out: &Path) -> Result { + /// Writes the replayed state as a snapshot into `dir`. + pub fn write_snapshot(mut self, dir: &Path) -> Result { + self.checkpoint()?; let (session, frame) = self.next.unwrap_or_default(); let size = self.db.seek(SeekFrom::End(0))?; self.db.seek(SeekFrom::Start(0))?; - let lane = self.live.then_some(size); - write_snapshot(out, session, frame, lane, |w| { + let lane = self.live.then_some((size, self.base)); + let header = Header { + session, + frame, + justification: SnapshotJustification::ClientRequested, + log_bytes: self.log_bytes, + }; + write_snapshot(dir, &header, lane, |w| { Ok(std::io::copy(&mut self.db, w).map(drop)?) }) } } -/// Writes a snapshot of the database lane, if it exists, of `size` bytes -/// that `payload` writes. -fn write_snapshot( - out: &Path, +struct Header { session: u32, - frame: u32, - size: Option, + frame: Sequence, + justification: SnapshotJustification, + log_bytes: u64, +} + +/// Writes a snapshot into `dir`: the database lane, if it exists, as its +/// size and where the log's next write goes, with content `payload` writes. +fn write_snapshot( + dir: &Path, + h: &Header, + lane: Option<(u64, u64)>, payload: impl FnOnce(&mut dyn Write) -> Result<()>, -) -> Result { +) -> Result { let mut message = capnp::message::Builder::new_default(); - let mut header = message.init_root::().init_snapshot(); - header.set_session(session); - header.set_frame(frame); - let mut lanes = header.init_lanes(u32::from(size.is_some())); - if let Some(size) = size { - let mut lane = lanes.reborrow().get(0); - lane.set_name(LANE); - lane.init_sqlite_pages().set_database_size(size); + let mut header = message.init_root::(); + header.set_session(h.session); + header.set_frame(h.frame); + header.set_justification(h.justification); + header.set_log_bytes_since_last_snapshot(h.log_bytes); + let mut lanes = header.init_lanes(u32::from(lane.is_some())); + if let Some((size, base)) = lane { + let mut l = lanes.reborrow().get(0); + l.set_name(LANE); + let mut sqlite = l.init_sqlite(); + sqlite.set_database_size(size); + sqlite.set_wal_base_offset(base); } - let tmp = with_suffix(out, ".tmp"); - let file = std::io::BufWriter::new(std::fs::File::create(&tmp)?); - let mut encoder = encoder(file); + let file = File { + snapshot: true, + session: h.session, + frame: h.frame, + size: 0, + }; + let out = dir.join(file.name()); + let tmp = with_suffix(&out, ".tmp"); + let mut encoder = encoder(std::io::BufWriter::new(std::fs::File::create(&tmp)?)); capnp::serialize::write_message(&mut encoder, &message)?; payload(&mut encoder)?; - let mut file = encoder + let mut w = encoder .finish() .map_err(|e| Error::Replica(e.to_string()))?; - file.flush()?; - drop(file); - std::fs::rename(&tmp, out)?; - Ok(std::fs::metadata(out)?.len()) + w.flush()?; + drop(w); + std::fs::rename(&tmp, &out)?; + let size = std::fs::metadata(&out)?.len(); + Ok(File { size, ..file }) } -/// Writes the database at `path`, which no connection has open, as a -/// snapshot at the start of `session` into `out`: its size, or `None` for an -/// empty database. Closing the last connection checkpointed the WAL into -/// the file. -pub fn snapshot_file(path: &Path, out: &Path, session: u32) -> Result> { +/// Writes the database at `path`, which no connection has open, as the +/// snapshot `session` starts from into `dir`, or `None` for an empty +/// database. Closing the last connection checkpointed the WAL into the +/// file. +pub fn snapshot_file(path: &Path, dir: &Path, session: u32) -> Result> { let mut db = match std::fs::File::open(path) { Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(None), db => db?, @@ -301,73 +352,114 @@ pub fn snapshot_file(path: &Path, out: &Path, session: u32) -> Result Result { - let db = with_suffix(out, ".db"); +/// Replays `files`, a plan with local copies, into a snapshot in `dir`. +pub fn compact(files: &[(File, PathBuf)], dir: &Path) -> Result { + let db = dir.join("compact.db"); let mut replay = Replay::new(&db)?; - for (_, path) in files { - replay.apply(path)?; + for (f, path) in files { + replay.apply(f, path)?; } - let size = replay.write_snapshot(out); + let snapshot = replay.write_snapshot(dir); let _ = std::fs::remove_file(&db); - size + snapshot } -/// One file of an epoch's chain: commits `first..=last` at level 0, or a -/// snapshot through `last` at `SNAPSHOT_LEVEL`. +/// One file of a session's log: a batch up to frame `frame`, or a snapshot +/// the log continues from at `frame`. #[derive(Clone, Copy, Debug, PartialEq, Eq)] pub struct File { - pub level: i32, - pub first: Txid, - pub last: Txid, + pub snapshot: bool, + pub session: u32, + pub frame: Sequence, pub size: u64, } impl File { /// This node's copy's name. pub fn name(&self) -> String { - format!( - "{:04x}-{:016x}-{:016x}.log", - self.level, self.first, self.last - ) + let kind = if self.snapshot { "snapshot" } else { "log" }; + format!("{kind}-{:08x}-{:08x}", self.session, self.frame) } fn key(&self) -> String { - format!( - "{:04x}/{:016x}-{:016x}.log", - self.level, self.first, self.last - ) + let session = descending(self.session.into()); + if self.snapshot { + format!("snapshot/{session}/{}", descending(self.frame.into())) + } else { + format!("log/{session}/{}", ascending(self.frame.into())) + } } } -/// The files a restore reads, in order: from txid 1 on, each file that -/// extends what came before, through the first gap. A file inside the range -/// already reached is skipped: tiering and bucket-acknowledged commits -/// upload concurrently, so a narrower range can land beside a wider one. -pub fn chain(files: &[File]) -> Vec { - let mut files = files.to_vec(); - files.sort_by_key(|f| (f.first, std::cmp::Reverse(f.last))); - let mut reached = 0; - let mut chain = Vec::new(); - for f in files { - if f.first > reached + 1 { - break; - } - if f.last > reached { - reached = f.last; - chain.push(f); - } +/// Hex, prefixed with its length so names sort as numbers: `a0`, `a1`, .., +/// `bff`. +fn ascending(v: u64) -> String { + let hex = format!("{v:x}"); + format!("{}{hex}", char::from(b'a' + hex.len() as u8 - 1)) +} + +/// Likewise, sorting in reverse: `zf`, `ze`, .., `y00`. +fn descending(v: u64) -> String { + let hex = format!("{v:x}"); + let digits: String = hex.bytes().map(|d| char::from(flip(d))).collect(); + format!("{}{digits}", char::from(b'z' + 1 - hex.len() as u8)) +} + +/// A hex digit's complement, `0` for `f`. +fn flip(digit: u8) -> u8 { + let value = if digit > b'9' { + digit - b'a' + 10 + } else { + digit - b'0' + }; + b"fedcba9876543210"[usize::from(value)] +} + +fn parse(name: &str, ascending: bool) -> Option { + let (&prefix, digits) = name.as_bytes().split_first()?; + let len = if ascending { + prefix.checked_sub(b'a')? + 1 + } else { + (b'z' + 1).checked_sub(prefix)? + }; + let valid = |d: &u8| d.is_ascii_digit() || (b'a'..=b'f').contains(d); + if digits.len() != usize::from(len) || !digits.iter().all(valid) { + return None; } - chain + let digits: String = digits + .iter() + .map(|&d| char::from(if ascending { d } else { flip(d) })) + .collect(); + u64::from_str_radix(&digits, 16).ok() } -/// An epoch's log in the bucket. +/// What a restore of `files`, a session's, reads, in order: the newest +/// snapshot, then each batch past it. +pub fn plan(files: &[File]) -> Vec { + let snapshot = files.iter().filter(|f| f.snapshot).max_by_key(|f| f.frame); + let from = snapshot.map_or(0, |f| f.frame); + let mut batches: Vec<_> = files + .iter() + .filter(|f| !f.snapshot && f.frame > from) + .copied() + .collect(); + batches.sort_by_key(|f| f.frame); + snapshot.into_iter().copied().chain(batches).collect() +} + +/// An object's log in the bucket. #[derive(Clone)] pub struct Log { bucket: Bucket, @@ -378,27 +470,22 @@ impl Log { Self { bucket } } - pub async fn files(&self) -> Result> { + /// The files of `session`. + pub async fn files(&self, session: u32) -> Result> { let mut files = Vec::new(); - for level in [0, SNAPSHOT_LEVEL] { - for entry in self.bucket.list(&format!("{level:04x}/")).await? { + for snapshot in [false, true] { + let kind = if snapshot { "snapshot" } else { "log" }; + let prefix = format!("{kind}/{}/", descending(session.into())); + for entry in self.bucket.list(&prefix).await? { let name = entry.key.rsplit('/').next().unwrap_or_default(); - let range = name.strip_suffix(".log").and_then(|r| r.split_once('-')); - let Some((first, last)) = range else { - continue; - }; - let (Ok(first), Ok(last)) = ( - u64::from_str_radix(first, 16), - u64::from_str_radix(last, 16), - ) else { - continue; - }; - files.push(File { - level, - first, - last, - size: 0, - }); + if let Some(frame) = parse(name, !snapshot).and_then(|f| f.try_into().ok()) { + files.push(File { + snapshot, + session, + frame, + size: 0, + }); + } } } Ok(files) @@ -421,19 +508,19 @@ impl Log { } } -/// Restores the log into `output`, which must not exist: the chain's -/// files spooled to `.restore/`, then replayed on the blocking -/// pool. The txid restored through, 0 for an empty log, which leaves no -/// file. -pub async fn restore(log: &Log, output: &Path) -> Result { - let chain = chain(&log.files().await?); - let Some(end) = chain.last().map(|f| f.last) else { +/// Restores `session` of the log into `output`, which must not exist: the +/// plan's files spooled to `.restore/`, then replayed on the +/// blocking pool. The frame the log continues from, 0 for an empty log, +/// which leaves no file. +pub async fn restore(log: &Log, session: u32, output: &Path) -> Result { + let plan = plan(&log.files(session).await?); + if plan.is_empty() { return Ok(0); - }; + } let spool = with_suffix(output, ".restore"); std::fs::create_dir_all(&spool)?; - let files: Vec<_> = (0..chain.len()) - .map(|i| (chain[i], spool.join(i.to_string()))) + let files: Vec<_> = (0..plan.len()) + .map(|i| (plan[i], spool.join(i.to_string()))) .collect(); let downloaded = stream::iter(&files) .map(|(f, path)| log.get(f, path)) @@ -445,8 +532,8 @@ pub async fn restore(log: &Log, output: &Path) -> Result { Ok(()) => { crate::owner::blocking(move || { let mut replay = Replay::new(&output)?; - for (_, path) in &files { - replay.apply(path)?; + for (f, path) in &files { + replay.apply(f, path)?; } replay.finish() }) @@ -455,7 +542,7 @@ pub async fn restore(log: &Log, output: &Path) -> Result { Err(e) => Err(e), }; let _ = std::fs::remove_dir_all(&spool); - result.map(|()| end) + result } pub fn with_suffix(path: &Path, suffix: &str) -> PathBuf { @@ -463,3 +550,42 @@ pub fn with_suffix(path: &Path, suffix: &str) -> PathBuf { path.push(suffix); path.into() } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn names_sort_as_their_numbers() { + let values = [0, 1, 0xf, 0x10, 0xff, 0x100, 0xffff_ffff]; + let up: Vec<_> = values.iter().map(|&v| ascending(v)).collect(); + let down: Vec<_> = values.iter().map(|&v| descending(v)).collect(); + assert_eq!(up[..3], ["a0", "a1", "af"]); + assert_eq!(down[..3], ["zf", "ze", "z0"]); + assert!(up.is_sorted()); + assert!(down.iter().rev().is_sorted()); + for (v, (up, down)) in values.iter().zip(up.iter().zip(&down)) { + assert_eq!(parse(up, true), Some(*v)); + assert_eq!(parse(down, false), Some(*v)); + } + assert_eq!(parse("b1", true), None); + } + + fn file(snapshot: bool, frame: Sequence) -> File { + File { + snapshot, + session: 1, + frame, + size: 0, + } + } + + #[test] + fn a_plan_is_the_newest_snapshot_and_the_batches_past_it() { + let (old, new) = (file(true, 0), file(true, 4)); + let files = [file(false, 9), file(false, 4), new, old, file(false, 6)]; + assert_eq!(plan(&files), [new, file(false, 6), file(false, 9)]); + assert_eq!(plan(&files[..2]), [file(false, 4), file(false, 9)]); + assert_eq!(plan(&[]), []); + } +} diff --git a/src/workerd/server/cluster/log_tests.rs b/src/workerd/server/cluster/log_tests.rs index e64c3a1bfa8..9b9314b42c2 100644 --- a/src/workerd/server/cluster/log_tests.rs +++ b/src/workerd/server/cluster/log_tests.rs @@ -18,10 +18,10 @@ use std::sync::OnceLock; use crate::Bucket; use crate::Error; +use crate::Sequence; use crate::capture::Capture; use crate::log::File as LogFile; use crate::log::Log; -use crate::log::SNAPSHOT_LEVEL; use crate::log::restore; use crate::owner::Owners; use crate::ownership::Claim; @@ -276,52 +276,49 @@ fn runtime() -> tokio::runtime::Runtime { .unwrap() } -/// The last captured txid. -fn last() -> u64 { +/// The frame after the capture's last commit. +fn last() -> Sequence { CAPTURE.with_borrow(|c| c.as_ref().unwrap().last().unwrap()) } -/// Upload the captured entries `first..=last` merged into one file, as -/// tiering does. -fn upload(store: &Bucket, first: u64, last: u64) { - upload_at(store, 0, first, last); -} - -/// The captured entries `first..=last` as one log file in `dir`. -fn entries(dir: &Path, first: u64, last: u64) -> (LogFile, PathBuf) { - let files: Vec> = CAPTURE.with_borrow(|c| { - let c = c.as_ref().unwrap(); - (first..=last).map(|t| c.entry(t).unwrap()).collect() - }); - let from = dir.join(format!("{first}-{last}")); - let merged = crate::log::merge(&files).unwrap(); - std::fs::write(&from, crate::log::compress(&merged).unwrap()).unwrap(); +/// The captured frames `from..to` as one batch in `dir`. +fn batch(dir: &Path, from: Sequence, to: Sequence) -> (LogFile, PathBuf) { + let bytes = CAPTURE.with_borrow(|c| c.as_ref().unwrap().batch(from, to).unwrap()); + let path = dir.join(format!("{from}-{to}")); + std::fs::write(&path, crate::log::compress(&bytes).unwrap()).unwrap(); let file = LogFile { - level: 0, - first, - last, + snapshot: false, + session: 0, + frame: to, size: 0, }; - (file, from) + (file, path) } -fn upload_at(store: &Bucket, level: i32, first: u64, last: u64) { - let dir = scratch(&format!("upload-{level}-{first}-{last}")); - let (file, mut from) = entries(&dir, first, last); - if level == SNAPSHOT_LEVEL { - // A compaction of the entries. - let out = dir.join("snapshot"); - crate::log::compact(&[(file, from)], &out).unwrap(); - from = out; - } - let file = LogFile { level, ..file }; - let log = Log::new(store.at("cell")); - runtime().block_on(log.put(&file, &from)).unwrap(); +fn log(store: &Bucket) -> Log { + Log::new(store.at("cell")) +} + +/// Upload the captured frames `from..to` as one batch, as a commit does. +fn upload(store: &Bucket, from: Sequence, to: Sequence) { + let dir = scratch(&format!("upload-{from}-{to}")); + let (file, path) = batch(&dir, from, to); + runtime().block_on(log(store).put(&file, &path)).unwrap(); +} + +/// Upload a snapshot at `to` of the captured frames, as compaction does. +fn upload_snapshot(store: &Bucket, to: Sequence) { + let dir = scratch(&format!("snapshot-{to}")); + let files = [batch(&dir, 0, to)]; + let snapshot = crate::log::compact(&files, &dir).unwrap(); + let path = dir.join(snapshot.name()); + runtime() + .block_on(log(store).put(&snapshot, &path)) + .unwrap(); } fn restored_rows(store: &Bucket, out: &Path) -> (i64, i64, i64) { - let log = Log::new(store.at("cell")); - runtime().block_on(restore(&log, out)).unwrap(); + runtime().block_on(restore(&log(store), 0, out)).unwrap(); rows(&Connection::open(out.to_str().unwrap())) } @@ -332,10 +329,10 @@ fn restore_reproduces_the_application_rows() { let store = Bucket::memory(""); insert(&app, 0, 100); let shipped = last(); - upload(&store, 1, shipped); + upload(&store, 0, shipped); insert(&app, 100, 250); app.execute("DELETE FROM t WHERE id % 7 = 0"); - upload(&store, shipped + 1, last()); + upload(&store, shipped, last()); let want = rows(&app); assert_eq!(want.0, 250 - 36); assert_eq!(restored_rows(&store, &dir.join("restored.db")), want); @@ -348,21 +345,20 @@ fn restore_reads_the_newest_snapshot_and_what_extends_it() { let store = Bucket::memory(""); insert(&app, 0, 100); let a = last(); - upload(&store, 1, a); - upload_at(&store, SNAPSHOT_LEVEL, 1, a); + upload(&store, 0, a); + upload_snapshot(&store, a); insert(&app, 100, 200); let b = last(); - upload(&store, a + 1, b); - upload_at(&store, SNAPSHOT_LEVEL, 1, b); + upload(&store, a, b); + upload_snapshot(&store, b); app.execute("DELETE FROM t WHERE id % 3 = 0"); - upload(&store, b + 1, last()); - let log = Log::new(store.at("cell")); - let files = runtime().block_on(log.files()).unwrap(); - let plan: Vec<_> = crate::log::chain(&files) + upload(&store, b, last()); + let files = runtime().block_on(log(&store).files(0)).unwrap(); + let plan: Vec<_> = crate::log::plan(&files) .iter() - .map(|f| (f.level, f.first, f.last)) + .map(|f| (f.snapshot, f.frame)) .collect(); - assert_eq!(plan, [(SNAPSHOT_LEVEL, 1, b), (0, b + 1, last())]); + assert_eq!(plan, [(true, b), (false, last())]); assert_eq!(restored_rows(&store, &dir.join("restored.db")), rows(&app)); } @@ -376,11 +372,35 @@ fn a_snapshot_inside_a_wal_carries_it_across_checkpoints() { insert(&app, from, from + 10); } let a = last(); - upload_at(&store, SNAPSHOT_LEVEL, 1, a); + upload_snapshot(&store, a); for from in (400..800).step_by(10) { insert(&app, from, from + 10); } - upload(&store, a + 1, last()); + upload(&store, a, last()); + assert_eq!(restored_rows(&store, &dir.join("restored.db")), rows(&app)); +} + +#[test] +fn a_snapshot_compacts_from_the_last_one_inside_a_wal() { + // No checkpoint: the second snapshot checkpoints a WAL that continues + // past the first, from a frame, not a WAL header. + let dir = scratch("resnapshot"); + let app = open_app(&dir.join("app.db")); + let store = Bucket::memory(""); + insert(&app, 0, 100); + let a = last(); + let files = [batch(&dir, 0, a)]; + let first = crate::log::compact(&files, &dir).unwrap(); + insert(&app, 100, 200); + let b = last(); + let files = [(first, dir.join(first.name())), batch(&dir, a, b)]; + let second = crate::log::compact(&files, &dir).unwrap(); + assert_eq!(second.frame, b); + runtime() + .block_on(log(&store).put(&second, &dir.join(second.name()))) + .unwrap(); + insert(&app, 200, 300); + upload(&store, b, last()); assert_eq!(restored_rows(&store, &dir.join("restored.db")), rows(&app)); } @@ -390,14 +410,13 @@ fn a_corrupt_file_fails_to_restore() { let app = open_app(&dir.join("app.db")); let store = Bucket::memory(""); insert(&app, 0, 100); - let (file, path) = entries(&dir, 1, last()); + let (file, path) = batch(&dir, 0, last()); let mut bytes = std::fs::read(&path).unwrap(); let mid = bytes.len() / 2; bytes[mid] ^= 1; std::fs::write(&path, bytes).unwrap(); - let log = Log::new(store.at("cell")); - runtime().block_on(log.put(&file, &path)).unwrap(); - let restored = runtime().block_on(restore(&log, &dir.join("restored.db"))); + runtime().block_on(log(&store).put(&file, &path)).unwrap(); + let restored = runtime().block_on(restore(&log(&store), 0, &dir.join("restored.db"))); assert!(restored.is_err(), "{restored:?}"); } @@ -418,13 +437,13 @@ fn a_rolled_back_transaction_leaves_nothing() { assert!(wal() > before + 100 * 4096, "no frames spilled"); app.execute("ROLLBACK;"); insert(&app, 10, 20); - upload(&store, 1, last()); + upload(&store, 0, last()); assert_eq!(rows(&app).0, 20); assert_eq!(restored_rows(&store, &dir.join("restored.db")), rows(&app)); } #[test] -fn checkpoints_and_a_restart_keep_the_chain_whole() { +fn checkpoints_and_a_restart_keep_the_log_whole() { let dir = scratch("checkpoint"); let app = open_app(&dir.join("app.db")); let store = Bucket::memory(""); @@ -435,10 +454,10 @@ fn checkpoints_and_a_restart_keep_the_chain_whole() { } app.execute("PRAGMA wal_checkpoint(TRUNCATE)"); insert(&app, 600, 650); - upload(&store, 1, last()); + upload(&store, 0, last()); assert_eq!(restored_rows(&store, &dir.join("one.db")), rows(&app)); - // A new chain from the live database, then more commits, shrinking at + // A new log from the live database, then more commits, shrinking at // the end. app.execute("PRAGMA wal_checkpoint(TRUNCATE)"); CAPTURE.with_borrow_mut(|c| c.as_mut().unwrap().restart().unwrap()); @@ -446,7 +465,7 @@ fn checkpoints_and_a_restart_keep_the_chain_whole() { app.execute("DELETE FROM t WHERE id > 20; VACUUM;"); insert(&app, 5000, 5010); let store = Bucket::memory(""); - upload(&store, 1, last()); + upload(&store, 0, last()); assert_eq!(rows(&app).0, 31); assert_eq!(restored_rows(&store, &dir.join("two.db")), rows(&app)); } @@ -471,25 +490,27 @@ fn a_restart_after_the_database_closed_reads_the_file() { assert_eq!(std::fs::metadata(dir.join("app.db-wal")).unwrap().len(), 0); CAPTURE.with_borrow_mut(|c| c.as_mut().unwrap().restart().unwrap()); let store = Bucket::memory(""); - upload(&store, 1, last()); + upload(&store, 0, last()); assert_eq!(restored_rows(&store, &dir.join("restored.db")), want); } #[test] -fn a_file_the_chain_already_covers_is_not_applied() { +fn a_batch_the_log_already_covers_is_not_applied() { // Tiering and a bucket-acknowledged commit upload concurrently, so a // stale range can land beside a newer, wider one (lost writes on the - // fleet): 3..=last-4 is inside 2..=last, and must not override it. + // fleet): the third batch is inside the second, and must not override + // it. let dir = scratch("covered"); let app = open_app(&dir.join("app.db")); let store = Bucket::memory(""); + let mut ends = Vec::new(); for from in (0..120).step_by(10) { insert(&app, from, from + 10); + ends.push(last()); } - let last = last(); - upload(&store, 1, 1); - upload(&store, 2, last); - upload(&store, 3, last - 4); + upload(&store, 0, ends[0]); + upload(&store, ends[0], ends[11]); + upload(&store, ends[1], ends[7]); assert_eq!(restored_rows(&store, &dir.join("restored.db")), rows(&app)); } @@ -506,21 +527,20 @@ fn a_deleted_database_restores_empty_then_as_recreated() { CAPTURE.with_borrow_mut(|c| c.as_mut().unwrap().reset()); let deleted = last(); let store = Bucket::memory(""); - upload(&store, 1, deleted); - let log = Log::new(store.at("cell")); + upload(&store, 0, deleted); let out = dir.join("empty.db"); - runtime().block_on(restore(&log, &out)).unwrap(); + runtime().block_on(restore(&log(&store), 0, &out)).unwrap(); assert_eq!(std::fs::metadata(&out).unwrap().len(), 0); let app = Connection::open(&format!("file:{}?vfs=capture-test", path.display())); app.execute("PRAGMA journal_mode=WAL; CREATE TABLE t (id INTEGER PRIMARY KEY, v TEXT)"); insert(&app, 100, 110); - upload(&store, deleted + 1, last()); + upload(&store, deleted, last()); assert_eq!(rows(&app).0, 10); assert_eq!(restored_rows(&store, &dir.join("split.db")), rows(&app)); - // Merged into one range, as tiering merges. + // As one batch. let store = Bucket::memory(""); - upload(&store, 1, last()); + upload(&store, 0, last()); assert_eq!(restored_rows(&store, &dir.join("merged.db")), rows(&app)); } diff --git a/src/workerd/server/cluster/owner.rs b/src/workerd/server/cluster/owner.rs index 29a1b0689ef..fe0fcf4bb2d 100644 --- a/src/workerd/server/cluster/owner.rs +++ b/src/workerd/server/cluster/owner.rs @@ -3,13 +3,12 @@ // https://opensource.org/licenses/Apache-2.0 //! The owner's side of an object: taking it over, the commit path, and -//! keeping its chain in the bucket compact. +//! keeping its log in the bucket compact. //! //! A commit is acknowledged once the bucket holds it and own.json still -//! names the epoch after the upload. Each epoch's commits form a chain of -//! log files under `do///log/e`, starting with a snapshot -//! of the state the epoch inherited; a takeover restores the last opened -//! epoch's chain. +//! names the epoch after the upload. Each epoch writes its own session of +//! the object's log, starting with a snapshot of the state it inherited; a +//! takeover restores the last opened epoch's session. use std::cell::Cell; use std::cell::RefCell; @@ -26,13 +25,12 @@ use crate::Epoch; use crate::Error; use crate::Ownership; use crate::Result; -use crate::Txid; +use crate::Sequence; use crate::capture::Capture; use crate::log::Cursor; use crate::log::File; use crate::log::Log; -use crate::log::SNAPSHOT_LEVEL; -use crate::log::chain; +use crate::log::plan; use crate::ownership::Owned; /// Objects compacting at once, and deletes in flight for one compaction. @@ -48,8 +46,8 @@ struct Slot { /// Outside the lock: `SQLite` reports WAL writes synchronously, and may /// while an upload holds the lock. capture: RefCell, - /// Through where commits are in the bucket, by epoch, then txid. - acked: tokio::sync::watch::Sender<(Epoch, Txid)>, + /// Up to where commits are in the bucket, by epoch, then frame. + acked: tokio::sync::watch::Sender<(Epoch, Sequence)>, /// Whether a commit is uploading on behalf of the others. pushing: Cell, } @@ -65,14 +63,14 @@ impl Slot { }) } - /// The bucket holds everything through `through`. - fn tiered(&self, object: &mut Object, through: Txid, file: Option) { + /// The bucket holds everything up to `through`. + fn tiered(&self, object: &mut Object, through: Sequence, file: Option) { object.tiered = object.tiered.max(through); object.files.extend(file); self.capture.borrow_mut().forget(through); } - fn ack(&self, epoch: Epoch, through: Txid) { + fn ack(&self, epoch: Epoch, through: Sequence) { self.acked.send_if_modified(|acked| { let more = (epoch, through) > *acked; *acked = (*acked).max((epoch, through)); @@ -99,12 +97,12 @@ struct Object { actor: String, epoch: Epoch, replica: Replica, - /// How far the epoch's chain in the bucket is contiguous. - tiered: Txid, + /// How far the epoch's session in the bucket is contiguous. + tiered: Sequence, /// Where the capture was at eviction. cursor: Cursor, - /// The files of this epoch's chain. Only this owner writes the epoch's - /// prefix, so compaction deletes from here rather than listing it. + /// The files of this epoch's session. Only this owner writes it, so + /// compaction deletes from here rather than listing it. files: Vec, } @@ -119,8 +117,8 @@ pub struct Owners { /// Evictions in flight. Objects written together go idle together, and /// a node's lease renewals must not wait behind all their uploads. evictions: tokio::sync::Semaphore, - /// Objects evicted with the bucket holding their epoch's chain: this - /// node's database file is that chain's state. + /// Objects evicted with the bucket holding their epoch's session: this + /// node's database file is that session's state. dormant: RefCell>, } @@ -140,8 +138,12 @@ impl Owners { } } - fn client(&self, ns: &str, actor: &str, epoch: Epoch) -> Log { - let path = format!("{}do/{ns}/{actor}/log/e{epoch}", self.bucket.prefix()); + fn client(&self, ns: &str, actor: &str) -> Log { + let path = format!( + "{}{}", + self.bucket.prefix(), + crate::object_prefix(ns, actor) + ); Log::new(self.bucket.at(&path)) } @@ -161,7 +163,7 @@ impl Owners { return Ok(()); } // Woken where it was evicted, own.json unchanged: continue the - // epoch's chain over the file, with no write to the bucket. A + // epoch's session over the file, with no write to the bucket. A // dormant entry forgotten since the claim fails this attempt; // the retry recovers at the next epoch. let dormant = self.dormant.borrow_mut().remove(&key); @@ -170,7 +172,7 @@ impl Owners { "{key}: not open here, nor dormant" ))); }; - let capture = Capture::resume(path, object.tiered, object.cursor); + let capture = Capture::resume(path, object.cursor); tracing::info!( "cluster: {ns}/{actor} epoch {} resumes at {}", object.epoch, @@ -192,35 +194,31 @@ impl Owners { remove_database(path)?; if let Some(recover) = recover { // Nothing in the bucket: an object that committed nothing. - let end = crate::log::restore(&self.client(ns, actor, recover), path).await?; + let log = self.client(ns, actor); + let end = crate::log::restore(&log, recover as u32, path).await?; tracing::info!( - "cluster: {ns}/{actor} epoch {} recovers {recover}: bucket through {end}", + "cluster: {ns}/{actor} epoch {} recovers {recover}: bucket up to {end}", owned.epoch ); } } - // The state the epoch inherits is its snapshot at txid 1, written - // from the file on the blocking pool; a new object has none. + // The state the epoch inherits is its session's snapshot at frame + // 0, written from the file on the blocking pool; a new object has + // none. let cache = open_cache(path, owned.epoch)?; - let snapshot = File { - level: SNAPSHOT_LEVEL, - first: 1, - last: 1, - size: 0, - }; let session = owned.epoch as u32; - let size = if self.opened.borrow().contains_key(&key) { + let snapshot = if self.opened.borrow().contains_key(&key) { None } else { - let (path, out) = (path.to_owned(), cache.join(snapshot.name())); - blocking(move || crate::log::snapshot_file(&path, &out, session)).await? + let (path, cache) = (path.to_owned(), cache.clone()); + blocking(move || crate::log::snapshot_file(&path, &cache, session)).await? }; - let snapshot = size.map(|size| File { size, ..snapshot }); let cursor = Cursor { session, + lane: snapshot.is_some(), ..Cursor::default() }; - let capture = Capture::resume(path, snapshot.map_or(0, |f| f.last), cursor); + let capture = Capture::resume(path, cursor); let object = self .open_epoch(ownership, ns, actor, owned, cache, snapshot) .await?; @@ -253,9 +251,9 @@ impl Owners { Ok(true) } - /// Open a claimed epoch over the recovered state. Its chain starts with - /// the state's snapshot at txid 1 (none for an empty database), which - /// the bucket holds before the epoch opens. + /// Open a claimed epoch over the recovered state. Its session starts + /// with the state's snapshot (none for an empty database), which the + /// bucket holds before the epoch opens. async fn open_epoch( &self, ownership: &Ownership, @@ -270,8 +268,9 @@ impl Owners { actor: actor.to_owned(), epoch: owned.epoch, replica: Replica { - client: self.client(ns, actor, owned.epoch), + client: self.client(ns, actor), cache, + session: owned.epoch as u32, }, tiered: 0, cursor: Cursor::default(), @@ -288,7 +287,6 @@ impl Owners { } if let Some(snapshot) = snapshot { object.files.push(upload(&object.replica, snapshot).await?); - object.tiered = 1; } if !ownership.open(ns, actor, owned.epoch).await? { return Err(Error::Contended(format!( @@ -335,11 +333,11 @@ impl Owners { let (ns, actor, epoch, round, staged, replica) = { let guard = slot.object.lock().await; let round = slot.capture.borrow().last()?; - let staged = stage(|t| slot.capture.borrow().entry(t), guard.tiered, round)?; + let staged = stage(&slot.capture.borrow(), guard.tiered, round)?; let (ns, actor) = (guard.ns.clone(), guard.actor.clone()); (ns, actor, guard.epoch, round, staged, guard.replica.clone()) }; - let file = put(&replica, 0, staged).await?; + let file = put(&replica, round, staged).await?; { let mut guard = slot.object.lock().await; if guard.epoch == epoch { @@ -355,7 +353,7 @@ impl Owners { Ok(()) } - /// Compact every owned object's chain that grew long enough. Runs off + /// Snapshot every owned object whose log outgrew its database. Runs off /// the commit path. pub async fn tier(&self) { let objects: Vec<_> = self.objects.borrow().values().cloned().collect(); @@ -375,44 +373,37 @@ impl Owners { let Ok(through) = object.capture.borrow().last() else { return; }; - let staged = stage(|t| object.capture.borrow().entry(t), guard.tiered, through); + let staged = stage(&object.capture.borrow(), guard.tiered, through); let Ok(Some(staged)) = staged else { return; }; let replica = guard.replica.clone(); drop(guard); - let Ok(file) = put(&replica, 0, Some(staged)).await else { + let Ok(file) = put(&replica, through, Some(staged)).await else { return; }; let mut guard = object.object.lock().await; object.tiered(&mut guard, through, file); } - /// Write a snapshot of an epoch's chain at `SNAPSHOT_LEVEL` and delete - /// what it covers, so a takeover's restore reads one file, not one per - /// upload. The snapshot merges the chain's files, never the live - /// database, so the merge runs off the event loop. + /// Snapshot an epoch's session from its newest snapshot and the batches + /// past it, and delete what the new one covers, so a takeover's restore + /// reads one file, not one per upload. The snapshot replays the log, + /// never the live database, so it runs off the event loop. async fn compact(&self, slot: &Slot) { - let (replica, epoch, chain, key) = { + let (replica, epoch, plan, key) = { let guard = slot.object.lock().await; - let chain = chain(&guard.files); - if chain.is_empty() { - return; - } - // A gap strands every file past it: say so, since the chain's - // prefix still compacts and nothing else would show it. - let end = guard.files.iter().map(|f| f.last).max(); let key = format!("{}/{}", guard.ns, guard.actor); - if end > chain.last().map(|f| f.last) { - tracing::warn!("cluster: {key} chain has a gap before {end:?}"); - } - (guard.replica.clone(), guard.epoch, chain, key) + (guard.replica.clone(), guard.epoch, plan(&guard.files), key) }; - let snapshot = match merged(&replica, &chain).await { + let snapshot = match merged(&replica, &plan).await { Ok(snapshot) => snapshot, Err(e) => return tracing::warn!("cluster: {key} compaction failed: {e}"), }; - let last = snapshot.last; + let from = plan.first().filter(|f| f.snapshot).map_or(0, |f| f.frame); + if snapshot.frame <= from { + return tracing::warn!("cluster: {key} log has a gap after {from}"); + } let snapshot = match upload(&replica, snapshot).await { Ok(snapshot) => snapshot, Err(e) => return tracing::warn!("cluster: {key} snapshot upload failed: {e}"), @@ -422,10 +413,10 @@ impl Owners { if guard.epoch != epoch { return; } - slot.tiered(&mut guard, last, Some(snapshot)); + slot.tiered(&mut guard, snapshot.frame, Some(snapshot)); let (replaced, kept) = std::mem::take(&mut guard.files) .into_iter() - .partition(|f| f.last <= last && *f != snapshot); + .partition(|f| f.frame <= snapshot.frame && *f != snapshot); guard.files = kept; replaced }; @@ -490,8 +481,8 @@ impl Owners { self.opened.borrow_mut().remove(&key); } - /// An owned object's database was deleted: its chain continues with an - /// empty database, which the next commit makes durable. + /// An owned object's database was deleted: its session continues with + /// an empty database, which the next commit makes durable. pub fn database_reset(&self, ns: &str, actor: &str) { if let Some(slot) = self.objects.borrow().get(&object_key(ns, actor)) { slot.capture.borrow_mut().reset(); @@ -506,54 +497,28 @@ impl Owners { } } -/// One log file covering txids `first..=last`, ready to upload. -struct Staged { - first: Txid, - last: Txid, - bytes: Vec, -} - -/// The capture's entries after `from` through `through`, as one log file. -fn stage( - entry: impl Fn(Txid) -> Result>, - from: Txid, - through: Txid, -) -> Result> { - if through <= from { +/// The capture's frames from `from` up to `to`, as one batch. +fn stage(capture: &Capture, from: Sequence, to: Sequence) -> Result>> { + if to <= from { return Ok(None); } - let files = (from + 1..=through) - .map(entry) - .collect::>>()?; - Ok(Some(Staged { - first: from + 1, - last: through, - bytes: crate::log::compress(&crate::log::merge(&files)?)?, - })) + Ok(Some(crate::log::compress(&capture.batch(from, to)?)?)) } -/// `chain`'s files replayed into this node's copy of a snapshot covering -/// txid 1 through the chain's end, on the blocking pool. A missing copy is -/// fetched from the bucket first. -async fn merged(replica: &Replica, chain: &[File]) -> Result { - let paths: Vec<_> = chain.iter().map(|f| replica.cache.join(f.name())).collect(); - for (f, path) in chain.iter().zip(&paths) { +/// `plan`'s files replayed into this node's copy of a snapshot, on the +/// blocking pool. A missing copy is fetched from the bucket first. +async fn merged(replica: &Replica, plan: &[File]) -> Result { + let paths: Vec<_> = plan.iter().map(|f| replica.cache.join(f.name())).collect(); + for (f, path) in plan.iter().zip(&paths) { if !path.exists() { let tmp = tmp_path(path); replica.client.get(f, &tmp).await?; std::fs::rename(&tmp, path)?; } } - let snapshot = File { - level: SNAPSHOT_LEVEL, - first: 1, - last: chain.last().map_or(0, |f| f.last), - size: 0, - }; - let out = replica.cache.join(snapshot.name()); - let files: Vec<_> = chain.iter().copied().zip(paths).collect(); - let size = blocking(move || crate::log::compact(&files, &out)).await?; - Ok(File { size, ..snapshot }) + let files: Vec<_> = plan.iter().copied().zip(paths).collect(); + let cache = replica.cache.clone(); + blocking(move || crate::log::compact(&files, &cache)).await } /// Writes a copy whole and renames it into place, so a reader never sees @@ -582,17 +547,17 @@ pub async fn blocking( .map_err(|e| Error::Replica(format!("blocking task: {e}")))? } -/// Where an epoch's chain is: the bucket, and this node's copy of it. +/// Where an epoch's session is: the bucket, and this node's copy of it. #[derive(Clone)] struct Replica { client: Log, /// `.log/e`. cache: PathBuf, + session: u32, } /// The copies' directory for `epoch` of the database at `path`, created -/// empty. Other epochs' copies are removed: a file name repeats in every -/// epoch's chain. +/// empty. Other epochs' copies are removed. fn open_cache(path: &Path, epoch: Epoch) -> Result { let root = cache_root(path); remove_dir(&root)?; @@ -614,22 +579,18 @@ fn remove_dir(path: &Path) -> Result<()> { } } -/// L0 files a snapshot waits for, so a small object does not snapshot on -/// nearly every commit. -const SNAPSHOT_MIN_FILES: usize = 32; - -/// Whether `files` call for a snapshot: the L0 bytes since the last one -/// reach its size, so bucket writes stay within about twice -/// the log whatever the database's size. +/// Whether `files`, a session's, call for a snapshot: the log since the +/// last one reached its size, so bucket writes stay within about twice the +/// log whatever the database's size. fn snapshot_due(files: &[File]) -> bool { - let image: u64 = files - .iter() - .filter(|f| f.level == SNAPSHOT_LEVEL) - .map(|f| f.size) - .max() - .unwrap_or(0); - let l0: Vec<_> = files.iter().filter(|f| f.level == 0).collect(); - l0.len() >= SNAPSHOT_MIN_FILES && l0.iter().map(|f| f.size).sum::() >= image + let (image, log) = plan(files).iter().fold((0, 0), |(image, log), f| { + if f.snapshot { + (f.size, log) + } else { + (image, log + f.size) + } + }); + log > 0 && log >= image } /// Uploads this node's copy of `file`. @@ -639,15 +600,16 @@ async fn upload(replica: &Replica, file: File) -> Result { Ok(file) } -/// Keeps `staged`, if any, as this node's copy at `level`, then uploads it. -async fn put(replica: &Replica, level: i32, staged: Option) -> Result> { - let Some(Staged { first, last, bytes }) = staged else { +/// Keeps `staged`, if any, as this node's copy of the batch up to `to`, +/// then uploads it. +async fn put(replica: &Replica, to: Sequence, staged: Option>) -> Result> { + let Some(bytes) = staged else { return Ok(None); }; let file = File { - level, - first, - last, + snapshot: false, + session: replica.session, + frame: to, size: bytes.len() as u64, }; let path = replica.cache.join(file.name()); @@ -677,48 +639,24 @@ fn remove_database(path: &Path) -> Result<()> { mod tests { use super::*; - fn file(level: i32, size: u64) -> File { + fn file(snapshot: bool, frame: Sequence, size: u64) -> File { File { - level, - first: 1, - last: 1, + snapshot, + session: 1, + frame, size, } } - fn span(level: i32, first: Txid, last: Txid) -> File { - File { - level, - first, - last, - size: 0, - } - } - - #[test] - fn a_chain_is_what_a_restore_reads() { - let snapshot = span(SNAPSHOT_LEVEL, 1, 4); - let (narrow, wide) = (span(0, 5, 6), span(0, 5, 8)); - let chain = chain(&[span(0, 9, 9), narrow, snapshot, wide]); - assert_eq!(chain, vec![snapshot, wide, span(0, 9, 9)]); - let gap = super::chain(&[snapshot, span(0, 6, 6)]); - assert_eq!(gap, vec![snapshot], "through the gap"); - assert_eq!(super::chain(&[]), vec![]); - } - #[test] fn a_snapshot_is_due_when_the_log_reaches_its_size() { - let mut files = vec![file(SNAPSHOT_LEVEL, 1000)]; - files.extend((0..SNAPSHOT_MIN_FILES).map(|_| file(0, 30))); - assert!( - !snapshot_due(&files), - "960 bytes of log under a 1000-byte image" - ); - files.push(file(0, 40)); + let mut files = vec![file(true, 0, 1000), file(false, 3, 960)]; + assert!(!snapshot_due(&files), "960 bytes of log, a 1000-byte image"); + files.push(file(false, 5, 40)); assert!(snapshot_due(&files)); - let few: Vec<_> = (0..SNAPSHOT_MIN_FILES - 1) - .map(|_| file(0, 1 << 20)) - .collect(); - assert!(!snapshot_due(&few), "fewer files than the floor"); + files.push(file(true, 5, 1000)); + assert!(!snapshot_due(&files), "a snapshot covers the log"); + assert!(snapshot_due(&[file(false, 2, 1)]), "no snapshot yet"); + assert!(!snapshot_due(&[])); } } diff --git a/src/workerd/server/cluster/ownership.rs b/src/workerd/server/cluster/ownership.rs index 2bfc86491b6..849e56c1983 100644 --- a/src/workerd/server/cluster/ownership.rs +++ b/src/workerd/server/cluster/ownership.rs @@ -23,7 +23,7 @@ use crate::registry::NodeRecord; struct OwnerRecord { /// The owning instance (`NodeRecord::node`); empty for none. node: String, - /// The owner's epoch: its log is `log/e/`. + /// The owner's epoch: its session of the log. epoch: u64, /// The last epoch that opened. A claim writes the next `epoch` and /// copies `from` from the record it replaced; the opening write sets it @@ -106,7 +106,7 @@ const ROUTE_REFRESH_MS: u64 = 5_000; const ATTEMPTS: usize = 8; fn owner_key(namespace: &str, actor: &str) -> String { - format!("do/{namespace}/{actor}/own.json") + format!("{}/own.json", crate::object_prefix(namespace, actor)) } impl Ownership { diff --git a/src/workerd/server/tests/cluster/cluster-test.mjs b/src/workerd/server/tests/cluster/cluster-test.mjs index ef2cf84d4a5..6ab25845479 100644 --- a/src/workerd/server/tests/cluster/cluster-test.mjs +++ b/src/workerd/server/tests/cluster/cluster-test.mjs @@ -23,7 +23,7 @@ import { spawn } from 'node:child_process'; import { mkdir, mkdtemp, readdir, rm } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { connect, createServer } from 'node:net'; -import { createHmac, randomBytes } from 'node:crypto'; +import { createHash, createHmac, randomBytes } from 'node:crypto'; import { join } from 'node:path'; import { env } from 'node:process'; import { test } from 'node:test'; @@ -280,7 +280,7 @@ class FakeS3 { // The key of the node that owns the Counter with this ID, or null when no // owner record exists. async owner(id) { - const record = await this.get(`do/counter-test-namespace/${id}/own.json`); + const record = await this.get(`${objectPrefix(id)}/own.json`); return record === null ? null : JSON.parse(record).node; } @@ -622,12 +622,24 @@ test('cluster: concurrent first requests to a DO read its owner record once per Array.from({ length: 64 }, (_, i) => i + 1) ); const requests = await bucket.requests( - `do/counter-test-namespace/${results[0].body.id}/own.json` + `${objectPrefix(results[0].body.id)}/own.json` ); assert(requests <= 8, `${requests} owner record requests for one cold DO`); }); }); +// Where a Durable Object's records are in the bucket: a hash of its +// namespace's uniqueKey, then its ID. +function objectPrefix(id, uniqueKey = 'counter-test-namespace') { + const hash = createHash('sha256'); + for (const part of ['workerd cluster namespace v1', '', uniqueKey]) { + const length = Buffer.alloc(8); + length.writeBigUInt64BE(BigInt(Buffer.byteLength(part))); + hash.update(length).update(part); + } + return `do/${hash.digest().subarray(0, 16).toString('base64url')}/${id}`; +} + // A port that was free a moment ago. async function freePort() { const server = createServer(); @@ -778,11 +790,12 @@ test('cluster: a DO reopened where it was closed resumes its epoch', async () => await sleep(500); const after = await fetchJson(node.httpPort, '/increment?name=dormant'); assert.strictEqual(after.body.count, 3, JSON.stringify(after.body)); - const prefix = `do/counter-test-namespace/${id}`; + const prefix = objectPrefix(id); const record = JSON.parse(await bucket.get(`${prefix}/own.json`)); assert.strictEqual(record.epoch, 1, JSON.stringify(record)); - assert.deepStrictEqual(await bucket.list(`${prefix}/ltx/e2/`), []); - assert.match(node.stderr, /epoch 1 resumes at 2/); + // No session 2, whose name is `zd`. + assert.deepStrictEqual(await bucket.list(`${prefix}/snapshot/zd/`), []); + assert.match(node.stderr, /epoch 1 resumes at \d+/); }); }); From e8f8382bb64ee99441b28907a4ba06e1c36da68c Mon Sep 17 00:00:00 2001 From: Ryan Dahl Date: Tue, 27 Oct 2026 15:56:08 -0700 Subject: [PATCH 14/38] cluster: hand a draining node's objects to peers at once A draining node kept serving its own objects: the owner check came before the drain check, so it reopened them for any request that reached it. It also stayed listed until its last connection finished, so peers routed its objects to a node that no longer accepted them, for up to a lease. Now a draining node refuses its own objects as it refuses claims, and deregisters when the drain starts: peers drop it at their next listing and take its objects over. A renewal in flight at that moment may find the record gone; once draining, its outcome is ignored rather than read as a fence. --- src/workerd/server/cluster/cluster.rs | 5 +++ src/workerd/server/cluster/ownership.rs | 10 ++++- src/workerd/server/server/run.rs | 13 +++--- .../server/tests/cluster/cluster-test.mjs | 43 +++++++++++++++++++ 4 files changed, 63 insertions(+), 8 deletions(-) diff --git a/src/workerd/server/cluster/cluster.rs b/src/workerd/server/cluster/cluster.rs index fcc792aa838..0984d3a7982 100644 --- a/src/workerd/server/cluster/cluster.rs +++ b/src/workerd/server/cluster/cluster.rs @@ -257,6 +257,11 @@ impl Cluster { Ok(Err(e)) => Err(e.to_string()), Err(_) => Err("renewal timed out".into()), }; + // A drain began meanwhile, and its deregistration may have + // failed this renewal: what it found no longer matters. + if self.lease.borrow().draining { + continue; + } step = self.lease.borrow_mut().renewed(now_ms(), result); } let delay = match step { diff --git a/src/workerd/server/cluster/ownership.rs b/src/workerd/server/cluster/ownership.rs index 849e56c1983..ab0051084d9 100644 --- a/src/workerd/server/cluster/ownership.rs +++ b/src/workerd/server/cluster/ownership.rs @@ -105,6 +105,10 @@ const ROUTE_REFRESH_MS: u64 = 5_000; /// How many times a claim retries when another instance keeps winning. const ATTEMPTS: usize = 8; +fn draining(key: &str) -> Error { + Error::Replica(format!("{key}: this node is draining")) +} + fn owner_key(namespace: &str, actor: &str) -> String { format!("{}/own.json", crate::object_prefix(namespace, actor)) } @@ -159,6 +163,10 @@ impl Ownership { Some((body, version)) => { let record: OwnerRecord = serde_json::from_slice(&body)?; if record.node == me { + // A draining node lets its objects go, its own too. + if !self.may_claim.get() { + return Err(draining(&key)); + } return Ok(Claim::Owned(record.owned())); } if !record.node.is_empty() @@ -178,7 +186,7 @@ impl Ownership { } }; if !self.may_claim.get() { - return Err(Error::Replica(format!("{key}: this node is draining"))); + return Err(draining(&key)); } let record = previous.successor(me); // A new object opens at its claim, where a loser still routes diff --git a/src/workerd/server/server/run.rs b/src/workerd/server/server/run.rs index 8e1ea48d24f..b39664130bc 100644 --- a/src/workerd/server/server/run.rs +++ b/src/workerd/server/server/run.rs @@ -405,19 +405,18 @@ pub async fn run( match futures::future::select(serving, drain).await { Either::Left((result, _)) => result?, Either::Right(((), serving)) => { - // A draining instance claims nothing more and lets its objects go, so peers take - // them over while its connections finish. + // A draining instance claims nothing more, lets its objects go, and leaves the + // cluster, so peers take them over at their next listing while its connections + // finish. if let Some(node) = server.cluster() { node.cluster.start_draining(); server.abort_all_actors(None); + if let Err(e) = node.cluster.deregister().await { + tracing::warn!("leaving the cluster: {e}"); + } } draining.send_replace(true); serving.await?; - if let Some(node) = server.cluster() - && let Err(e) = node.cluster.deregister().await - { - tracing::warn!("leaving the cluster: {e}"); - } } } diff --git a/src/workerd/server/tests/cluster/cluster-test.mjs b/src/workerd/server/tests/cluster/cluster-test.mjs index 6ab25845479..5f5277d3766 100644 --- a/src/workerd/server/tests/cluster/cluster-test.mjs +++ b/src/workerd/server/tests/cluster/cluster-test.mjs @@ -605,6 +605,49 @@ test( } ); +test( + "cluster: a peer takes over a draining owner's DO before its lease ends", + SIGTERM_DRAINS, + async () => { + await withCluster(2, async ({ nodes }) => { + const initial = await fetchJson( + nodes[0].httpPort, + '/increment?name=handoff' + ); + assert.strictEqual(initial.status, 200); + const owner = nodes.find((n) => n.nodeId === initial.body.nodeId); + const peer = nodes.find((n) => n.nodeId !== initial.body.nodeId); + + // A slow request holds the owner draining for longer than a peer + // needs to see it leave, but within its lease. + const holding = fetchJson( + owner.httpPort, + '/increment?name=hold&delay=12000', + { timeoutMs: 20_000 } + ); + await sleep(250); + const drainedAt = Date.now(); + const exitPromise = owner.stop({ signal: 'SIGTERM', timeoutMs: 20_000 }); + + // A request waits on the owner until the peer's next listing drops + // it, then the peer claims. One timed out would still increment. + let response = null; + while (Date.now() < drainedAt + 10_000) { + response = await fetchJson(peer.httpPort, '/increment?name=handoff', { + timeoutMs: drainedAt + 10_000 - Date.now(), + }).catch(() => null); + if (response?.status === 200) break; + await sleep(200); + } + assert.strictEqual(response?.status, 200, JSON.stringify(response?.body)); + assert.strictEqual(response.body.nodeId, peer.nodeId); + assert.strictEqual(response.body.count, 2); + await holding.catch(() => {}); + assert.deepStrictEqual(await exitPromise, { code: 0, signal: null }); + }); + } +); + test('cluster: concurrent first requests to a DO read its owner record once per node', async () => { await withCluster(2, async ({ nodes, bucket }) => { // Without coalescing, each of these reads own.json and the losers retry From 2a06e37a91aa038846a849aabc79e03dc1f28b50 Mon Sep 17 00:00:00 2001 From: Ryan Dahl Date: Tue, 27 Oct 2026 15:57:01 -0700 Subject: [PATCH 15/38] cluster: use the cluster key only through labelled derivations Drop the unused accessor that handed out the raw key. --- src/workerd/server/cluster/cluster.rs | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/src/workerd/server/cluster/cluster.rs b/src/workerd/server/cluster/cluster.rs index 0984d3a7982..6c1a1936792 100644 --- a/src/workerd/server/cluster/cluster.rs +++ b/src/workerd/server/cluster/cluster.rs @@ -106,10 +106,6 @@ impl Cluster { self.registry.node() } - pub fn key(&self) -> &[u8] { - &self.key - } - pub async fn renew(&self, now_ms: u64) -> Result { self.registry.renew(now_ms).await } @@ -231,7 +227,9 @@ impl Cluster { self.ownership.refresh_route(ns, actor, now_ms).await } - /// The MAC a peer call carries. + /// The MAC a peer call carries. The cluster key is used only through + /// labelled derivations, so no two uses share a key: this, with each + /// message labelled, and the channel token key. pub fn mac(&self, data: &[u8]) -> Vec { hmac_sha256(&self.key, data) } From a3ebfc4361b4ad5a7d816444615c368bf40cda98 Mon Sep 17 00:00:00 2001 From: Ryan Dahl Date: Tue, 27 Oct 2026 16:29:32 -0700 Subject: [PATCH 16/38] cluster: log pages, merged per batch, in an SRS-compatible lane Logging the WAL's bytes put a page a hot object rewrites in the log once per commit: a batch of grouped commits carried every copy, and the log reached the database's size, and so a snapshot, as fast as commits came. The log is a `sqlitePages` lane instead, workerd's own lane type at new ordinals of SRS's schema, which is otherwise unchanged. A commit is a frame of the pages it wrote; a batch merges each run of them to one frame with the newest version of each page, covering the commits `first..=sequence`. A hundred updates of one row log one page, not a hundred: 4 KB instead of 418 KB, 199 bytes instead of 3.3 KB after LZ4. A new database is still SRS's `initialize`, snapshots still SRS's header and the database file, and compaction still SRS's rule. --- src/workerd/server/cluster/capture.rs | 90 +++++---- src/workerd/server/cluster/frames.capnp | 44 +++-- src/workerd/server/cluster/log.rs | 231 ++++++++++++------------ src/workerd/server/cluster/log_tests.rs | 31 +++- src/workerd/server/cluster/owner.rs | 5 +- 5 files changed, 222 insertions(+), 179 deletions(-) diff --git a/src/workerd/server/cluster/capture.rs b/src/workerd/server/cluster/capture.rs index ad6fb22e455..dca996b49bb 100644 --- a/src/workerd/server/cluster/capture.rs +++ b/src/workerd/server/cluster/capture.rs @@ -3,19 +3,17 @@ // https://opensource.org/licenses/Apache-2.0 //! A Durable Object's commits, cut from the writes `SQLite` makes to its WAL -//! as workerd's VFS reports them: per transaction, the frames of -//! `frames.capnp` that write its bytes to the database lane. +//! as workerd's VFS reports them: per transaction, a frame of +//! `frames.capnp` with the pages it wrote. //! //! `SQLite` writes a frame as its 24-byte header and then its page, at offsets //! that follow the 32-byte WAL header, and the header of a transaction's //! last frame carries the database's size, which marks the commit. A write //! at offset 0 starts a new WAL, after a checkpoint has copied every -//! earlier frame into the database file. So the bytes since the last commit -//! are buffered, and the write that completes a commit frame completes one -//! transaction: a write of those bytes at the lane's next offset, which -//! keeps growing across checkpoints, each marked by a consume. A -//! rolled-back transaction's frames start where the next one's do, which -//! overwrites them. +//! earlier frame into the database file. So the frames since the last +//! commit are buffered, and the write that completes a commit frame +//! completes one transaction. A rolled-back transaction's frames start +//! where the next one's do, which overwrites them. use std::collections::BTreeMap; use std::path::Path; @@ -29,7 +27,7 @@ use crate::log::Cursor; use crate::log::push; const WAL_HEADER: u64 = 32; -const FRAME_HEADER: u64 = 24; +const FRAME_HEADER: usize = 24; pub struct Capture { /// The database file. @@ -38,14 +36,11 @@ pub struct Capture { /// frame after it. entries: BTreeMap>, cursor: Cursor, - page_size: u64, - /// Where in the WAL the next transaction's bytes start; `None` before a - /// WAL header. - start: Option, + page_size: usize, + /// Where the next transaction's first frame goes: 0 before a WAL header. + start: u64, /// What was written from `start` on. - pending: Vec, - /// A checkpoint consumed the lane's WAL since the last commit. - consumed: bool, + frames: Vec, /// Why the capture can no longer be trusted, if it cannot. lost: Option, } @@ -70,9 +65,8 @@ impl Capture { entries: BTreeMap::new(), cursor, page_size: 0, - start: None, - pending: Vec::new(), - consumed: false, + start: 0, + frames: Vec::new(), lost: None, } } @@ -82,7 +76,6 @@ impl Capture { #[cfg(test)] pub fn restart(&mut self) -> Result<()> { self.entries.clear(); - self.start = None; self.cursor = Cursor { session: self.cursor.session, ..Cursor::default() @@ -103,9 +96,8 @@ impl Capture { /// The database's files were deleted: the session continues without the /// lane, and the next commit creates it. pub fn reset(&mut self) { - self.start = None; - self.pending.clear(); - self.consumed = false; + self.start = 0; + self.frames.clear(); if !self.cursor.lane { return; } @@ -119,36 +111,35 @@ impl Capture { if offset == 0 { // A new WAL: a checkpoint copied the last one into the file. if let Some(size) = data.get(8..12) { - self.page_size = - u64::from(u32::from_be_bytes([size[0], size[1], size[2], size[3]])); - self.consumed |= self.cursor.end > self.cursor.base; - self.start = Some(0); - self.pending.clear(); + self.page_size = u32::from_be_bytes([size[0], size[1], size[2], size[3]]) as usize; + self.start = WAL_HEADER; + self.frames.clear(); } + return; } - let Some(start) = self.start.filter(|&s| offset >= s) else { - self.lost = Some(format!("a WAL write at {offset}, before {:?}", self.start)); + if self.start == 0 || offset < self.start { + self.lost = Some(format!("a WAL write at {offset}, before {}", self.start)); return; - }; - let at = usize::try_from(offset - start).unwrap_or(usize::MAX); + } + let at = usize::try_from(offset - self.start).unwrap_or(usize::MAX); let end = at.saturating_add(data.len()); - if self.pending.len() < end { - self.pending.resize(end, 0); + if self.frames.len() < end { + self.frames.resize(end, 0); } - self.pending[at..end].copy_from_slice(data); + self.frames[at..end].copy_from_slice(data); let size = FRAME_HEADER + self.page_size; - let wal_end = start + end as u64; - if wal_end <= WAL_HEADER || !(wal_end - WAL_HEADER).is_multiple_of(size) { + if !end.is_multiple_of(size) { return; } - let header = &self.pending[end - size as usize..]; - if u32::from_be_bytes([header[4], header[5], header[6], header[7]]) != 0 { - self.commit(start, end); + let header = &self.frames[end - size..end - size + FRAME_HEADER]; + let commit = u32::from_be_bytes([header[4], header[5], header[6], header[7]]); + if commit != 0 { + self.commit(end, commit); } } - /// The bytes before `end` are one committed transaction. - fn commit(&mut self, start: u64, end: usize) { + /// The frames before `end` are one committed transaction. + fn commit(&mut self, end: usize, commit: u32) { let mut entry = Vec::new(); if !self.cursor.lane { match std::fs::read(&self.path) { @@ -159,14 +150,15 @@ impl Capture { Err(e) => self.lost = Some(format!("the new database: {e}")), } } - if std::mem::take(&mut self.consumed) { - push(&mut entry, &mut self.cursor, Body::Consume); + let mut pages = BTreeMap::new(); + for frame in self.frames[..end].chunks_exact(FRAME_HEADER + self.page_size) { + let pgno = u32::from_be_bytes([frame[0], frame[1], frame[2], frame[3]]); + pages.insert(pgno, &frame[FRAME_HEADER..]); } - let at = self.cursor.base + start; - let bytes = &self.pending[..end]; - push(&mut entry, &mut self.cursor, Body::Write(at, bytes)); - self.start = Some(start + end as u64); - self.pending.drain(..end); + pages.retain(|&pgno, _| pgno <= commit); + push(&mut entry, &mut self.cursor, Body::Pages(commit, &pages)); + self.start += end as u64; + self.frames.drain(..end); self.entries.insert(self.cursor.sequence, entry); } diff --git a/src/workerd/server/cluster/frames.capnp b/src/workerd/server/cluster/frames.capnp index 9f3f4f6371e..12eb61f23c8 100644 --- a/src/workerd/server/cluster/frames.capnp +++ b/src/workerd/server/cluster/frames.capnp @@ -4,9 +4,10 @@ @0xf699edadaece33b7; # The format of a Durable Object's log in the bucket: the storage relay's -# (SRS's), field for field, so the two stay wire-compatible. Workerd writes -# one `sqlite` lane and leaves unset what it has no use for, noted below. -# Compression and encryption are not part of it. +# (SRS's), field for field, so the two stay wire-compatible, and one lane +# type of workerd's own, `sqlitePages`, at new ordinals. Workerd leaves +# unset what it has no use for, noted below. Compression and encryption are +# not part of it. # # A log batch is a run of frames, each one message, that starts and ends at # a commit. A snapshot is a `SnapshotHeader` and then each lane's content in @@ -50,7 +51,7 @@ struct Frame { write :group { # For a `sqlite` lane, bytes SQLite wrote to the WAL, at an offset # that keeps growing across checkpoints, though SQLite sees each WAL - # start over at zero. + # start over at zero. Unused by workerd. offset @5 :UInt64; data @6 :Data; } @@ -59,19 +60,36 @@ struct Frame { consume :group { # For a `sqlite` lane, a checkpoint copied the WAL's committed pages - # into the database: data before `offset` is not needed again. + # into the database: data before `offset` is not needed again. Unused + # by workerd. offset @8 :UInt64; } initialize @9 :Data; - # For a `sqlite` lane, the whole database file, written before the WAL - # is in use. + # For a `sqlite` or `sqlitePages` lane, the whole database file, + # written before the WAL is in use. setReplicaInformation @12 :AnyPointer; # Unused by workerd. + + pages :group { + # For a `sqlitePages` lane, the newest version of each page that the + # commits `first..=sequence` wrote, and the database's size after, in + # pages. A batch's commits merge to one such frame between other + # frames, so a page a hot object rewrites is in the log once per + # upload, not once per commit. + first @15 :UInt32; + commit @16 :UInt32; + list @17 :List(Page); + } } } +struct Page { + number @0 :UInt32; + data @1 :Data; +} + struct LaneDescription { name @0 :Text; # The lane's stable identifier; numbers differ between objects. @@ -82,6 +100,8 @@ struct LaneDescription { # Unused by workerd. replicaInformation @3 :Void; # Unused by workerd. + sqlitePages @4 :Void; + # Workerd's: a SQLite database whose writes are pages (`Frame.pages`). } } @@ -121,18 +141,12 @@ struct SnapshotHeader { # A lane number freed while later ones are in use. sqlite :group { + # Also a `sqlitePages` lane's, with only `databaseSize` set: the + # database file, every committed page in it. databaseSize @2 :UInt64; walSize @3 :UInt64; - # The WAL bytes after the database's. Always 0 in the bucket: a - # snapshot's database has every committed page. - walBaseOffset @6 :UInt64; - # Where the log's next write continues. Workerd snapshots from the - # log, not the live database, so that write can continue a WAL - # past its header, at a frame. - walShiftOffset @7 :UInt64; - # Always 0: workerd keeps no lane files of its own to shift. } journal :group { diff --git a/src/workerd/server/cluster/log.rs b/src/workerd/server/cluster/log.rs index 5c0d0c66167..76f01fb8c24 100644 --- a/src/workerd/server/cluster/log.rs +++ b/src/workerd/server/cluster/log.rs @@ -3,13 +3,13 @@ // https://opensource.org/licenses/Apache-2.0 //! An object's log in the bucket (`frames.capnp`), laid out as the storage -//! relay's: one `sqlite` lane, the database, whose frames are the bytes -//! each commit wrote to the WAL. +//! relay's: one `sqlitePages` lane, the database, whose frames are the +//! pages each commit wrote. //! //! Under `do///`, each file one LZ4 frame: //! -//! - `log//`: a batch, the frames of a run of commits, up to -//! frame `end`. +//! - `log//`: a batch, the frames of a run of commits up to +//! frame `end`, each run of page frames merged to one. //! - `snapshot//`: a snapshot, the log continuing from //! `frame`. //! @@ -18,6 +18,7 @@ //! newest snapshot and the batches after it; a frame a batch before //! already applied is skipped by its sequence. +use std::collections::BTreeMap; use std::io::BufReader; use std::io::Read; use std::io::Seek; @@ -42,63 +43,110 @@ use crate::Sequence; const LANE: &str = "db"; /// Downloads in flight during a restore, to hide the store's round trips. const RESTORE_CONCURRENCY: usize = 16; -const WAL_HEADER: usize = 32; -const FRAME_HEADER: usize = 24; -/// Where a session's next frame goes, and the database lane's state. +/// Where a session's next frame goes. #[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] pub struct Cursor { pub session: u32, pub sequence: Sequence, - /// Whether the lane exists. + /// Whether the database lane exists. pub lane: bool, - /// Where the lane's next write goes. - pub end: u64, - /// Where the current WAL starts. - pub base: u64, } +/// Pages by number. +pub type Pages<'a> = BTreeMap; + #[derive(Clone, Copy)] pub enum Body<'a> { NewLane, DeleteLane, Initialize(&'a [u8]), - Write(u64, &'a [u8]), - /// A checkpoint consumed everything written so far. - Consume, + /// A commit's pages, and the database's size after, in pages. + Pages(u32, &'a Pages<'a>), } /// Appends a frame to `out` at `cursor`, and moves it past. pub fn push(out: &mut Vec, cursor: &mut Cursor, body: Body) { + match body { + Body::NewLane => cursor.lane = true, + Body::DeleteLane => cursor.lane = false, + _ => {} + } + let (session, sequence) = (cursor.session, cursor.sequence); + frame_at(out, session, sequence..=sequence, body); + cursor.sequence += 1; +} + +/// Appends a frame covering `sequences` to `out`. +fn frame_at( + out: &mut Vec, + session: u32, + sequences: std::ops::RangeInclusive, + body: Body, +) { let mut message = capnp::message::Builder::new_default(); let mut f = message.init_root::(); - f.set_session(cursor.session); - f.set_sequence(cursor.sequence); + f.set_session(session); + f.set_sequence(*sequences.end()); match body { Body::NewLane => { let mut lane = f.init_new_lane(); lane.set_name(LANE); - lane.set_sqlite(()); - cursor.lane = true; - } - Body::DeleteLane => { - f.set_delete_lane(()); - (cursor.lane, cursor.end, cursor.base) = (false, 0, 0); + lane.set_sqlite_pages(()); } + Body::DeleteLane => f.set_delete_lane(()), Body::Initialize(image) => f.set_initialize(image), - Body::Write(offset, data) => { - let mut write = f.init_write(); - write.set_offset(offset); - write.set_data(data); - cursor.end = offset + data.len() as u64; - } - Body::Consume => { - f.init_consume().set_offset(cursor.end); - cursor.base = cursor.end; + Body::Pages(commit, pages) => { + let mut group = f.init_pages(); + group.set_first(*sequences.start()); + group.set_commit(commit); + let mut list = group.init_list(pages.len() as u32); + for (i, (&number, data)) in pages.iter().enumerate() { + let mut page = list.reborrow().get(i as u32); + page.set_number(number); + page.set_data(data); + } } } let _ = capnp::serialize::write_message(&mut *out, &message); - cursor.sequence += 1; +} + +/// `batch`, a run of frames, with each run of page frames merged to one +/// with the newest version of each page. +pub fn merge(batch: &[u8]) -> Result> { + let mut messages = Vec::new(); + let mut r = batch; + while let Some(message) = capnp::serialize::try_read_message(&mut r, options())? { + messages.push(message); + } + let mut out = Vec::new(); + // The run's session, first and last sequence, size, and pages. + let mut run: Option<(u32, Sequence, Sequence, u32, Pages)> = None; + let flush = |out: &mut Vec, run: &mut Option<(u32, Sequence, Sequence, u32, Pages)>| { + if let Some((session, first, last, commit, mut pages)) = run.take() { + pages.retain(|&number, _| number <= commit); + frame_at(out, session, first..=last, Body::Pages(commit, &pages)); + } + }; + for message in &messages { + let f = message.get_root::()?; + let (session, sequence) = (f.get_session(), f.get_sequence()); + if let frame::Pages(group) = f.which()? { + let first = group.get_first(); + let r = run.get_or_insert_with(|| (session, first, sequence, 0, BTreeMap::new())); + (r.2, r.3) = (sequence, group.get_commit()); + for page in group.get_list()? { + r.4.insert(page.get_number(), page.get_data()?); + } + } else { + flush(&mut out, &mut run); + let mut copy = capnp::message::Builder::new_default(); + copy.set_root(f)?; + capnp::serialize::write_message(&mut out, ©)?; + } + } + flush(&mut out, &mut run); + Ok(out) } /// LZ4-frames `bytes`, as every file in the bucket is. @@ -119,20 +167,10 @@ fn options() -> capnp::message::ReaderOptions { *capnp::message::ReaderOptions::new().traversal_limit_in_words(None) } -fn be32(b: &[u8]) -> u32 { - u32::from_be_bytes([b[0], b[1], b[2], b[3]]) -} - /// The database lane, replayed into a file. pub struct Replay { db: std::fs::File, live: bool, - /// The WAL bytes not yet checkpointed, from `base` on. - wal: Vec, - base: u64, - /// Whether `wal` starts with a WAL header, rather than a frame past a - /// snapshot. - header: bool, /// The next frame, by session and sequence. next: Option<(u32, Sequence)>, log_bytes: u64, @@ -148,9 +186,6 @@ impl Replay { .truncate(true) .open(db)?, live: false, - wal: Vec::new(), - base: 0, - header: true, next: None, log_bytes: 0, }) @@ -167,12 +202,21 @@ impl Replay { } while let Some(message) = capnp::serialize::try_read_message(&mut r, options())? { let f = message.get_root::()?; - let at = (f.get_session(), f.get_sequence()); - match self.next.unwrap_or((at.0, 0)) { - next if at < next => continue, - next if at > next => return Ok(()), - _ => self.next = Some((at.0, at.1 + 1)), + let (session, last) = (f.get_session(), f.get_sequence()); + let first = match f.which()? { + frame::Pages(group) => group.get_first(), + _ => last, + }; + let next = self.next.unwrap_or((session, 0)); + if (session, last) < next { + continue; } + if (session, first) > next { + return Ok(()); + } + // Pages a frame before already applied are rewritten with the + // same or newer versions. + self.next = Some((session, last + 1)); self.frame(f)?; } self.log_bytes += size; @@ -183,7 +227,7 @@ impl Replay { let message = capnp::serialize::read_message(&mut *r, options())?; let header = message.get_root::()?; self.next = Some((header.get_session(), header.get_frame())); - (self.wal, self.base, self.live, self.log_bytes) = (Vec::new(), 0, false, 0); + (self.live, self.log_bytes) = (false, 0); self.db.set_len(0)?; self.db.seek(SeekFrom::Start(0))?; let lanes = header.get_lanes()?; @@ -193,9 +237,7 @@ impl Replay { for lane in lanes { if let snapshot_header::lane::Sqlite(sqlite) = lane.which()? { std::io::copy(&mut r.take(sqlite.get_database_size()), &mut self.db)?; - r.take(sqlite.get_wal_size()).read_to_end(&mut self.wal)?; - (self.base, self.live) = (sqlite.get_wal_base_offset(), true); - self.header = !self.wal.is_empty(); + self.live = true; } } Ok(()) @@ -208,7 +250,6 @@ impl Replay { match f.which()? { frame::NewLane(_) | frame::DeleteLane(()) => { self.db.set_len(0)?; - (self.wal, self.base, self.header) = (Vec::new(), 0, true); self.live = matches!(f.which()?, frame::NewLane(_)); } frame::Initialize(image) => { @@ -216,74 +257,42 @@ impl Replay { self.db.seek(SeekFrom::Start(0))?; self.db.write_all(image?)?; } - frame::Write(w) => { - if w.get_offset() != self.base + self.wal.len() as u64 { - return Err(Error::Replica(format!("a write at {}", w.get_offset()))); + frame::Pages(group) => { + let mut page_size = 0; + for page in group.get_list()? { + let data = page.get_data()?; + page_size = data.len() as u64; + let at = u64::from(page.get_number() - 1) * page_size; + self.db.seek(SeekFrom::Start(at))?; + self.db.write_all(data)?; + } + if page_size > 0 { + self.db.set_len(u64::from(group.get_commit()) * page_size)?; } - self.wal.extend_from_slice(w.get_data()?); - } - frame::Consume(c) => { - self.checkpoint()?; - (self.base, self.header) = (c.get_offset(), true); } _ => return Err(Error::Replica("an unsupported frame".into())), } Ok(()) } - /// Copies the WAL's committed pages into the database. - fn checkpoint(&mut self) -> Result<()> { - let wal = std::mem::take(&mut self.wal); - self.base += wal.len() as u64; - let (page_size, frames) = if std::mem::replace(&mut self.header, false) { - let Some(size) = wal.get(8..12) else { - return Ok(()); - }; - (be32(size) as usize, &wal[WAL_HEADER..]) - } else if wal.is_empty() { - return Ok(()); - } else { - // Past a snapshot, the database's header has the page size. - let mut size = [0; 2]; - self.db.seek(SeekFrom::Start(16))?; - self.db.read_exact(&mut size)?; - let size = usize::from(u16::from_be_bytes(size)); - (if size == 1 { 1 << 16 } else { size }, &wal[..]) - }; - let frames: Vec<_> = frames.chunks_exact(FRAME_HEADER + page_size).collect(); - let Some(last) = frames.iter().rposition(|f| be32(&f[4..]) != 0) else { - return Ok(()); - }; - for f in &frames[..=last] { - let at = u64::from(be32(f) - 1) * page_size as u64; - self.db.seek(SeekFrom::Start(at))?; - self.db.write_all(&f[FRAME_HEADER..])?; - } - let pages = u64::from(be32(&frames[last][4..])); - Ok(self.db.set_len(pages * page_size as u64)?) - } - /// The replayed database, whole, and the frame the log continues from. - pub fn finish(mut self) -> Result { - self.checkpoint()?; + pub fn finish(self) -> Result { self.db.sync_data()?; Ok(self.next.map_or(0, |n| n.1)) } /// Writes the replayed state as a snapshot into `dir`. pub fn write_snapshot(mut self, dir: &Path) -> Result { - self.checkpoint()?; let (session, frame) = self.next.unwrap_or_default(); let size = self.db.seek(SeekFrom::End(0))?; self.db.seek(SeekFrom::Start(0))?; - let lane = self.live.then_some((size, self.base)); let header = Header { session, frame, justification: SnapshotJustification::ClientRequested, log_bytes: self.log_bytes, }; - write_snapshot(dir, &header, lane, |w| { + write_snapshot(dir, &header, self.live.then_some(size), |w| { Ok(std::io::copy(&mut self.db, w).map(drop)?) }) } @@ -296,12 +305,12 @@ struct Header { log_bytes: u64, } -/// Writes a snapshot into `dir`: the database lane, if it exists, as its -/// size and where the log's next write goes, with content `payload` writes. +/// Writes a snapshot into `dir`: the database lane, if it exists, of `size` +/// bytes that `payload` writes. fn write_snapshot( dir: &Path, h: &Header, - lane: Option<(u64, u64)>, + size: Option, payload: impl FnOnce(&mut dyn Write) -> Result<()>, ) -> Result { let mut message = capnp::message::Builder::new_default(); @@ -310,13 +319,11 @@ fn write_snapshot( header.set_frame(h.frame); header.set_justification(h.justification); header.set_log_bytes_since_last_snapshot(h.log_bytes); - let mut lanes = header.init_lanes(u32::from(lane.is_some())); - if let Some((size, base)) = lane { - let mut l = lanes.reborrow().get(0); - l.set_name(LANE); - let mut sqlite = l.init_sqlite(); - sqlite.set_database_size(size); - sqlite.set_wal_base_offset(base); + let mut lanes = header.init_lanes(u32::from(size.is_some())); + if let Some(size) = size { + let mut lane = lanes.reborrow().get(0); + lane.set_name(LANE); + lane.init_sqlite().set_database_size(size); } let file = File { snapshot: true, @@ -358,7 +365,7 @@ pub fn snapshot_file(path: &Path, dir: &Path, session: u32) -> Result Sequence { CAPTURE.with_borrow(|c| c.as_ref().unwrap().last().unwrap()) } +/// The captured frames `from..to`, merged. +fn merged(from: Sequence, to: Sequence) -> Vec { + let bytes = CAPTURE.with_borrow(|c| c.as_ref().unwrap().batch(from, to).unwrap()); + crate::log::merge(&bytes).unwrap() +} + /// The captured frames `from..to` as one batch in `dir`. fn batch(dir: &Path, from: Sequence, to: Sequence) -> (LogFile, PathBuf) { - let bytes = CAPTURE.with_borrow(|c| c.as_ref().unwrap().batch(from, to).unwrap()); + let bytes = merged(from, to); let path = dir.join(format!("{from}-{to}")); std::fs::write(&path, crate::log::compress(&bytes).unwrap()).unwrap(); let file = LogFile { @@ -404,6 +410,29 @@ fn a_snapshot_compacts_from_the_last_one_inside_a_wal() { assert_eq!(restored_rows(&store, &dir.join("restored.db")), rows(&app)); } +#[test] +fn a_batch_keeps_one_copy_of_a_page_its_commits_rewrote() { + let dir = scratch("dedup"); + let app = open_app(&dir.join("app.db")); + let store = Bucket::memory(""); + insert(&app, 0, 10); + let a = last(); + for i in 0..100 { + app.execute(&format!("UPDATE t SET v = '{i}' WHERE id = 1")); + } + let raw = CAPTURE.with_borrow(|c| c.as_ref().unwrap().batch(a, last()).unwrap()); + let merged = merged(a, last()); + assert!( + merged.len() * 20 < raw.len(), + "{} of {}", + merged.len(), + raw.len() + ); + upload(&store, 0, a); + upload(&store, a, last()); + assert_eq!(restored_rows(&store, &dir.join("restored.db")), rows(&app)); +} + #[test] fn a_corrupt_file_fails_to_restore() { let dir = scratch("corrupt"); diff --git a/src/workerd/server/cluster/owner.rs b/src/workerd/server/cluster/owner.rs index fe0fcf4bb2d..2653eabcacf 100644 --- a/src/workerd/server/cluster/owner.rs +++ b/src/workerd/server/cluster/owner.rs @@ -497,12 +497,13 @@ impl Owners { } } -/// The capture's frames from `from` up to `to`, as one batch. +/// The capture's frames from `from` up to `to`, merged into one batch. fn stage(capture: &Capture, from: Sequence, to: Sequence) -> Result>> { if to <= from { return Ok(None); } - Ok(Some(crate::log::compress(&capture.batch(from, to)?)?)) + let batch = crate::log::merge(&capture.batch(from, to)?)?; + Ok(Some(crate::log::compress(&batch)?)) } /// `plan`'s files replayed into this node's copy of a snapshot, on the From a90ea8ebdea9e4c21c8c3d043cc5483c22b31419 Mon Sep 17 00:00:00 2001 From: Ryan Dahl Date: Wed, 28 Oct 2026 09:18:45 -0700 Subject: [PATCH 17/38] cluster: shut cluster test nodes down cleanly workerd exits without dropping its tokio runtime, leaving idle blocking-pool threads unjoined, which TSan reports as leaked: the drain handoff test's owner then exits 66, not 0. Set KJ_CLEAN_SHUTDOWN for the nodes, as the asan and coverage configs do. --- src/workerd/server/tests/cluster/cluster-test.mjs | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/workerd/server/tests/cluster/cluster-test.mjs b/src/workerd/server/tests/cluster/cluster-test.mjs index 5f5277d3766..6927865943e 100644 --- a/src/workerd/server/tests/cluster/cluster-test.mjs +++ b/src/workerd/server/tests/cluster/cluster-test.mjs @@ -114,6 +114,9 @@ class ClusterNode { stdio: ['ignore', 'inherit', 'pipe', 'pipe'], env: { ...env, + // Drop the runtime on exit, joining its threads, which TSan reports + // as leaked otherwise. + KJ_CLEAN_SHUTDOWN: '1', NODE_ID: this.#nodeId, AWS_ENDPOINT_URL: `http://127.0.0.1:${this.#s3Port}`, }, From 41b31a52b860394d0aa5c757909c0089d5ebe727 Mon Sep 17 00:00:00 2001 From: Ryan Dahl Date: Wed, 28 Oct 2026 09:39:16 -0700 Subject: [PATCH 18/38] cluster: abort preventEviction objects when draining Every namespace is clustered, but drain aborted only evictable ones, so an object with preventEviction kept serving its in-flight requests after its node left the cluster, while a peer took it over. --- src/workerd/server/server/run.rs | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/src/workerd/server/server/run.rs b/src/workerd/server/server/run.rs index b39664130bc..66fe121414e 100644 --- a/src/workerd/server/server/run.rs +++ b/src/workerd/server/server/run.rs @@ -405,12 +405,17 @@ pub async fn run( match futures::future::select(serving, drain).await { Either::Left((result, _)) => result?, Either::Right(((), serving)) => { - // A draining instance claims nothing more, lets its objects go, and leaves the - // cluster, so peers take them over at their next listing while its connections - // finish. + // A draining instance claims nothing more, lets its objects go, `preventEviction` + // or not, and leaves the cluster, so peers take them over at their next listing + // while its connections finish. if let Some(node) = server.cluster() { node.cluster.start_draining(); - server.abort_all_actors(None); + let workers = server + .services() + .filter_map(|(_, service)| service.as_worker()); + for (_, namespace) in workers.flat_map(|worker| worker.namespaces()) { + namespace.abort_all(None); + } if let Err(e) = node.cluster.deregister().await { tracing::warn!("leaving the cluster: {e}"); } From 5642729a796746e0dd27cc2e9f577b2400f9be96 Mon Sep 17 00:00:00 2001 From: Ryan Dahl Date: Wed, 28 Oct 2026 09:39:16 -0700 Subject: [PATCH 19/38] cluster: refuse an empty cluster key Without one, every connection's MAC is keyed by the empty string, which anyone who reaches the cluster port can forge. --- src/workerd/server/server/cluster.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/workerd/server/server/cluster.rs b/src/workerd/server/server/cluster.rs index c2a7e392367..243387e8c5b 100644 --- a/src/workerd/server/server/cluster.rs +++ b/src/workerd/server/server/cluster.rs @@ -139,13 +139,17 @@ impl ClusterNode { address: String, ) -> Result> { let options = conf.get_bucket_options().map_err(|e| capnp(&e))?; + let key = text(conf.get_key())?.into_bytes(); + if key.is_empty() { + return Err(kj::failed!("cluster.key must be set")); + } let setup = cluster::Setup { url: text(conf.get_bucket())?, options: options.iter().map(text).collect::>()?, node: cluster::hex(&rand::random::<[u8; 16]>()), address: address.clone(), ttl_ms: cluster_logic::lease::LEASE_MS, - key: text(conf.get_key())?.into_bytes(), + key, }; let cluster = Cluster::new(setup).map_err(|e| kj::failed!("cluster: {e}"))?; Ok(Rc::new(Self { From 0a8162c2f1f04060c4e0e114bed26b3c6cbe1635 Mon Sep 17 00:00:00 2001 From: Ryan Dahl Date: Wed, 28 Oct 2026 13:11:21 -0700 Subject: [PATCH 20/38] cluster: explain what route() returns --- src/workerd/server/server/cluster.rs | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/workerd/server/server/cluster.rs b/src/workerd/server/server/cluster.rs index 243387e8c5b..292a1134429 100644 --- a/src/workerd/server/server/cluster.rs +++ b/src/workerd/server/server/cluster.rs @@ -161,8 +161,11 @@ impl ClusterNode { })) } - /// The owner of `actor` in `namespace`, the class `class` of worker `service`: here, at an - /// epoch, or a capnp session elsewhere. + /// Where a request for Durable Object `actor` of `namespace` should run. + /// If this instance owns the object, or claims it now, the result is + /// `Owner::Here` with the ownership epoch. Otherwise it is + /// `Owner::Remote`: a stub, over the session to the instance that owns + /// the object, for the object as class `class` of worker `service`. pub async fn route( &self, namespace: &str, From adf04a22545bcc07bbb6ca080b9b600daa9ae36e Mon Sep 17 00:00:00 2001 From: Ryan Dahl Date: Wed, 28 Oct 2026 13:11:21 -0700 Subject: [PATCH 21/38] cluster: share one dial to a peer between concurrent routes Concurrent first routes to a peer each saw no session, dialed their own, and overwrote each other's in the map, while the stubs they returned kept every one of those sessions alive. The map now holds the dial itself, as a shared future, so they wait on the same one. --- src/workerd/server/server/cluster.rs | 130 +++++++++++++++------------ 1 file changed, 74 insertions(+), 56 deletions(-) diff --git a/src/workerd/server/server/cluster.rs b/src/workerd/server/server/cluster.rs index 292a1134429..4a3d3962340 100644 --- a/src/workerd/server/server/cluster.rs +++ b/src/workerd/server/server/cluster.rs @@ -22,6 +22,7 @@ use cluster::Route; use futures::FutureExt; use futures::TryFutureExt; use futures::future::LocalBoxFuture; +use futures::future::Shared; use kj::http::Headers; use kj::http::HeadersRef; use kj::http::Method; @@ -55,8 +56,8 @@ const TUNNEL_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5); pub struct ClusterNode { pub cluster: Rc, factory: Rc, - /// One session per peer address, dropped when it disconnects. - sessions: Rc>>>, + /// One session per peer address, dialing or open, dropped when it disconnects. + sessions: Rc>>, /// The address peers dial, which their CONNECTs are bound to. advertise: String, /// The nonces of CONNECTs accepted within twice the window, by timestamp. @@ -65,6 +66,9 @@ pub struct ClusterNode { pub struct RpcConnection(KjOwn); +/// A session to a peer, shared by the requests that wait for its dial. +type Session = Shared, String>>>; + /// Binds the cluster port, before the services start. /// Returns the listener and the address to advertise. pub async fn bind( @@ -204,67 +208,81 @@ impl ClusterNode { )))) } - /// The session to the peer at `address`: the open one, or a new one. + /// The session to the peer at `address`: the open one, the one being + /// dialed, or a new one. async fn session(&self, address: &str) -> std::io::Result> { - if let Some(session) = self.sessions.borrow().get(address) { - return Ok(Rc::clone(session)); - } - // A stalled peer accepts the connection and never answers. - let tunnel = tokio::time::timeout(TUNNEL_TIMEOUT, self.tunnel(address)); - let stream = tunnel.await.map_err(|_| { - std::io::Error::new(std::io::ErrorKind::TimedOut, "no answer to CONNECT") - })??; - let session = Rc::new(RpcConnection(ffi::new_rpc_client( - self.factory.raw(), - Dialed::from(stream).into_kj(), - ))); - let replaced = self - .sessions - .borrow_mut() - .insert(address.to_owned(), Rc::clone(&session)); - drop(replaced); - let (sessions, address, watched) = ( - Rc::clone(&self.sessions), - address.to_owned(), - Rc::clone(&session), - ); - self.factory.spawn_detached(async move { - let _ = ffi::rpc_client_on_disconnect(&watched.0).await; - let mut sessions = sessions.borrow_mut(); - if sessions - .get(&address) - .is_some_and(|s| Rc::ptr_eq(s, &watched)) - { - sessions.remove(&address); - } + let session = self.sessions.borrow().get(address).cloned(); + let session = session.unwrap_or_else(|| { + let session = self.dial(address); + let mut sessions = self.sessions.borrow_mut(); + sessions.insert(address.to_owned(), session.clone()); + session }); - Ok(session) + let result = session.clone().await; + if result.is_err() { + // The next request dials again. + let mut sessions = self.sessions.borrow_mut(); + if sessions.get(address).is_some_and(|s| s.ptr_eq(&session)) { + sessions.remove(address); + } + } + result.map_err(std::io::Error::other) } - /// A TCP connection to the peer at `address`, through its CONNECT for a session. - async fn tunnel(&self, address: &str) -> std::io::Result { - let mut stream = TcpStream::connect(address).await?; - stream.set_nodelay(true)?; - let ms = cluster::now_ms(); - let nonce = cluster::hex(&rand::random::<[u8; 16]>()); - let mac = cluster::hex(&connect_mac(&self.cluster, address, ms, &nonce)); - let auth = format!("{AUTH_HEADER}: {ms}.{nonce}.{mac}"); - let request = format!("CONNECT cluster:1 HTTP/1.1\r\nhost: cluster:1\r\n{auth}\r\n\r\n"); - stream.write_all(request.as_bytes()).await?; - // The response head, read a byte at a time so that nothing past it is consumed. - let mut head = Vec::new(); - while !head.ends_with(b"\r\n\r\n") { - if head.len() > 4096 { - return Err(std::io::Error::other("response head too long")); - } - head.push(stream.read_u8().await?); + fn dial(&self, address: &str) -> Session { + let cluster = Rc::clone(&self.cluster); + let factory = Rc::clone(&self.factory); + let sessions = Rc::clone(&self.sessions); + let address = address.to_owned(); + async move { + // A stalled peer accepts the connection and never answers. + let tunnel = tokio::time::timeout(TUNNEL_TIMEOUT, tunnel(&cluster, &address)); + let stream = tunnel + .await + .map_err(|_| "no answer to CONNECT".to_owned())? + .map_err(|e| e.to_string())?; + let session = Rc::new(RpcConnection(ffi::new_rpc_client( + factory.raw(), + Dialed::from(stream).into_kj(), + ))); + let watched = Rc::clone(&session); + factory.spawn_detached(async move { + let _ = ffi::rpc_client_on_disconnect(&watched.0).await; + let mut sessions = sessions.borrow_mut(); + let open = |s: &Session| matches!(s.peek(), Some(Ok(c)) if Rc::ptr_eq(c, &watched)); + if sessions.get(&address).is_some_and(open) { + sessions.remove(&address); + } + }); + Ok(session) } - if !head.starts_with(b"HTTP/1.1 200") { - let status = String::from_utf8_lossy(head.split(|&b| b == b'\r').next().unwrap_or(&[])); - return Err(std::io::Error::other(format!("refused: {status}"))); + .boxed_local() + .shared() + } +} + +async fn tunnel(node: &Cluster, address: &str) -> std::io::Result { + let mut stream = TcpStream::connect(address).await?; + stream.set_nodelay(true)?; + let ms = cluster::now_ms(); + let nonce = cluster::hex(&rand::random::<[u8; 16]>()); + let mac = cluster::hex(&connect_mac(node, address, ms, &nonce)); + let auth = format!("{AUTH_HEADER}: {ms}.{nonce}.{mac}"); + let request = format!("CONNECT cluster:1 HTTP/1.1\r\nhost: cluster:1\r\n{auth}\r\n\r\n"); + stream.write_all(request.as_bytes()).await?; + // The response head, read a byte at a time so that nothing past it is consumed. + let mut head = Vec::new(); + while !head.ends_with(b"\r\n\r\n") { + if head.len() > 4096 { + return Err(std::io::Error::other("response head too long")); } - Ok(stream) + head.push(stream.read_u8().await?); + } + if !head.starts_with(b"HTTP/1.1 200") { + let status = String::from_utf8_lossy(head.split(|&b| b == b'\r').next().unwrap_or(&[])); + return Err(std::io::Error::other(format!("refused: {status}"))); } + Ok(stream) } /// Where a Durable Object runs, for a request about to start on it. From a138beb00967e3e82560bb722cc19a49770a2ffd Mon Sep 17 00:00:00 2001 From: Ryan Dahl Date: Wed, 28 Oct 2026 13:11:21 -0700 Subject: [PATCH 22/38] cluster: delete the sessions an opened epoch supersedes Once an epoch opens, own.json names it and its session starts with a snapshot of everything the recovered one held, so no restore reads an earlier session again. Nothing deleted them, leaving another copy of the object in the bucket at every handoff. The next tier pass after an epoch opens deletes every earlier session, including what a superseded owner uploaded after the takeover. --- src/workerd/server/cluster/log.rs | 25 ++++++++++++++--- src/workerd/server/cluster/log_tests.rs | 8 ++++++ src/workerd/server/cluster/owner.rs | 36 ++++++++++++++++++++++--- 3 files changed, 62 insertions(+), 7 deletions(-) diff --git a/src/workerd/server/cluster/log.rs b/src/workerd/server/cluster/log.rs index 76f01fb8c24..f359d41479c 100644 --- a/src/workerd/server/cluster/log.rs +++ b/src/workerd/server/cluster/log.rs @@ -479,13 +479,30 @@ impl Log { /// The files of `session`. pub async fn files(&self, session: u32) -> Result> { + self.list(&format!("/{}/", descending(session.into()))) + .await + } + + /// The files of every session before `session`. + pub async fn older(&self, session: u32) -> Result> { + let files = self.list("/").await?; + Ok(files.into_iter().filter(|f| f.session < session).collect()) + } + + /// The files under ``, for both kinds. + async fn list(&self, under: &str) -> Result> { let mut files = Vec::new(); for snapshot in [false, true] { let kind = if snapshot { "snapshot" } else { "log" }; - let prefix = format!("{kind}/{}/", descending(session.into())); - for entry in self.bucket.list(&prefix).await? { - let name = entry.key.rsplit('/').next().unwrap_or_default(); - if let Some(frame) = parse(name, !snapshot).and_then(|f| f.try_into().ok()) { + for entry in self.bucket.list(&format!("{kind}{under}")).await? { + let mut parts = entry.key.rsplit('/'); + let number = |p: Option<&str>, asc| { + p.and_then(|p| parse(p, asc)) + .and_then(|n| u32::try_from(n).ok()) + }; + let frame = number(parts.next(), !snapshot); + let session = number(parts.next(), false); + if let (Some(frame), Some(session)) = (frame, session) { files.push(File { snapshot, session, diff --git a/src/workerd/server/cluster/log_tests.rs b/src/workerd/server/cluster/log_tests.rs index 9bede9fc739..e46a315b4a2 100644 --- a/src/workerd/server/cluster/log_tests.rs +++ b/src/workerd/server/cluster/log_tests.rs @@ -620,4 +620,12 @@ fn a_superseded_owners_commit_is_not_acknowledged() { matches!(&commit, Err(Error::Contended(m)) if m.contains("superseded")), "{commit:?}" ); + + // b's epoch is open, so no restore reads a's session: b deletes it, + // with the upload a made after b took over. + let x = format!("{}{}", store.prefix(), crate::object_prefix("ns", "x")); + let x = Log::new(store.at(&x)); + assert!(!rt.block_on(x.older(2)).unwrap().is_empty()); + rt.block_on(b_owners.tier()); + assert!(rt.block_on(x.older(2)).unwrap().is_empty()); } diff --git a/src/workerd/server/cluster/owner.rs b/src/workerd/server/cluster/owner.rs index 2653eabcacf..c903eb24bb0 100644 --- a/src/workerd/server/cluster/owner.rs +++ b/src/workerd/server/cluster/owner.rs @@ -104,6 +104,9 @@ struct Object { /// The files of this epoch's session. Only this owner writes it, so /// compaction deletes from here rather than listing it. files: Vec, + /// Whether earlier sessions may remain in the bucket, which no restore + /// reads once this epoch opened. + prune: bool, } pub struct Owners { @@ -275,6 +278,7 @@ impl Owners { tiered: 0, cursor: Cursor::default(), files: Vec::new(), + prune: false, }; if self .opened @@ -295,6 +299,7 @@ impl Owners { ))); } tracing::info!("cluster: {ns}/{actor} epoch {} open", owned.epoch); + object.prune = true; Ok(object) } @@ -353,13 +358,24 @@ impl Owners { Ok(()) } - /// Snapshot every owned object whose log outgrew its database. Runs off - /// the commit path. + /// Delete the sessions before each newly opened epoch's, and snapshot + /// every owned object whose log outgrew its database. Runs off the + /// commit path. pub async fn tier(&self) { let objects: Vec<_> = self.objects.borrow().values().cloned().collect(); stream::iter(objects) .for_each_concurrent(TIER_CONCURRENCY, |object| async move { - let due = snapshot_due(&object.object.lock().await.files); + let (due, prune) = { + let mut guard = object.object.lock().await; + let prune = std::mem::take(&mut guard.prune); + ( + snapshot_due(&guard.files), + prune.then(|| guard.replica.clone()), + ) + }; + if let Some(replica) = prune { + prune_older(&replica).await; + } if due { self.compact(&object).await; } @@ -595,6 +611,20 @@ fn snapshot_due(files: &[File]) -> bool { } /// Uploads this node's copy of `file`. +/// Delete the sessions before `replica`'s. A superseded owner still +/// uploading may leave files behind, which the next epoch deletes. +async fn prune_older(replica: &Replica) { + let files = match replica.client.older(replica.session).await { + Ok(files) => files, + Err(e) => return tracing::warn!("cluster: pruning old sessions: {e}"), + }; + stream::iter(files) + .for_each_concurrent(TIER_CONCURRENCY, |f| async move { + let _ = replica.client.delete(&f).await; + }) + .await; +} + async fn upload(replica: &Replica, file: File) -> Result { let path = replica.cache.join(file.name()); replica.client.put(&file, &path).await?; From dc8f1c60d997b187c5803b20d933619578539e78 Mon Sep 17 00:00:00 2001 From: Ryan Dahl Date: Wed, 28 Oct 2026 13:38:13 -0700 Subject: [PATCH 23/38] cluster: join workflow namespaces to the cluster before they link A namespace's storage decides at link whether its database writes go to the cluster. Workflow namespaces linked before the cluster existed, so they routed through the cluster but replicated nothing, and a takeover restored stale state. The cluster is now made first and set on each workflow namespace before its storage opens. --- src/workerd/server/server/config.rs | 26 ++++++++++++++++---------- src/workerd/server/server/worker.rs | 8 +++++++- 2 files changed, 23 insertions(+), 11 deletions(-) diff --git a/src/workerd/server/server/config.rs b/src/workerd/server/server/config.rs index cf46daf0557..f37378eb03c 100644 --- a/src/workerd/server/server/config.rs +++ b/src/workerd/server/server/config.rs @@ -639,6 +639,7 @@ fn init_workflows( services: &LinkedHashMap, actor_configs: &ActorConfigs, report: &Reporter, + cluster: Option<&Rc>, namespaces_by_unique_key: &mut HashMap>, ) -> Result<()> { for service_conf in config.get_services().map_err(capnp_error)? { @@ -670,7 +671,7 @@ fn init_workflows( else { continue; }; - if let Some(namespace) = engine.init_workflow(workflow, key, config)? { + if let Some(namespace) = engine.init_workflow(workflow, key, config, cluster)? { namespaces_by_unique_key.insert(key.clone(), namespace); } } @@ -759,6 +760,7 @@ impl<'a> WorkflowEngine<'a> { workflow: &Workflow, key: &str, config: &ActorConfig, + cluster: Option<&Rc>, ) -> Result>> { let (workflow_name, class_name) = (workflow.name.as_str(), workflow.class_name.as_str()); if self.app.has_actor_class(key) { @@ -835,6 +837,7 @@ impl<'a> WorkflowEngine<'a> { actor_class, storage_path, self.engine.persistent_self_tokens(), + cluster, ) .map(Some) } @@ -1062,15 +1065,6 @@ impl Server { .insert("internet".to_owned(), Service::Leaf(internet)); } - // Every service exists: wire each worker's Workflows to the engine that runs them. - init_workflows( - config, - &services.0, - &actor_configs, - &report, - &mut namespaces_by_unique_key, - )?; - let cluster = match cluster_address { Some(address) if config.has_cluster() => { make_cluster(&factory, config, address, &report)? @@ -1084,6 +1078,18 @@ impl Server { .join() .await .map_err(|e| kj::failed!("joining the cluster: {e}"))?; + } + + // Every service exists: wire each worker's Workflows to the engine that runs them. + init_workflows( + config, + &services.0, + &actor_configs, + &report, + cluster.as_ref(), + &mut namespaces_by_unique_key, + )?; + if let Some(cluster) = &cluster { for namespace in namespaces_by_unique_key.values() { namespace.set_cluster(Rc::clone(cluster)); } diff --git a/src/workerd/server/server/worker.rs b/src/workerd/server/server/worker.rs index d0c10c78620..73bbf974590 100644 --- a/src/workerd/server/server/worker.rs +++ b/src/workerd/server/server/worker.rs @@ -54,6 +54,7 @@ use crate::channels::WorkerEntrypoint; use crate::channels::WorkerInterface; use crate::channels::WorkerInterfaceList; use crate::channels::WorkerStub; +use crate::cluster::ClusterNode; use crate::config::Factory; use crate::config::Reporter; use crate::config::Server; @@ -475,7 +476,8 @@ impl WorkerService { /// Creates the namespace backing one configured Workflow, keyed by `key`, its unique key. /// Its actors are `actor_class`, the engine's class with the Workflow's props bound, and its /// storage is in `storage_path`, the directory of the Workflow's `bindingService` worker. - /// `persistent_self_tokens` is the engine worker's. + /// `persistent_self_tokens` is the engine worker's. In cluster mode the namespace joins + /// `cluster` before its storage opens. pub fn init_workflow_namespace( &self, key: &str, @@ -483,6 +485,7 @@ impl WorkerService { actor_class: Rc, storage_path: Option<&str>, persistent_self_tokens: Persistent, + cluster: Option<&Rc>, ) -> Result> { let namespace = ActorNamespace::new( Rc::clone(&self.factory), @@ -492,6 +495,9 @@ impl WorkerService { actor_class, persistent_self_tokens, ); + if let Some(cluster) = cluster { + namespace.set_cluster(Rc::clone(cluster)); + } namespace.link(storage_path)?; self.namespaces .borrow_mut() From 0291d08f0beae1b9e8d21c9e9824d6c5241fc6bc Mon Sep 17 00:00:00 2001 From: Ryan Dahl Date: Wed, 28 Oct 2026 13:39:02 -0700 Subject: [PATCH 24/38] cluster: abort only clustered objects when draining Drain aborted every namespace, so in-memory ones, which have nothing to hand off, failed their in-flight requests too. --- src/workerd/server/server/actor.rs | 2 +- src/workerd/server/server/run.rs | 10 ++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/workerd/server/server/actor.rs b/src/workerd/server/server/actor.rs index b174149baf3..49c6b9ddd43 100644 --- a/src/workerd/server/server/actor.rs +++ b/src/workerd/server/server/actor.rs @@ -159,7 +159,7 @@ impl ActorNamespace { } /// The cluster, for a namespace linked on disk; in-memory namespaces stay local. - fn cluster(&self) -> Option<&Rc> { + pub(crate) fn cluster(&self) -> Option<&Rc> { self.directory.get().and_then(|_| self.cluster.get()) } diff --git a/src/workerd/server/server/run.rs b/src/workerd/server/server/run.rs index 66fe121414e..184b0b571a1 100644 --- a/src/workerd/server/server/run.rs +++ b/src/workerd/server/server/run.rs @@ -405,15 +405,17 @@ pub async fn run( match futures::future::select(serving, drain).await { Either::Left((result, _)) => result?, Either::Right(((), serving)) => { - // A draining instance claims nothing more, lets its objects go, `preventEviction` - // or not, and leaves the cluster, so peers take them over at their next listing - // while its connections finish. + // A draining instance claims nothing more, lets its clustered objects go, + // `preventEviction` or not, and leaves the cluster, so peers take them over at their + // next listing while its connections finish. A request in flight on one could not + // commit anyway once a peer owns the object. if let Some(node) = server.cluster() { node.cluster.start_draining(); let workers = server .services() .filter_map(|(_, service)| service.as_worker()); - for (_, namespace) in workers.flat_map(|worker| worker.namespaces()) { + let namespaces = workers.flat_map(|worker| worker.namespaces()); + for (_, namespace) in namespaces.filter(|(_, ns)| ns.cluster().is_some()) { namespace.abort_all(None); } if let Err(e) = node.cluster.deregister().await { From 1da25c63e68586d2c6bbc57db2bbc63801687583 Mon Sep 17 00:00:00 2001 From: Ryan Dahl Date: Wed, 28 Oct 2026 13:42:45 -0700 Subject: [PATCH 25/38] cluster: refuse Durable Objects from before cluster mode A takeover replaces an object's local database with the bucket's copy, so adding cluster to a config whose localDisk already held Durable Objects deleted each one on first access. A namespace directory is now marked the first time cluster mode uses it, and one that holds databases without the mark is refused at startup. --- src/workerd/server/server/actor.rs | 32 ++++++++++++++++++- .../server/tests/cluster/cluster-test.mjs | 20 +++++++++++- src/workerd/server/workerd.capnp | 4 ++- 3 files changed, 53 insertions(+), 3 deletions(-) diff --git a/src/workerd/server/server/actor.rs b/src/workerd/server/server/actor.rs index 49c6b9ddd43..290998232f3 100644 --- a/src/workerd/server/server/actor.rs +++ b/src/workerd/server/server/actor.rs @@ -397,7 +397,11 @@ impl ActorNamespace { ActorConfig::Ephemeral { .. } => ("", ""), }; if !path.is_empty() { - let _ = self.directory.set(format!("{path}/{unique_key}")); + let directory = format!("{path}/{unique_key}"); + if self.cluster.get().is_some() { + claim_directory(&directory)?; + } + let _ = self.directory.set(directory); } let storage = ffi::factory_new_actor_storage( self.factory.raw(), @@ -435,6 +439,32 @@ impl ActorNamespace { } } +/// Marks a namespace's `directory` as the cluster's. The cluster replaces an +/// object's database with the bucket's copy, so a directory holding Durable +/// Objects from before cluster mode is refused rather than deleted. +fn claim_directory(directory: &str) -> Result<()> { + let directory = std::path::Path::new(directory); + let marker = directory.join(".cluster"); + if marker.exists() { + return Ok(()); + } + let databases = std::fs::read_dir(directory).into_iter().flatten().flatten(); + let held = databases.map(|e| e.file_name()).any(|name| { + let name = name.to_string_lossy(); + name.ends_with(".sqlite") && name != "metadata.sqlite" + }); + if held { + return Err(kj::failed!( + "{} holds Durable Objects from before cluster mode, which the cluster would \ + replace: move them away", + directory.display() + )); + } + std::fs::create_dir_all(directory) + .and_then(|()| std::fs::write(&marker, "")) + .map_err(|e| kj::failed!("{}: {e}", marker.display())) +} + impl ActorNamespaceHandle { pub(crate) fn actor_for_alarm( &self, diff --git a/src/workerd/server/tests/cluster/cluster-test.mjs b/src/workerd/server/tests/cluster/cluster-test.mjs index 6927865943e..c8a86fa4574 100644 --- a/src/workerd/server/tests/cluster/cluster-test.mjs +++ b/src/workerd/server/tests/cluster/cluster-test.mjs @@ -20,7 +20,7 @@ // breaks, while a fresh stub reaches the re-instantiated DO. import { spawn } from 'node:child_process'; -import { mkdir, mkdtemp, readdir, rm } from 'node:fs/promises'; +import { mkdir, mkdtemp, readdir, rm, writeFile } from 'node:fs/promises'; import { tmpdir } from 'node:os'; import { connect, createServer } from 'node:net'; import { createHash, createHmac, randomBytes } from 'node:crypto'; @@ -801,6 +801,24 @@ test('cluster: a public cluster address is refused', async () => { await rm(local, { recursive: true }); }); +test('cluster: Durable Objects from before cluster mode are refused', async () => { + await withCluster(1, async ({ bucket }) => { + const local = await mkdtemp(join(tmpdir(), 'workerd-cluster-')); + await mkdir(join(local, 'counter-test-namespace')); + await writeFile(join(local, 'counter-test-namespace', 'a.sqlite'), ''); + const node = new ClusterNode({ + binary: env.WORKERD_BINARY, + config: env.WD_TEST_CONFIG, + localPath: local, + s3Port: bucket.port, + nodeId: 'old', + }); + await assert.rejects(node.start(), /exited before listening/); + assert.match(node.stderr, /from before cluster mode/); + await rm(local, { recursive: true }); + }); +}); + test('cluster: each running instance holds a lease record', async () => { await withCluster(2, async ({ nodes, bucket, nodeKeys }) => { const records = await bucket.nodes(); diff --git a/src/workerd/server/workerd.capnp b/src/workerd/server/workerd.capnp index f4265be28e2..90d9fd51309 100644 --- a/src/workerd/server/workerd.capnp +++ b/src/workerd/server/workerd.capnp @@ -108,7 +108,9 @@ struct ClusterConfig { # record naming its address, and each Durable Object's owner is a record claimed by a conditional # write. Taking an object over from an owner whose lease expired bumps its ownership epoch. # Instances need no shared filesystem: `durableObjectStorage.localDisk` is each instance's own - # disk. Durable Objects stored in memory stay local to each instance. + # disk. Durable Objects stored in memory stay local to each instance. The bucket's copy of an + # object replaces the instance's, so a `localDisk` already holding Durable Objects from before + # cluster mode is refused at startup; they are not imported. # # All instances in the cluster MUST have identical configuration: any instance may forward any # request to any other and it must produce the same result. During a rolling upgrade, instances From 1e1cceb0cf445b02913d332ab7599c48bf9c2be6 Mon Sep 17 00:00:00 2001 From: Ryan Dahl Date: Wed, 28 Oct 2026 13:44:21 -0700 Subject: [PATCH 26/38] cluster: fail on a short, truncated or oversized file A restore read less than a damaged file held without failing: - a snapshot's database was copied without checking its length, and the frame was never read to its end, where the LZ4 checksum is checked; - the LZ4 decoder takes a frame cut at a block boundary, or an empty file, for a whole one, so a batch's last commits were dropped; - with no traversal limit, a corrupt message's segment table was allocated at whatever size it claimed, aborting the process. Every file is now checked to be one whole LZ4 frame before it is read, a snapshot must hold exactly its database, and a message is bounded at 1 GiB. --- src/workerd/server/cluster/log.rs | 47 +++++++++++++++++++++++-- src/workerd/server/cluster/log_tests.rs | 20 +++++++++++ 2 files changed, 64 insertions(+), 3 deletions(-) diff --git a/src/workerd/server/cluster/log.rs b/src/workerd/server/cluster/log.rs index f359d41479c..faf1d407606 100644 --- a/src/workerd/server/cluster/log.rs +++ b/src/workerd/server/cluster/log.rs @@ -163,8 +163,39 @@ fn encoder(w: W) -> lz4_flex::frame::FrameEncoder { lz4_flex::frame::FrameEncoder::with_frame_info(info, w) } +/// Bounds a message, so that a corrupt one fails rather than allocating +/// whatever its segment table claims: 1 GiB, far more than a batch holds. fn options() -> capnp::message::ReaderOptions { - *capnp::message::ReaderOptions::new().traversal_limit_in_words(None) + *capnp::message::ReaderOptions::new().traversal_limit_in_words(Some(1 << 27)) +} + +/// Fails unless `file` is one whole LZ4 frame, through its end mark and +/// checksum: the decoder takes a frame cut at a block boundary for a +/// whole, shorter one. +fn check_whole(file: &mut std::fs::File) -> Result<()> { + let truncated = || Error::Replica("a truncated file".into()); + let len = file.metadata()?.len(); + let mut header = [0u8; 6]; + file.read_exact(&mut header).map_err(|_| truncated())?; + let flags = header[4]; + let bit = |mask: u8, n: u64| if flags & mask != 0 { n } else { 0 }; + let mut at = 7 + bit(0x08, 8) + bit(0x01, 4); + loop { + let mut size = [0u8; 4]; + file.seek(SeekFrom::Start(at))?; + file.read_exact(&mut size).map_err(|_| truncated())?; + at += 4; + let size = u32::from_le_bytes(size) & 0x7fff_ffff; + if size == 0 { + break; + } + at += u64::from(size) + bit(0x10, 4); + } + if at + bit(0x04, 4) != len { + return Err(truncated()); + } + file.seek(SeekFrom::Start(0))?; + Ok(()) } /// The database lane, replayed into a file. @@ -194,7 +225,8 @@ impl Replay { /// Applies `f`, the local copy at `path`. A batch that does not reach /// back to the next frame is skipped: a later, wider one covers it. pub fn apply(&mut self, f: &File, path: &Path) -> Result<()> { - let file = std::fs::File::open(path)?; + let mut file = std::fs::File::open(path)?; + check_whole(&mut file)?; let size = file.metadata()?.len(); let mut r = BufReader::new(lz4_flex::frame::FrameDecoder::new(file)); if f.snapshot { @@ -236,10 +268,19 @@ impl Replay { } for lane in lanes { if let snapshot_header::lane::Sqlite(sqlite) = lane.which()? { - std::io::copy(&mut r.take(sqlite.get_database_size()), &mut self.db)?; + let size = sqlite.get_database_size(); + if std::io::copy(&mut r.take(size), &mut self.db)? != size { + return Err(Error::Replica( + "a snapshot shorter than its database".into(), + )); + } self.live = true; } } + // Read to the frame's end, where the decoder checks the checksum. + if std::io::copy(r, &mut std::io::sink())? != 0 { + return Err(Error::Replica("a snapshot longer than its lanes".into())); + } Ok(()) } diff --git a/src/workerd/server/cluster/log_tests.rs b/src/workerd/server/cluster/log_tests.rs index e46a315b4a2..4c53585d1c5 100644 --- a/src/workerd/server/cluster/log_tests.rs +++ b/src/workerd/server/cluster/log_tests.rs @@ -449,6 +449,26 @@ fn a_corrupt_file_fails_to_restore() { assert!(restored.is_err(), "{restored:?}"); } +#[test] +fn a_truncated_file_fails_to_restore() { + // The LZ4 decoder takes a frame without its end mark, or no frame at + // all, for a whole one. + let dir = scratch("truncated"); + let app = open_app(&dir.join("app.db")); + insert(&app, 0, 100); + let (file, path) = batch(&dir, 0, last()); + let bytes = std::fs::read(&path).unwrap(); + for cut in [&bytes[..bytes.len() - 8], &[]] { + let store = Bucket::memory(""); + std::fs::write(&path, cut).unwrap(); + runtime().block_on(log(&store).put(&file, &path)).unwrap(); + let out = dir.join("restored.db"); + let _ = std::fs::remove_file(&out); + let restored = runtime().block_on(restore(&log(&store), 0, &out)); + assert!(restored.is_err(), "{restored:?}"); + } +} + #[test] fn a_rolled_back_transaction_leaves_nothing() { let dir = scratch("rollback"); From 006709455f4ef93fcc1b63ad1e6cf667fbdf3f77 Mon Sep 17 00:00:00 2001 From: Ryan Dahl Date: Wed, 28 Oct 2026 13:46:46 -0700 Subject: [PATCH 27/38] cluster: refuse an alarm when setAlarm() is called The refusal was thrown from scheduleRun(), which runs when the write commits, after setAlarm() had resolved: the implicit transaction failed, rolling back the writes since the last await and breaking the actor, and the caller could not catch it. ActorSqlite::Hooks gains checkAlarm(), which setAlarm() calls before it changes anything; the default allows every alarm. --- src/workerd/io/actor-sqlite.c++ | 1 + src/workerd/io/actor-sqlite.h | 4 ++++ src/workerd/server/factory/worker-factory-actor.c++ | 9 ++++++--- src/workerd/server/tests/cluster/cluster-test.mjs | 2 ++ 4 files changed, 13 insertions(+), 3 deletions(-) diff --git a/src/workerd/io/actor-sqlite.c++ b/src/workerd/io/actor-sqlite.c++ index c6788ed4d3b..886e8352e58 100644 --- a/src/workerd/io/actor-sqlite.c++ +++ b/src/workerd/io/actor-sqlite.c++ @@ -824,6 +824,7 @@ kj::OneOf> ActorSqlite::delete_( kj::Maybe> ActorSqlite::setAlarm( kj::Maybe newAlarmTime, WriteOptions options, SpanParent traceSpan) { requireNotBroken(); + hooks.checkAlarm(newAlarmTime); // Capture trace span for the output gate lock hold trace. currentCommitSpan = kj::mv(traceSpan); diff --git a/src/workerd/io/actor-sqlite.h b/src/workerd/io/actor-sqlite.h index 527353fe2ae..e363a84cc2e 100644 --- a/src/workerd/io/actor-sqlite.h +++ b/src/workerd/io/actor-sqlite.h @@ -41,6 +41,10 @@ class ActorSqlite final: public ActorCacheInterface, private kj::TaskSet::ErrorH virtual kj::Promise scheduleRun( kj::Maybe newAlarmTime, kj::Promise priorTask); + // Called by setAlarm() before it changes anything, so that an alarm the embedder cannot + // schedule fails the call itself rather than the commit that follows. The default allows any. + virtual void checkAlarm(kj::Maybe newAlarmTime) {} + static const Hooks DEFAULT; static constexpr inline Hooks& getDefaultHooks() { diff --git a/src/workerd/server/factory/worker-factory-actor.c++ b/src/workerd/server/factory/worker-factory-actor.c++ index b51b0bb7c64..e7c160eda63 100644 --- a/src/workerd/server/factory/worker-factory-actor.c++ +++ b/src/workerd/server/factory/worker-factory-actor.c++ @@ -382,12 +382,15 @@ class ActorSqliteHooks final: public ActorSqlite::Hooks { : storage(kj::mv(storage)), actor(kj::mv(actor)) {} + void checkAlarm(kj::Maybe newAlarmTime) override { + // TODO(cluster): an alarm must follow its object between instances. + JSG_REQUIRE(newAlarmTime == kj::none || !storage->actors->replicated(), Error, + "Alarms are not yet supported in cluster mode."); + } + kj::Promise scheduleRun( kj::Maybe newAlarmTime, kj::Promise priorTask) override { KJ_IF_SOME(scheduledTime, newAlarmTime) { - // TODO(cluster): an alarm must follow its object between instances. - JSG_REQUIRE( - !storage->actors->replicated(), Error, "Alarms are not yet supported in cluster mode."); storage->alarmScheduler->setAlarm(actor, scheduledTime); } else { storage->alarmScheduler->deleteAlarm(actor); diff --git a/src/workerd/server/tests/cluster/cluster-test.mjs b/src/workerd/server/tests/cluster/cluster-test.mjs index c8a86fa4574..15104601f21 100644 --- a/src/workerd/server/tests/cluster/cluster-test.mjs +++ b/src/workerd/server/tests/cluster/cluster-test.mjs @@ -836,7 +836,9 @@ test('cluster: each running instance holds a lease record', async () => { test('cluster: setting an alarm is refused', async () => { await withCluster(1, async ({ nodes }) => { + // The handler catches the refusal: setAlarm() throws, not the commit. const res = await fetchJson(nodes[0].httpPort, '/set-alarm?name=alarm'); + assert.strictEqual(res.status, 200); assert.match(res.body.error ?? '', /not yet supported in cluster mode/); }); }); From 66742a317fa2ffb3511eace70c91f345d597b60b Mon Sep 17 00:00:00 2001 From: Ryan Dahl Date: Wed, 28 Oct 2026 13:50:38 -0700 Subject: [PATCH 28/38] cluster: require a cluster key of at least 32 bytes The MAC's other inputs cross the wire, so a short or guessable key can be found offline from one captured CONNECT, and the key grants the whole debug port. The check is a config error, so server-test now covers it and the --experimental refusal. --- src/workerd/server/server-test.c++ | 29 +++++++++++++++++++ src/workerd/server/server/cluster.rs | 6 +--- src/workerd/server/server/config.rs | 5 ++++ .../server/tests/cluster/cluster-test.mjs | 2 +- .../server/tests/cluster/cluster-test.wd-test | 2 +- src/workerd/server/workerd.capnp | 3 +- 6 files changed, 39 insertions(+), 8 deletions(-) diff --git a/src/workerd/server/server-test.c++ b/src/workerd/server/server-test.c++ index d143fd5bae6..c4a45f41b9f 100644 --- a/src/workerd/server/server-test.c++ +++ b/src/workerd/server/server-test.c++ @@ -1075,6 +1075,35 @@ KJ_TEST("Server: wrapped bindings work under the new module registry") { conn.httpGet200("/", "wrapped: object"); } +kj::String clusterConfig(kj::StringPtr key) { + return kj::str(R"(( + services = [( + name = "hello", + worker = ( + compatibilityDate = "2024-01-01", + modules = [(name = "main.js", esModule = "export default {}")] + ) + )], + cluster = (bucket = "s3://bucket", address = "127.0.0.1:0", key = ")", + key, R"("), + ))"); +} + +KJ_TEST("Server: cluster mode requires --experimental") { + TestServer test(clusterConfig("0123456789abcdef0123456789abcdef")); + test.expectErrors(R"( + Cluster mode is experimental; it requires --experimental. + )"_blockquote); +} + +KJ_TEST("Server: a short cluster key is refused") { + TestServer test(clusterConfig("0123456789abcdef")); + test.server.allowExperimental(); + test.expectErrors(R"( + cluster.key must be at least 32 bytes. + )"_blockquote); +} + KJ_TEST("Server: duplicate module names are reported as config errors (legacy registry)") { TestServer test(singleWorker(R"(( compatibilityDate = "2024-10-01", diff --git a/src/workerd/server/server/cluster.rs b/src/workerd/server/server/cluster.rs index 4a3d3962340..01d9de76d30 100644 --- a/src/workerd/server/server/cluster.rs +++ b/src/workerd/server/server/cluster.rs @@ -143,17 +143,13 @@ impl ClusterNode { address: String, ) -> Result> { let options = conf.get_bucket_options().map_err(|e| capnp(&e))?; - let key = text(conf.get_key())?.into_bytes(); - if key.is_empty() { - return Err(kj::failed!("cluster.key must be set")); - } let setup = cluster::Setup { url: text(conf.get_bucket())?, options: options.iter().map(text).collect::>()?, node: cluster::hex(&rand::random::<[u8; 16]>()), address: address.clone(), ttl_ms: cluster_logic::lease::LEASE_MS, - key, + key: text(conf.get_key())?.into_bytes(), }; let cluster = Cluster::new(setup).map_err(|e| kj::failed!("cluster: {e}"))?; Ok(Rc::new(Self { diff --git a/src/workerd/server/server/config.rs b/src/workerd/server/server/config.rs index f37378eb03c..9be554cc1e0 100644 --- a/src/workerd/server/server/config.rs +++ b/src/workerd/server/server/config.rs @@ -1522,5 +1522,10 @@ fn make_cluster( return Ok(None); } let conf = config.get_cluster().map_err(capnp_error)?; + // The MAC's other inputs cross the wire, so a short key falls to an offline search. + if text(conf.get_key())?.len() < 32 { + report.error("cluster.key must be at least 32 bytes.".to_owned()); + return Ok(None); + } Ok(Some(ClusterNode::new(Rc::clone(factory), conf, address)?)) } diff --git a/src/workerd/server/tests/cluster/cluster-test.mjs b/src/workerd/server/tests/cluster/cluster-test.mjs index 15104601f21..baabebf573a 100644 --- a/src/workerd/server/tests/cluster/cluster-test.mjs +++ b/src/workerd/server/tests/cluster/cluster-test.mjs @@ -751,7 +751,7 @@ async function clusterConnect(port, auth) { // test config's key makes it. function clusterAuth(target, ms = Date.now(), nonce = randomBytes(16)) { const n = nonce.toString('hex'); - const mac = createHmac('sha256', 'test-cluster-shared-secret') + const mac = createHmac('sha256', 'test-cluster-shared-secret-of-32-bytes') .update(`workerd cluster connect v1\n${target}\n${ms}\n${n}`) .digest('hex'); return `${ms}.${n}.${mac}`; diff --git a/src/workerd/server/tests/cluster/cluster-test.wd-test b/src/workerd/server/tests/cluster/cluster-test.wd-test index 5ac6c1f01d2..147a47732a2 100644 --- a/src/workerd/server/tests/cluster/cluster-test.wd-test +++ b/src/workerd/server/tests/cluster/cluster-test.wd-test @@ -48,6 +48,6 @@ const config :Workerd.Config = ( "aws_secret_access_key=wdtest-secret", ], address = "127.0.0.1:0", - key = "test-cluster-shared-secret", + key = "test-cluster-shared-secret-of-32-bytes", ), ); diff --git a/src/workerd/server/workerd.capnp b/src/workerd/server/workerd.capnp index 90d9fd51309..63457f6613c 100644 --- a/src/workerd/server/workerd.capnp +++ b/src/workerd/server/workerd.capnp @@ -143,7 +143,8 @@ struct ClusterConfig { # `--cluster-advertise`. A public IP is refused unless `unsafePublicAddress` is set. key @3 :Text; - # A secret known only to members of the cluster. Each connection between instances opens with + # A secret known only to members of the cluster, at least 32 bytes: random, such as the output + # of `openssl rand -hex 32`. Each connection between instances opens with # a MAC keyed by it, over the target, a timestamp and a nonce, so a header seen on the wire # cannot be replayed; the instances' clocks must agree within 30 seconds. It also keys channel # tokens, which pass service and Durable Object stubs over RPC. Traffic is not encrypted: run From ae4745d2894b74d082d84991a10b133837eb6de6 Mon Sep 17 00:00:00 2001 From: Ryan Dahl Date: Wed, 28 Oct 2026 13:52:30 -0700 Subject: [PATCH 29/38] cluster: cap a bucket response held in memory at 16 MiB Records, listings and errors are read into memory, so an endpoint that kept sending could exhaust the process before the body was parsed. --- src/workerd/server/cluster/http.rs | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/workerd/server/cluster/http.rs b/src/workerd/server/cluster/http.rs index 4e88c5894be..5dad1ae7596 100644 --- a/src/workerd/server/cluster/http.rs +++ b/src/workerd/server/cluster/http.rs @@ -56,6 +56,8 @@ pub enum Payload { /// The chunks a file is sent and received in. const CHUNK: usize = 256 << 10; +/// The most of a response body held in memory rather than written to a file. +const MAX_BODY: usize = 16 << 20; pub struct Http { client: legacy::Client, @@ -125,6 +127,10 @@ impl Http { .into_data() .unwrap_or_default(); if writer.is_none() { + // Records, listings and errors: an endpoint sending more is broken. + if body.len() + chunk.len() > MAX_BODY { + return Err("response body too large".to_owned()); + } body.extend(chunk); } else if tx.send(chunk).await.is_err() { break; From 7132ccb4a7b0afd90fc017457d6089c9be128d1a Mon Sep 17 00:00:00 2001 From: Ryan Dahl Date: Wed, 28 Oct 2026 14:11:59 -0700 Subject: [PATCH 30/38] cluster: test that a failed dial to a peer is retried The shared dial moves into Sessions, apart from the factory it needs, so its guarantees are tested on their own: concurrent callers share one dial, and a failed dial or a disconnected session is forgotten, so the next route dials again. A failure kept instead would leave the peer unreachable from this node until it restarts. --- src/workerd/server/server/cluster-test.rs | 71 ++++++++++++++++++ src/workerd/server/server/cluster.rs | 88 ++++++++++++++++------- 2 files changed, 132 insertions(+), 27 deletions(-) create mode 100644 src/workerd/server/server/cluster-test.rs diff --git a/src/workerd/server/server/cluster-test.rs b/src/workerd/server/server/cluster-test.rs new file mode 100644 index 00000000000..4e3c3897af2 --- /dev/null +++ b/src/workerd/server/server/cluster-test.rs @@ -0,0 +1,71 @@ +// Copyright (c) 2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +use std::cell::Cell; + +use futures::future::join_all; + +use super::*; + +fn runtime() -> tokio::runtime::Runtime { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap() +} + +/// A dial that counts itself and, while `fail` is set, fails. +fn dial(dials: &Rc>, fail: bool) -> Dial { + let dials = Rc::clone(dials); + async move { + dials.set(dials.get() + 1); + tokio::task::yield_now().await; + if fail { + Err("refused".to_owned()) + } else { + Ok(dials.get()) + } + } + .boxed_local() +} + +#[test] +fn concurrent_callers_share_one_dial() { + let (sessions, dials) = (Sessions::default(), Rc::new(Cell::new(0))); + let gets = (0..3).map(|_| sessions.get("peer", || dial(&dials, false))); + let got = runtime().block_on(join_all(gets)); + assert_eq!(got, vec![Ok(1), Ok(1), Ok(1)]); + assert_eq!(dials.get(), 1); +} + +#[test] +fn a_failed_dial_is_forgotten() { + let (sessions, dials) = (Sessions::default(), Rc::new(Cell::new(0))); + let rt = runtime(); + let failed = rt.block_on(sessions.get("peer", || dial(&dials, true))); + assert_eq!(failed, Err("refused".to_owned())); + let retried = rt.block_on(sessions.get("peer", || dial(&dials, false))); + assert_eq!(retried, Ok(2)); +} + +#[test] +fn a_disconnected_session_is_forgotten() { + let (sessions, dials) = (Sessions::default(), Rc::new(Cell::new(0))); + let rt = runtime(); + assert_eq!( + rt.block_on(sessions.get("peer", || dial(&dials, false))), + Ok(1) + ); + // Another session's disconnect leaves this one. + sessions.forget("peer", |&s| s == 7); + assert_eq!( + rt.block_on(sessions.get("peer", || dial(&dials, false))), + Ok(1) + ); + sessions.forget("peer", |&s| s == 1); + assert_eq!( + rt.block_on(sessions.get("peer", || dial(&dials, false))), + Ok(2) + ); +} diff --git a/src/workerd/server/server/cluster.rs b/src/workerd/server/server/cluster.rs index 01d9de76d30..88e40055a28 100644 --- a/src/workerd/server/server/cluster.rs +++ b/src/workerd/server/server/cluster.rs @@ -56,8 +56,8 @@ const TUNNEL_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5); pub struct ClusterNode { pub cluster: Rc, factory: Rc, - /// One session per peer address, dialing or open, dropped when it disconnects. - sessions: Rc>>, + /// One session per peer address, dropped when it disconnects. + sessions: Rc>>, /// The address peers dial, which their CONNECTs are bound to. advertise: String, /// The nonces of CONNECTs accepted within twice the window, by timestamp. @@ -66,8 +66,56 @@ pub struct ClusterNode { pub struct RpcConnection(KjOwn); -/// A session to a peer, shared by the requests that wait for its dial. -type Session = Shared, String>>>; +type Dial = LocalBoxFuture<'static, std::result::Result>; + +/// One value per key, a peer's session: the callers that arrive while it is +/// being made share the attempt, and a failed attempt is forgotten, so the +/// next caller makes another. +pub struct Sessions { + map: RefCell>>>, +} + +impl Default for Sessions { + fn default() -> Self { + Self { + map: RefCell::default(), + } + } +} + +impl Sessions { + /// `key`'s value: the one made, the one being made, or one `make` makes. + pub async fn get( + &self, + key: &str, + make: impl FnOnce() -> Dial, + ) -> std::result::Result { + let attempt = self.map.borrow().get(key).cloned(); + let attempt = attempt.unwrap_or_else(|| { + let attempt = make().shared(); + let mut map = self.map.borrow_mut(); + map.insert(key.to_owned(), attempt.clone()); + attempt + }); + let result = attempt.clone().await; + if result.is_err() { + let mut map = self.map.borrow_mut(); + if map.get(key).is_some_and(|a| a.ptr_eq(&attempt)) { + map.remove(key); + } + } + result + } + + /// Forgets `key`'s value if it is the one `is` matches. + pub fn forget(&self, key: &str, is: impl Fn(&V) -> bool) { + let mut map = self.map.borrow_mut(); + let made = map.get(key).and_then(|a| a.peek()); + if made.is_some_and(|r| r.as_ref().is_ok_and(&is)) { + map.remove(key); + } + } +} /// Binds the cluster port, before the services start. /// Returns the listener and the address to advertise. @@ -207,25 +255,12 @@ impl ClusterNode { /// The session to the peer at `address`: the open one, the one being /// dialed, or a new one. async fn session(&self, address: &str) -> std::io::Result> { - let session = self.sessions.borrow().get(address).cloned(); - let session = session.unwrap_or_else(|| { - let session = self.dial(address); - let mut sessions = self.sessions.borrow_mut(); - sessions.insert(address.to_owned(), session.clone()); - session - }); - let result = session.clone().await; - if result.is_err() { - // The next request dials again. - let mut sessions = self.sessions.borrow_mut(); - if sessions.get(address).is_some_and(|s| s.ptr_eq(&session)) { - sessions.remove(address); - } - } - result.map_err(std::io::Error::other) + let dial = || self.dial(address); + let session = self.sessions.get(address, dial).await; + session.map_err(std::io::Error::other) } - fn dial(&self, address: &str) -> Session { + fn dial(&self, address: &str) -> Dial> { let cluster = Rc::clone(&self.cluster); let factory = Rc::clone(&self.factory); let sessions = Rc::clone(&self.sessions); @@ -244,16 +279,11 @@ impl ClusterNode { let watched = Rc::clone(&session); factory.spawn_detached(async move { let _ = ffi::rpc_client_on_disconnect(&watched.0).await; - let mut sessions = sessions.borrow_mut(); - let open = |s: &Session| matches!(s.peek(), Some(Ok(c)) if Rc::ptr_eq(c, &watched)); - if sessions.get(&address).is_some_and(open) { - sessions.remove(&address); - } + sessions.forget(&address, |s| Rc::ptr_eq(s, &watched)); }); Ok(session) } .boxed_local() - .shared() } } @@ -506,3 +536,7 @@ impl Drop for Owned { }); } } + +#[cfg(test)] +#[path = "cluster-test.rs"] +mod tests; From 5759710b427a3f888e3b38a30f0d4a9e34964ff4 Mon Sep 17 00:00:00 2001 From: Ryan Dahl Date: Wed, 28 Oct 2026 14:45:03 -0700 Subject: [PATCH 31/38] cluster: drop the experimental exemption for workerd: services A Worker whose service name began with workerd: could use experimental compatibility flags without --experimental, and the name is the config's to choose. No built-in service uses the exemption. --- src/workerd/server/factory/worker-factory.c++ | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/src/workerd/server/factory/worker-factory.c++ b/src/workerd/server/factory/worker-factory.c++ index e03e2333645..2c0a790105e 100644 --- a/src/workerd/server/factory/worker-factory.c++ +++ b/src/workerd/server/factory/worker-factory.c++ @@ -1405,13 +1405,10 @@ MainModuleIsPython isPythonMainModule(config::Worker::Reader conf) { // Compiles a config worker's compatibility flags into `arena`, as the worker's errors. CompatibilityFlags::Reader compileFlags(const WorkerFactory::Options& options, - kj::StringPtr name, config::Worker::Reader conf, capnp::MallocMessageBuilder& arena, ErrorReporter& errorReporter) { auto flags = arena.initRoot(); - // Built-in services' Workers need `experimental`, as under Miniflare (`sql.prepare()`). - bool experimental = options.experimental || name.startsWith("workerd:"); KJ_IF_SOME(overrideDate, options.testCompatibilityDateOverride) { if (conf.hasCompatibilityDate()) { errorReporter.addError( @@ -1421,11 +1418,11 @@ CompatibilityFlags::Reader compileFlags(const WorkerFactory::Options& options, } // FUTURE_FOR_TEST admits any valid date, such as 2999-12-31. compileCompatibilityFlags(overrideDate, conf.getCompatibilityFlags(), flags, errorReporter, - experimental, CompatibilityDateValidation::FUTURE_FOR_TEST, nullptr, + options.experimental, CompatibilityDateValidation::FUTURE_FOR_TEST, nullptr, isPythonMainModule(conf)); } else if (conf.hasCompatibilityDate()) { compileCompatibilityFlags(conf.getCompatibilityDate(), conf.getCompatibilityFlags(), flags, - errorReporter, experimental, CompatibilityDateValidation::CODE_VERSION, nullptr, + errorReporter, options.experimental, CompatibilityDateValidation::CODE_VERSION, nullptr, isPythonMainModule(conf)); } else KJ_IF_SOME(defaultDate, options.defaultCompatibilityDate) { compileCompatibilityFlags(defaultDate, conf.getCompatibilityFlags(), flags, errorReporter, @@ -1538,7 +1535,7 @@ kj::Promise> factory_new_worker(const WorkerFactory& con auto conf = factory.config.getServices()[index].getWorker(); extensions = factory.config.getExtensions(); impl->flagsArena = kj::heap(); - featureFlags = compileFlags(options, impl->name, conf, *impl->flagsArena, errorReporter); + featureFlags = compileFlags(options, conf, *impl->flagsArena, errorReporter); source = WorkerdApi::extractSource(impl->name, conf, featureFlags, errorReporter); if (conf.hasModuleFallback()) moduleFallback = conf.getModuleFallback(); compileBindings = [globals = kj::heapArray(kj::from(spec.globals))]( From 7c4e74048f863d157974fab7f87b68566b5cdea7 Mon Sep 17 00:00:00 2001 From: Ryan Dahl Date: Wed, 28 Oct 2026 14:45:03 -0700 Subject: [PATCH 32/38] cluster: fail a restore whose log is missing a batch A batch that starts past the frame the log has reached is skipped, for a later, wider batch covers it when uploads race. When none does, a batch between is missing, and the restore succeeded with the state before it, which compaction could then make permanent. The replay now remembers the furthest skipped batch and fails unless it got past it. --- src/workerd/server/cluster/log.rs | 22 +++++++++++++++++++++- src/workerd/server/cluster/log_tests.rs | 19 +++++++++++++++++++ 2 files changed, 40 insertions(+), 1 deletion(-) diff --git a/src/workerd/server/cluster/log.rs b/src/workerd/server/cluster/log.rs index faf1d407606..3e9a37488ab 100644 --- a/src/workerd/server/cluster/log.rs +++ b/src/workerd/server/cluster/log.rs @@ -204,6 +204,9 @@ pub struct Replay { live: bool, /// The next frame, by session and sequence. next: Option<(u32, Sequence)>, + /// The furthest end of a batch skipped for starting past `next`, which + /// a later batch must reach. + skipped: Option<(u32, Sequence)>, log_bytes: u64, } @@ -218,12 +221,13 @@ impl Replay { .open(db)?, live: false, next: None, + skipped: None, log_bytes: 0, }) } /// Applies `f`, the local copy at `path`. A batch that does not reach - /// back to the next frame is skipped: a later, wider one covers it. + /// back to the next frame is skipped: a later, wider one must cover it. pub fn apply(&mut self, f: &File, path: &Path) -> Result<()> { let mut file = std::fs::File::open(path)?; check_whole(&mut file)?; @@ -232,6 +236,7 @@ impl Replay { if f.snapshot { return self.snapshot(&mut r); } + let end = (f.session, f.frame); while let Some(message) = capnp::serialize::try_read_message(&mut r, options())? { let f = message.get_root::()?; let (session, last) = (f.get_session(), f.get_sequence()); @@ -244,6 +249,7 @@ impl Replay { continue; } if (session, first) > next { + self.skipped = self.skipped.max(Some(end)); return Ok(()); } // Pages a frame before already applied are rewritten with the @@ -316,14 +322,28 @@ impl Replay { Ok(()) } + /// Fails if a skipped batch's frames were never replayed: a batch + /// between is missing. + fn check_gaps(&self) -> Result<()> { + match self.skipped { + Some(end) if self.next.is_none_or(|next| next < end) => Err(Error::Replica(format!( + "the log is missing frames before {}", + end.1 + ))), + _ => Ok(()), + } + } + /// The replayed database, whole, and the frame the log continues from. pub fn finish(self) -> Result { + self.check_gaps()?; self.db.sync_data()?; Ok(self.next.map_or(0, |n| n.1)) } /// Writes the replayed state as a snapshot into `dir`. pub fn write_snapshot(mut self, dir: &Path) -> Result { + self.check_gaps()?; let (session, frame) = self.next.unwrap_or_default(); let size = self.db.seek(SeekFrom::End(0))?; self.db.seek(SeekFrom::Start(0))?; diff --git a/src/workerd/server/cluster/log_tests.rs b/src/workerd/server/cluster/log_tests.rs index 4c53585d1c5..8b106f2267c 100644 --- a/src/workerd/server/cluster/log_tests.rs +++ b/src/workerd/server/cluster/log_tests.rs @@ -563,6 +563,25 @@ fn a_batch_the_log_already_covers_is_not_applied() { assert_eq!(restored_rows(&store, &dir.join("restored.db")), rows(&app)); } +#[test] +fn a_missing_batch_fails_to_restore() { + // The batch after a lost one starts past the frame the log reached, + // and no wider one covers it. + let dir = scratch("missing"); + let app = open_app(&dir.join("app.db")); + let store = Bucket::memory(""); + let mut ends = Vec::new(); + for from in (0..30).step_by(10) { + insert(&app, from, from + 10); + ends.push(last()); + } + upload(&store, 0, ends[0]); + upload(&store, ends[1], ends[2]); + let out = dir.join("restored.db"); + let restored = runtime().block_on(restore(&log(&store), 0, &out)); + assert!(restored.is_err(), "{restored:?}"); +} + #[test] fn a_deleted_database_restores_empty_then_as_recreated() { // A facet's delete() or a deleteAll() removes the file; the new From 2d92646e15ce303e1b88e169fd5f00cd3c253f6b Mon Sep 17 00:00:00 2001 From: Ryan Dahl Date: Wed, 28 Oct 2026 14:45:03 -0700 Subject: [PATCH 33/38] cluster: test the cap on a response held in memory --- src/workerd/server/cluster/http-test.rs | 35 +++++++++++++++++++++++++ src/workerd/server/cluster/http.rs | 4 +++ 2 files changed, 39 insertions(+) create mode 100644 src/workerd/server/cluster/http-test.rs diff --git a/src/workerd/server/cluster/http-test.rs b/src/workerd/server/cluster/http-test.rs new file mode 100644 index 00000000000..ab8cde9e915 --- /dev/null +++ b/src/workerd/server/cluster/http-test.rs @@ -0,0 +1,35 @@ +// Copyright (c) 2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +use tokio::io::AsyncReadExt; +use tokio::io::AsyncWriteExt; +use tokio::net::TcpListener; + +use super::*; + +#[test] +fn a_body_held_in_memory_is_capped() { + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + rt.block_on(async { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + tokio::spawn(async move { + let (mut stream, _) = listener.accept().await.unwrap(); + let _ = stream.read(&mut [0; 4096]).await; + let size = MAX_BODY + 1; + let head = format!("HTTP/1.1 200 OK\r\ncontent-length: {size}\r\n\r\n"); + stream.write_all(head.as_bytes()).await.unwrap(); + let _ = stream.write_all(&vec![0; size]).await; + }); + let http = Http::new().unwrap(); + let wait = Duration::from_secs(10); + let url = format!("http://127.0.0.1:{port}/"); + let payload = Payload::Bytes(bytes::Bytes::new()); + let sent = http.send("GET".into(), url, vec![], payload, None, (wait, wait)); + assert_eq!(sent.await.err().as_deref(), Some("response body too large")); + }); +} diff --git a/src/workerd/server/cluster/http.rs b/src/workerd/server/cluster/http.rs index 5dad1ae7596..a90b71ee407 100644 --- a/src/workerd/server/cluster/http.rs +++ b/src/workerd/server/cluster/http.rs @@ -291,3 +291,7 @@ impl AsyncWrite for Stream { } } } + +#[cfg(test)] +#[path = "http-test.rs"] +mod tests; From 62882a0bb835c45e055a2fa9fdd5cdfc64b63c0f Mon Sep 17 00:00:00 2001 From: Ryan Dahl Date: Wed, 28 Oct 2026 14:59:30 -0700 Subject: [PATCH 34/38] cluster: test that a cluster's instances redeem each other's tokens Handlers given the same cluster key decode each other's channel tokens, and one given another key rejects them. --- src/workerd/server/channel-token-test.c++ | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/workerd/server/channel-token-test.c++ b/src/workerd/server/channel-token-test.c++ index f1cab81be8f..98c2d8bb161 100644 --- a/src/workerd/server/channel-token-test.c++ +++ b/src/workerd/server/channel-token-test.c++ @@ -414,6 +414,24 @@ KJ_TEST("actor channel tokens") { "channel token type mismatch", handler.decodeActorClassChannelToken(Usage::RPC, token)); } +KJ_TEST("instances of a cluster redeem each other's channel tokens") { + MockResolver senderResolver, receiverResolver, strangerResolver; + auto key = "test-cluster-shared-secret-of-32-bytes"_kj; + ChannelTokenHandler sender(senderResolver, key); + ChannelTokenHandler receiver(receiverResolver, key); + ChannelTokenHandler stranger(strangerResolver, "another-cluster-secret-of-32-bytes"_kj); + + const byte actorId[] = {12, 34, 56, 78}; + auto token = sender.encodeActorChannelToken( + Usage::RPC, "foo-namespace", actorId, "my-actor"_kj, Persistent::NO); + auto channel = + receiver.decodeSubrequestChannelToken(Usage::RPC, token).downcast(); + expectActorChannel(*channel, "foo-namespace", actorId, "my-actor"_kj); + + KJ_EXPECT_THROW_MESSAGE( + "failed authentication", stranger.decodeSubrequestChannelToken(Usage::RPC, token)); +} + KJ_TEST("channel token with nested channels (all synchronous)") { MockResolver resolver; ChannelTokenHandler handler(resolver); From 9d48e083bda663385687114ec767f7f74e8f5ed9 Mon Sep 17 00:00:00 2001 From: Ryan Dahl Date: Wed, 28 Oct 2026 14:59:30 -0700 Subject: [PATCH 35/38] cluster: test that compaction refuses a log missing a batch --- src/workerd/server/cluster/log_tests.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/workerd/server/cluster/log_tests.rs b/src/workerd/server/cluster/log_tests.rs index 8b106f2267c..adf39d33835 100644 --- a/src/workerd/server/cluster/log_tests.rs +++ b/src/workerd/server/cluster/log_tests.rs @@ -564,7 +564,7 @@ fn a_batch_the_log_already_covers_is_not_applied() { } #[test] -fn a_missing_batch_fails_to_restore() { +fn a_missing_batch_fails_to_restore_or_compact() { // The batch after a lost one starts past the frame the log reached, // and no wider one covers it. let dir = scratch("missing"); @@ -580,6 +580,10 @@ fn a_missing_batch_fails_to_restore() { let out = dir.join("restored.db"); let restored = runtime().block_on(restore(&log(&store), 0, &out)); assert!(restored.is_err(), "{restored:?}"); + // Nor does compaction snapshot it, then delete the batches. + let files = [batch(&dir, 0, ends[0]), batch(&dir, ends[1], ends[2])]; + let compacted = crate::log::compact(&files, &dir); + assert!(compacted.is_err(), "{compacted:?}"); } #[test] From b1b05a3aff7f1bb3aba3095d64132937e2d92637 Mon Sep 17 00:00:00 2001 From: Ryan Dahl Date: Wed, 28 Oct 2026 15:33:40 -0700 Subject: [PATCH 36/38] cluster: leave the alarm refusal at commit time This reverts 006709455, which added ActorSqlite::Hooks::checkAlarm() to core io/ only to move a temporary error message in experimental code. Alarm support in cluster mode replaces the refusal. --- src/workerd/io/actor-sqlite.c++ | 1 - src/workerd/io/actor-sqlite.h | 4 ---- src/workerd/server/factory/worker-factory-actor.c++ | 9 +++------ src/workerd/server/tests/cluster/cluster-test.mjs | 2 -- 4 files changed, 3 insertions(+), 13 deletions(-) diff --git a/src/workerd/io/actor-sqlite.c++ b/src/workerd/io/actor-sqlite.c++ index 886e8352e58..c6788ed4d3b 100644 --- a/src/workerd/io/actor-sqlite.c++ +++ b/src/workerd/io/actor-sqlite.c++ @@ -824,7 +824,6 @@ kj::OneOf> ActorSqlite::delete_( kj::Maybe> ActorSqlite::setAlarm( kj::Maybe newAlarmTime, WriteOptions options, SpanParent traceSpan) { requireNotBroken(); - hooks.checkAlarm(newAlarmTime); // Capture trace span for the output gate lock hold trace. currentCommitSpan = kj::mv(traceSpan); diff --git a/src/workerd/io/actor-sqlite.h b/src/workerd/io/actor-sqlite.h index e363a84cc2e..527353fe2ae 100644 --- a/src/workerd/io/actor-sqlite.h +++ b/src/workerd/io/actor-sqlite.h @@ -41,10 +41,6 @@ class ActorSqlite final: public ActorCacheInterface, private kj::TaskSet::ErrorH virtual kj::Promise scheduleRun( kj::Maybe newAlarmTime, kj::Promise priorTask); - // Called by setAlarm() before it changes anything, so that an alarm the embedder cannot - // schedule fails the call itself rather than the commit that follows. The default allows any. - virtual void checkAlarm(kj::Maybe newAlarmTime) {} - static const Hooks DEFAULT; static constexpr inline Hooks& getDefaultHooks() { diff --git a/src/workerd/server/factory/worker-factory-actor.c++ b/src/workerd/server/factory/worker-factory-actor.c++ index e7c160eda63..b51b0bb7c64 100644 --- a/src/workerd/server/factory/worker-factory-actor.c++ +++ b/src/workerd/server/factory/worker-factory-actor.c++ @@ -382,15 +382,12 @@ class ActorSqliteHooks final: public ActorSqlite::Hooks { : storage(kj::mv(storage)), actor(kj::mv(actor)) {} - void checkAlarm(kj::Maybe newAlarmTime) override { - // TODO(cluster): an alarm must follow its object between instances. - JSG_REQUIRE(newAlarmTime == kj::none || !storage->actors->replicated(), Error, - "Alarms are not yet supported in cluster mode."); - } - kj::Promise scheduleRun( kj::Maybe newAlarmTime, kj::Promise priorTask) override { KJ_IF_SOME(scheduledTime, newAlarmTime) { + // TODO(cluster): an alarm must follow its object between instances. + JSG_REQUIRE( + !storage->actors->replicated(), Error, "Alarms are not yet supported in cluster mode."); storage->alarmScheduler->setAlarm(actor, scheduledTime); } else { storage->alarmScheduler->deleteAlarm(actor); diff --git a/src/workerd/server/tests/cluster/cluster-test.mjs b/src/workerd/server/tests/cluster/cluster-test.mjs index baabebf573a..5591d1bdac1 100644 --- a/src/workerd/server/tests/cluster/cluster-test.mjs +++ b/src/workerd/server/tests/cluster/cluster-test.mjs @@ -836,9 +836,7 @@ test('cluster: each running instance holds a lease record', async () => { test('cluster: setting an alarm is refused', async () => { await withCluster(1, async ({ nodes }) => { - // The handler catches the refusal: setAlarm() throws, not the commit. const res = await fetchJson(nodes[0].httpPort, '/set-alarm?name=alarm'); - assert.strictEqual(res.status, 200); assert.match(res.body.error ?? '', /not yet supported in cluster mode/); }); }); From a002d5d913b54f1a290b14cf74f22c451cc04ca9 Mon Sep 17 00:00:00 2001 From: Ryan Dahl Date: Wed, 28 Oct 2026 15:37:25 -0700 Subject: [PATCH 37/38] cluster: open the alarm scheduler through the actors' VFS A namespace opened its scheduler's database through a second VFS, to keep it outside the cluster's wrapper. The wrapper needs no exception: the cluster ignores WAL writes of files no object owns, and the scheduler opens its database at startup, under the lease. So a namespace has one VFS again, as outside cluster mode. --- src/workerd/server/factory/worker-factory-actor.c++ | 3 +-- src/workerd/server/factory/worker-factory-impl.h | 6 ++---- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/src/workerd/server/factory/worker-factory-actor.c++ b/src/workerd/server/factory/worker-factory-actor.c++ index b51b0bb7c64..b60393e82e1 100644 --- a/src/workerd/server/factory/worker-factory-actor.c++ +++ b/src/workerd/server/factory/worker-factory-actor.c++ @@ -175,11 +175,10 @@ ActorStorage::Impl::Impl(const WorkerFactory& factory, ? kj::heap(vfsDirectory->clone(), *actors) : vfsDirectory->clone()), vfs(*sqliteDirectory), - schedulerVfs(*vfsDirectory), uniqueKey(kj::mv(uniqueKey)), alarmScheduler(kj::heap(kj::systemPreciseCalendarClock(), factory.getImpl().timer, - schedulerVfs, + vfs, kj::Path({"metadata.sqlite"}), [this](const ActorKey& actor) -> kj::Own { // The id carries the persisted name so that `ctx.id.name` is set when the alarm runs. diff --git a/src/workerd/server/factory/worker-factory-impl.h b/src/workerd/server/factory/worker-factory-impl.h index caafb2e1e60..c94183b650f 100644 --- a/src/workerd/server/factory/worker-factory-impl.h +++ b/src/workerd/server/factory/worker-factory-impl.h @@ -219,12 +219,10 @@ class ActorStorage::Impl final: public kj::Refcounted { ::rust::Box actors; kj::Maybe> directory; kj::Own vfsDirectory; // the in-memory directory when there is none - // What actors' SQLite opens files through: `vfsDirectory`, or in a cluster a wrapper that - // fences opens on the lease and reports WAL writes. + // What SQLite opens files through: `vfsDirectory`, or in a cluster a wrapper that fences opens + // on the lease and reports WAL writes, which the cluster ignores for the scheduler's database. kj::Own sqliteDirectory; SqliteDatabase::Vfs vfs; - // The scheduler's database is no actor's, so it opens outside the cluster's wrapper. - SqliteDatabase::Vfs schedulerVfs; kj::String uniqueKey; kj::Own alarmScheduler; From 18efd126d6d6a63f13e711b1d813430e1929f674 Mon Sep 17 00:00:00 2001 From: Ryan Dahl Date: Wed, 28 Oct 2026 15:37:25 -0700 Subject: [PATCH 38/38] cluster: gzip the bucket's files instead of LZ4 flate2 is already a dependency, so lz4_flex and twox-hash go. Gzip's trailer holds the content's CRC-32 and length, which the decoder checks, so a corrupt or truncated file fails on its own, without walking the frame first. --- deps/rust/Cargo.lock | 16 ----- deps/rust/Cargo.toml | 2 - deps/rust/cargo-bazel-lock.json | 78 +------------------------ src/workerd/server/cluster/BUILD.bazel | 2 +- src/workerd/server/cluster/log.rs | 56 ++++-------------- src/workerd/server/cluster/log_tests.rs | 7 ++- 6 files changed, 18 insertions(+), 143 deletions(-) diff --git a/deps/rust/Cargo.lock b/deps/rust/Cargo.lock index d33d075ca6f..8ce972233a3 100644 --- a/deps/rust/Cargo.lock +++ b/deps/rust/Cargo.lock @@ -639,7 +639,6 @@ dependencies = [ "libm", "libz-rs-sys", "lol_html_c_api", - "lz4_flex", "nix", "notify", "percent-encoding", @@ -1494,15 +1493,6 @@ dependencies = [ "thiserror", ] -[[package]] -name = "lz4_flex" -version = "0.11.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "373f5eceeeab7925e0c1098212f2fbc4d416adec9d35051a6ab251e824c1854a" -dependencies = [ - "twox-hash", -] - [[package]] name = "memchr" version = "2.8.3" @@ -2978,12 +2968,6 @@ version = "0.2.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" -[[package]] -name = "twox-hash" -version = "2.1.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86a801b3cea342a06d468c8710662aa29e5e05e4f5c0d62f00bbb7f2ad7941c2" - [[package]] name = "typenum" version = "1.20.1" diff --git a/deps/rust/Cargo.toml b/deps/rust/Cargo.toml index 7206b86eec7..8def4e053f3 100644 --- a/deps/rust/Cargo.toml +++ b/deps/rust/Cargo.toml @@ -67,8 +67,6 @@ libc = "0" libm = "0" lol_html_c_api = { git = "https://github.com/cloudflare/lol-html" } hashlink = "0" -# cluster: the LZ4 frame of every log file in the bucket. -lz4_flex = { version = "0.11", default-features = false, features = ["frame", "safe-decode", "safe-encode"] } # kj-rs-io: `fs` for fcntl (KJ's fd flags), `signal` for ignoring SIGPIPE once per process like # kj::UnixEventPort does. nix = { version = "0", features = ["fs", "signal"] } diff --git a/deps/rust/cargo-bazel-lock.json b/deps/rust/cargo-bazel-lock.json index e1bdc79b571..a70d882adef 100644 --- a/deps/rust/cargo-bazel-lock.json +++ b/deps/rust/cargo-bazel-lock.json @@ -2,7 +2,7 @@ "binary_crates": [ "capnpc 0.27.0" ], - "checksum": "33898b4c3fe680c38aa89fe347f491daec819d6fc982c720d95288722a69dde1", + "checksum": "4f7c905fb454b3d367e5ae35edc0a5d3b38300d86a842ca721465a1cb99a6cc0", "conditions": { "aarch64-apple-darwin": [ "aarch64-apple-darwin" @@ -2990,10 +2990,6 @@ "id": "lol_html_c_api 1.4.0", "target": "lolhtml" }, - { - "id": "lz4_flex 0.11.6", - "target": "lz4_flex" - }, { "id": "nix 0.31.3", "target": "nix" @@ -7062,46 +7058,6 @@ ], "version": "1.4.0" }, - "lz4_flex 0.11.6": { - "common_attrs": { - "crate_features": [ - "frame", - "safe-decode", - "safe-encode", - "std" - ], - "deps": [ - { - "id": "twox-hash 2.1.5", - "target": "twox_hash" - } - ], - "edition": "2021" - }, - "library_target_name": "lz4_flex", - "license": "MIT", - "license_file": "LICENSE", - "license_ids": [ - "MIT" - ], - "name": "lz4_flex", - "package_url": "https://github.com/pseitz/lz4_flex", - "repository": { - "Http": { - "sha256": "373f5eceeeab7925e0c1098212f2fbc4d416adec9d35051a6ab251e824c1854a", - "url": "https://static.crates.io/crates/lz4_flex/0.11.6/download" - } - }, - "targets": [ - { - "Library": { - "crate_name": "lz4_flex", - "crate_root": "src/lib.rs" - } - } - ], - "version": "0.11.6" - }, "memchr 2.8.3": { "common_attrs": { "crate_features": [ @@ -13885,37 +13841,6 @@ ], "version": "0.2.5" }, - "twox-hash 2.1.5": { - "common_attrs": { - "crate_features": [ - "xxhash32" - ], - "edition": "2021" - }, - "library_target_name": "twox_hash", - "license": "MIT", - "license_file": "LICENSE.txt", - "license_ids": [ - "MIT" - ], - "name": "twox-hash", - "package_url": "https://github.com/shepmaster/twox-hash", - "repository": { - "Http": { - "sha256": "86a801b3cea342a06d468c8710662aa29e5e05e4f5c0d62f00bbb7f2ad7941c2", - "url": "https://static.crates.io/crates/twox-hash/2.1.5/download" - } - }, - "targets": [ - { - "Library": { - "crate_name": "twox_hash", - "crate_root": "src/lib.rs" - } - } - ], - "version": "2.1.5" - }, "typenum 1.20.1": { "common_attrs": { "crate_features": [ @@ -16090,7 +16015,6 @@ "libm 0.2.16", "libz-rs-sys 0.6.8", "lol_html_c_api 1.4.0", - "lz4_flex 0.11.6", "nix 0.31.3", "notify 8.2.1", "percent-encoding 2.3.2", diff --git a/src/workerd/server/cluster/BUILD.bazel b/src/workerd/server/cluster/BUILD.bazel index 5c014a06ee6..e5fceb0788e 100644 --- a/src/workerd/server/cluster/BUILD.bazel +++ b/src/workerd/server/cluster/BUILD.bazel @@ -18,12 +18,12 @@ wd_rust_crate( "@crates_vendor//:base64", "@crates_vendor//:bytes", "@crates_vendor//:capnp", + "@crates_vendor//:flate2", "@crates_vendor//:futures", "@crates_vendor//:http", "@crates_vendor//:http-body", "@crates_vendor//:httpdate", "@crates_vendor//:hyper-util", - "@crates_vendor//:lz4_flex", "@crates_vendor//:nix", "@crates_vendor//:ring", "@crates_vendor//:rustls", diff --git a/src/workerd/server/cluster/log.rs b/src/workerd/server/cluster/log.rs index 3e9a37488ab..183f58e3e58 100644 --- a/src/workerd/server/cluster/log.rs +++ b/src/workerd/server/cluster/log.rs @@ -6,7 +6,7 @@ //! relay's: one `sqlitePages` lane, the database, whose frames are the //! pages each commit wrote. //! -//! Under `do///`, each file one LZ4 frame: +//! Under `do///`, each file gzipped: //! //! - `log//`: a batch, the frames of a run of commits up to //! frame `end`, each run of page frames merged to one. @@ -149,18 +149,18 @@ pub fn merge(batch: &[u8]) -> Result> { Ok(out) } -/// LZ4-frames `bytes`, as every file in the bucket is. +/// Gzips `bytes`, as every file in the bucket is. pub fn compress(bytes: &[u8]) -> Result> { let mut encoder = encoder(Vec::new()); encoder.write_all(bytes)?; - encoder.finish().map_err(|e| Error::Replica(e.to_string())) + Ok(encoder.finish()?) } -/// An LZ4 frame with a checksum of its content, which the decoder checks: -/// a corrupt copy fails to replay rather than replaying wrong. -fn encoder(w: W) -> lz4_flex::frame::FrameEncoder { - let info = lz4_flex::frame::FrameInfo::new().content_checksum(true); - lz4_flex::frame::FrameEncoder::with_frame_info(info, w) +/// Gzip ends with the content's CRC-32 and length, which the decoder +/// checks: a corrupt or truncated copy fails to replay rather than +/// replaying wrong or short. +fn encoder(w: W) -> flate2::write::GzEncoder { + flate2::write::GzEncoder::new(w, flate2::Compression::fast()) } /// Bounds a message, so that a corrupt one fails rather than allocating @@ -169,35 +169,6 @@ fn options() -> capnp::message::ReaderOptions { *capnp::message::ReaderOptions::new().traversal_limit_in_words(Some(1 << 27)) } -/// Fails unless `file` is one whole LZ4 frame, through its end mark and -/// checksum: the decoder takes a frame cut at a block boundary for a -/// whole, shorter one. -fn check_whole(file: &mut std::fs::File) -> Result<()> { - let truncated = || Error::Replica("a truncated file".into()); - let len = file.metadata()?.len(); - let mut header = [0u8; 6]; - file.read_exact(&mut header).map_err(|_| truncated())?; - let flags = header[4]; - let bit = |mask: u8, n: u64| if flags & mask != 0 { n } else { 0 }; - let mut at = 7 + bit(0x08, 8) + bit(0x01, 4); - loop { - let mut size = [0u8; 4]; - file.seek(SeekFrom::Start(at))?; - file.read_exact(&mut size).map_err(|_| truncated())?; - at += 4; - let size = u32::from_le_bytes(size) & 0x7fff_ffff; - if size == 0 { - break; - } - at += u64::from(size) + bit(0x10, 4); - } - if at + bit(0x04, 4) != len { - return Err(truncated()); - } - file.seek(SeekFrom::Start(0))?; - Ok(()) -} - /// The database lane, replayed into a file. pub struct Replay { db: std::fs::File, @@ -229,10 +200,9 @@ impl Replay { /// Applies `f`, the local copy at `path`. A batch that does not reach /// back to the next frame is skipped: a later, wider one must cover it. pub fn apply(&mut self, f: &File, path: &Path) -> Result<()> { - let mut file = std::fs::File::open(path)?; - check_whole(&mut file)?; + let file = std::fs::File::open(path)?; let size = file.metadata()?.len(); - let mut r = BufReader::new(lz4_flex::frame::FrameDecoder::new(file)); + let mut r = BufReader::new(flate2::read::GzDecoder::new(file)); if f.snapshot { return self.snapshot(&mut r); } @@ -283,7 +253,7 @@ impl Replay { self.live = true; } } - // Read to the frame's end, where the decoder checks the checksum. + // Read to the end, where the decoder checks the checksum. if std::io::copy(r, &mut std::io::sink())? != 0 { return Err(Error::Replica("a snapshot longer than its lanes".into())); } @@ -397,9 +367,7 @@ fn write_snapshot( let mut encoder = encoder(std::io::BufWriter::new(std::fs::File::create(&tmp)?)); capnp::serialize::write_message(&mut encoder, &message)?; payload(&mut encoder)?; - let mut w = encoder - .finish() - .map_err(|e| Error::Replica(e.to_string()))?; + let mut w = encoder.finish()?; w.flush()?; drop(w); std::fs::rename(&tmp, &out)?; diff --git a/src/workerd/server/cluster/log_tests.rs b/src/workerd/server/cluster/log_tests.rs index adf39d33835..fd5a7d5846a 100644 --- a/src/workerd/server/cluster/log_tests.rs +++ b/src/workerd/server/cluster/log_tests.rs @@ -451,14 +451,15 @@ fn a_corrupt_file_fails_to_restore() { #[test] fn a_truncated_file_fails_to_restore() { - // The LZ4 decoder takes a frame without its end mark, or no frame at - // all, for a whole one. + // Cut anywhere, including before the trailer that holds the checksum, + // or empty. let dir = scratch("truncated"); let app = open_app(&dir.join("app.db")); insert(&app, 0, 100); let (file, path) = batch(&dir, 0, last()); let bytes = std::fs::read(&path).unwrap(); - for cut in [&bytes[..bytes.len() - 8], &[]] { + let len = bytes.len(); + for cut in [&bytes[..len - 1], &bytes[..len - 8], &bytes[..len / 2], &[]] { let store = Bucket::memory(""); std::fs::write(&path, cut).unwrap(); runtime().block_on(log(&store).put(&file, &path)).unwrap();