diff --git a/deps/rust/Cargo.lock b/deps/rust/Cargo.lock index 38020894e07..8ce972233a3 100644 --- a/deps/rust/Cargo.lock +++ b/deps/rust/Cargo.lock @@ -646,6 +646,7 @@ dependencies = [ "proc-macro2", "quote", "rand 0.9.5", + "ring", "ruff_python_ast", "ruff_python_parser", "rust_icu_sys", diff --git a/deps/rust/Cargo.toml b/deps/rust/Cargo.toml index d278c2bd7b5..8def4e053f3 100644 --- a/deps/rust/Cargo.toml +++ b/deps/rust/Cargo.toml @@ -77,6 +77,8 @@ pico-args = "0" rand = { version = "0.9", default-features = false, features = ["os_rng", "thread_rng"] } ruff_python_ast = { git = "https://github.com/astral-sh/ruff", tag = "0.12.1" } ruff_python_parser = { git = "https://github.com/astral-sh/ruff", tag = "0.12.1" } +# cluster: SigV4's SHA-256 and HMAC, and its peer MACs. +ring = { version = "0.17", default-features = false } # Use rust_icu's checked-in bindings for the Chromium ICU version linked by workerd. ICU itself is # supplied by Bazel, so bindgen, icu-config, and rust_icu's library-linking features stay disabled. rust_icu_sys = { version = "5.8", default-features = false, features = ["renaming", "icu_version_in_env"] } diff --git a/deps/rust/cargo-bazel-lock.json b/deps/rust/cargo-bazel-lock.json index 18348f4ce9d..a70d882adef 100644 --- a/deps/rust/cargo-bazel-lock.json +++ b/deps/rust/cargo-bazel-lock.json @@ -2,7 +2,7 @@ "binary_crates": [ "capnpc 0.27.0" ], - "checksum": "7c291f282fd54549d51644602320eca9c686cea67d071b55fb7e6b84270a7b27", + "checksum": "4f7c905fb454b3d367e5ae35edc0a5d3b38300d86a842ca721465a1cb99a6cc0", "conditions": { "aarch64-apple-darwin": [ "aarch64-apple-darwin" @@ -3018,6 +3018,10 @@ "id": "rand 0.9.5", "target": "rand" }, + { + "id": "ring 0.17.14", + "target": "ring" + }, { "id": "ruff_python_ast 0.0.0", "target": "ruff_python_ast" @@ -16018,6 +16022,7 @@ "proc-macro2 1.0.107", "quote 1.0.47", "rand 0.9.5", + "ring 0.17.14", "ruff_python_ast 0.0.0", "ruff_python_parser 0.0.0", "rust_icu_sys 5.8.0", diff --git a/src/workerd/server/AGENTS.md b/src/workerd/server/AGENTS.md index 7e48f711da5..20490043aea 100644 --- a/src/workerd/server/AGENTS.md +++ b/src/workerd/server/AGENTS.md @@ -7,6 +7,7 @@ The `workerd` binary and its server. `:workerd` (`workerd.rs`, this package) is - `cli/` -- the `workerd-cli` crate (declared in this package's `BUILD.bazel`, sources globbed from `cli/`): parses the command line (clap), produces the encoded config (schema files are compiled by this package's `config-compiler.c++`), handles `compile`, `--watch` and re-exec, and hands `serve`, `test` (which `fuzzilli` and `make-pyodide-baseline-snapshot` are built on) and `bench` to the server crate's `entry` module. - `server/` -- the `workerd-server` crate: the single-tenant server. It interprets the config, builds the service graph, binds the sockets, runs the listeners, actors and dynamic workers, drains on SIGTERM, and runs `workerd test` and `workerd bench`. `Error` is `cxx::KjError`; `bridge.rs` is the one module that allows `unsafe`. - `factory/` -- C++: `WorkerFactory` (`worker-factory*`), everything that needs the isolate, driven by the server through the cxx bridge in `server/bridge.rs`; and `bootstrap.{h,c++}`, the C++ half of a command. `with_process_context` sets up the process's logging (the JSON logger, when the config asks for structured logging) on the C++ stack around the Rust command; `bootstrap` sets up perfetto, autogates and V8 on the Rust-owned event loop and builds the `WorkerFactory`, which owns them, from the encoded config and the command-line options for `server/entry.rs`, which owns the loop (`kj_rs_tokio::Runtime`) and blocks on `server_serve` / `server_test` / `server_bench`. +- `cluster/` -- the `cluster` crate and, in `cluster/logic/`, `cluster-logic`: cluster mode (`Config.cluster`, experimental), Durable Objects whose shared state is an S3-compatible bucket. Leases, ownership, LTX capture and restore; pure Rust, driven by `server/cluster.rs`. `cluster-logic` holds decisions only, with no dependencies to do I/O with. This package keeps the config schema (`workerd.capnp`, `compiled-bindings.capnp`), the config compiler, the `WorkerdApi` and the C++ libraries the factory is built from (alarm scheduler, channel tokens, container client, facet tree index, fallback service, actor IDs, JSON logger, Pyodide, V8 platform), and their C++ unit tests. diff --git a/src/workerd/server/channel-token-test.c++ b/src/workerd/server/channel-token-test.c++ index f1cab81be8f..98c2d8bb161 100644 --- a/src/workerd/server/channel-token-test.c++ +++ b/src/workerd/server/channel-token-test.c++ @@ -414,6 +414,24 @@ KJ_TEST("actor channel tokens") { "channel token type mismatch", handler.decodeActorClassChannelToken(Usage::RPC, token)); } +KJ_TEST("instances of a cluster redeem each other's channel tokens") { + MockResolver senderResolver, receiverResolver, strangerResolver; + auto key = "test-cluster-shared-secret-of-32-bytes"_kj; + ChannelTokenHandler sender(senderResolver, key); + ChannelTokenHandler receiver(receiverResolver, key); + ChannelTokenHandler stranger(strangerResolver, "another-cluster-secret-of-32-bytes"_kj); + + const byte actorId[] = {12, 34, 56, 78}; + auto token = sender.encodeActorChannelToken( + Usage::RPC, "foo-namespace", actorId, "my-actor"_kj, Persistent::NO); + auto channel = + receiver.decodeSubrequestChannelToken(Usage::RPC, token).downcast(); + expectActorChannel(*channel, "foo-namespace", actorId, "my-actor"_kj); + + KJ_EXPECT_THROW_MESSAGE( + "failed authentication", stranger.decodeSubrequestChannelToken(Usage::RPC, token)); +} + KJ_TEST("channel token with nested channels (all synchronous)") { MockResolver resolver; ChannelTokenHandler handler(resolver); diff --git a/src/workerd/server/channel-token.c++ b/src/workerd/server/channel-token.c++ index e8967b8d77f..0c911faa690 100644 --- a/src/workerd/server/channel-token.c++ +++ b/src/workerd/server/channel-token.c++ @@ -9,6 +9,7 @@ #include #include +#include #include #include @@ -22,8 +23,18 @@ namespace workerd::server { -ChannelTokenHandler::ChannelTokenHandler(Resolver& resolver): resolver(resolver) { - getEntropy(tokenKey); +ChannelTokenHandler::ChannelTokenHandler(Resolver& resolver, kj::Maybe clusterKey) + : resolver(resolver) { + KJ_IF_SOME(key, clusterKey) { + // Labeled so this key never equals another one derived from the cluster key. + auto label = "workerd channel token v1"_kj; + uint outLen = 0; + KJ_ASSERT(HMAC(EVP_sha256(), label.begin(), label.size(), key.asBytes().begin(), key.size(), + tokenKey, &outLen) != nullptr); + KJ_ASSERT(outLen == sizeof(tokenKey)); + } else { + getEntropy(tokenKey); + } SHA256_CTX ctx{}; KJ_ASSERT(SHA256_Init(&ctx)); diff --git a/src/workerd/server/channel-token.h b/src/workerd/server/channel-token.h index 5d2a8f79672..e3fe946e5de 100644 --- a/src/workerd/server/channel-token.h +++ b/src/workerd/server/channel-token.h @@ -60,7 +60,8 @@ class ChannelTokenHandler: public kj::PtrTarget { IoChannelFactory::ChannelTokenUsage usage) = 0; }; - explicit ChannelTokenHandler(Resolver& resolver); + // Instances of a cluster share `clusterKey`, so each decodes the others' tokens. + explicit ChannelTokenHandler(Resolver& resolver, kj::Maybe clusterKey = kj::none); // Helpers to implement `IoChannelFactory::{SubrequestChannel,ActorClassChannel}::getToken()`. // diff --git a/src/workerd/server/cli/args-test.rs b/src/workerd/server/cli/args-test.rs index 09f8bc93061..4787dcaa452 100644 --- a/src/workerd/server/cli/args-test.rs +++ b/src/workerd/server/cli/args-test.rs @@ -65,6 +65,8 @@ fn serve_options() { "--async-trace=trace.ndjson", "--async-trace-stacks=8", "--async-trace-promises", + "--cluster-addr=0.0.0.0:7000", + "--cluster-advertise=10.0.0.4:7000", ]) .command else { @@ -85,6 +87,8 @@ fn serve_options() { assert_eq!(serve_or_test.async_trace_stacks, Some(8)); assert!(serve_or_test.async_trace_promises); assert_eq!(serve.control_fd, Some(3)); + assert_eq!(serve.cluster_address.as_deref(), Some("0.0.0.0:7000")); + assert_eq!(serve.cluster_advertise.as_deref(), Some("10.0.0.4:7000")); assert_eq!( serve.socket_addr_overrides, [Override { diff --git a/src/workerd/server/cli/args.rs b/src/workerd/server/cli/args.rs index 766fa92a7d5..153d4a13d38 100644 --- a/src/workerd/server/cli/args.rs +++ b/src/workerd/server/cli/args.rs @@ -55,7 +55,13 @@ pub enum Command { /// run the server #[command( long_about = "Serve requests based on a config.", - after_long_help = "Serves requests based on the configuration specified in ." + after_long_help = "Serves requests based on the configuration specified in .\n\n\ + Cluster mode (`Config.cluster`, with --experimental): instances that share an \ + S3-compatible bucket serve Durable Objects together. Each `cluster.bucketOptions` \ + setting falls back to an environment variable: AWS_ENDPOINT_URL, AWS_REGION, \ + AWS_ACCESS_KEY_ID, AWS_SECRET_ACCESS_KEY, AWS_SESSION_TOKEN, or \ + AWS_CREDENTIALS_FILE. The cluster port must stay on a private network; see \ + --cluster-addr." )] Serve { #[command(flatten)] @@ -325,6 +331,21 @@ pub struct ServeArgs { #[arg(long, value_name = "fd", value_parser = parse_control_fd)] pub control_fd: Option, + /// Bind the cluster's peer port to instead of the config's `cluster.address`. The + /// port gives full access to every service in the process to holders of the cluster key, and + /// its traffic is not encrypted: only the cluster's instances should reach it, on a private + /// network (a VPC or Tailscale). A public IP is refused unless the config sets + /// `cluster.unsafePublicAddress`; a wildcard (0.0.0.0) listens on every interface, so + /// firewall the port. + #[arg(long = "cluster-addr", value_name = "addr")] + pub cluster_address: Option, + + /// Advertise to the cluster's other instances instead of the config's + /// `cluster.advertise` or the bound address. A public IP is refused unless the config sets + /// `cluster.unsafePublicAddress`. + #[arg(long = "cluster-advertise", value_name = "addr")] + pub cluster_advertise: Option, + /// Listen on the specified address for debug RPC connections. This exposes a privileged /// interface that allows access to all services in the process. For use by miniflare and local /// development only. diff --git a/src/workerd/server/cli/lib.rs b/src/workerd/server/cli/lib.rs index 37a6eeb416c..2f3005a5640 100644 --- a/src/workerd/server/cli/lib.rs +++ b/src/workerd/server/cli/lib.rs @@ -467,6 +467,8 @@ fn serve_options( socket_fds, control, debug_port: serve.debug_port, + cluster_address: serve.cluster_address, + cluster_advertise: serve.cluster_advertise, ..run_options(serve_or_test) }; (options, inherited) diff --git a/src/workerd/server/cluster/BUILD.bazel b/src/workerd/server/cluster/BUILD.bazel new file mode 100644 index 00000000000..e5fceb0788e --- /dev/null +++ b/src/workerd/server/cluster/BUILD.bazel @@ -0,0 +1,42 @@ +load("//:build/wd_capnp_library.bzl", "wd_capnp_library") +load("//:build/wd_rust_crate.bzl", "wd_rust_crate") + +# Durable Objects whose shared state is an S3-compatible bucket. Pure Rust: +# the server (//src/workerd/server/server) drives it. +wd_rust_crate( + name = "cluster", + proc_macro_deps = ["@crates_vendor//:async-trait"], + # The capture's tests drive a real SQLite. + test_deps = ["@sqlite3"], + # The capture's tests hand SQLite Rust callbacks, which a downstream + # build's cfi-icall rejects. + test_tags = ["no-downstream"], + visibility = ["//src/workerd/server:__subpackages__"], + deps = [ + ":frames_capnp_rust", + "//src/workerd/server/cluster/logic:cluster-logic", + "@crates_vendor//:base64", + "@crates_vendor//:bytes", + "@crates_vendor//:capnp", + "@crates_vendor//:flate2", + "@crates_vendor//:futures", + "@crates_vendor//:http", + "@crates_vendor//:http-body", + "@crates_vendor//:httpdate", + "@crates_vendor//:hyper-util", + "@crates_vendor//:nix", + "@crates_vendor//:ring", + "@crates_vendor//:rustls", + "@crates_vendor//:rustls-platform-verifier", + "@crates_vendor//:serde", + "@crates_vendor//:serde_json", + "@crates_vendor//:thiserror", + "@crates_vendor//:tokio", + "@crates_vendor//:tokio-rustls", + "@crates_vendor//:tower-service", + "@crates_vendor//:tracing", + ], +) + +# The format of an object's log in the bucket. +wd_capnp_library(src = "frames.capnp") diff --git a/src/workerd/server/cluster/bucket.rs b/src/workerd/server/cluster/bucket.rs new file mode 100644 index 00000000000..f65e38200b3 --- /dev/null +++ b/src/workerd/server/cluster/bucket.rs @@ -0,0 +1,233 @@ +// Copyright (c) 2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +use std::path::Path; +use std::rc::Rc; + +use crate::Result; +pub use crate::http::Payload; + +/// The identity of one stored version of an object: what a conditional +/// write names to replace exactly that version. An S3 `ETag`. +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct Version(pub(crate) String); + +/// The precondition of a conditional write. +pub enum Guard { + /// The object must not exist. + Absent, + /// The object must still be this version. + Match(Version), +} + +/// One listed object; `key` is relative to the bucket's prefix. +pub struct Entry { + pub key: String, +} + +/// An object store: S3's semantics for the calls the cluster makes. +#[async_trait::async_trait(?Send)] +pub trait Store { + /// The object and its version; with `to`, the body goes to that file + /// instead. + async fn get(&self, key: &str, to: Option<&Path>) -> Result, Version)>>; + /// `Ok(None)` when the guard failed and nothing was written. + async fn put( + &self, + key: &str, + body: Payload, + guard: Option, + metadata: &[(&str, String)], + ) -> Result>; + async fn list(&self, prefix: &str) -> Result>; + /// Idempotent: deleting an absent key succeeds. + async fn delete(&self, key: &str) -> Result<()>; + /// See `S3::clock_skew_ms`. + fn clock_skew_ms(&self) -> Option { + None + } +} + +/// An object store under a key prefix. Clones share the store. +#[derive(Clone)] +pub struct Bucket { + store: Rc, + prefix: String, +} + +impl Bucket { + pub fn new(store: impl Store + 'static, prefix: &str) -> Self { + Self { + store: Rc::new(store), + prefix: normalize(prefix), + } + } + + /// The same store under another prefix. + #[must_use] + pub fn at(&self, prefix: &str) -> Self { + Self { + store: self.store.clone(), + prefix: normalize(prefix), + } + } + + #[cfg(test)] + pub fn memory(prefix: &str) -> Self { + Self::new(memory::Memory::default(), prefix) + } + + pub fn clock_skew_ms(&self) -> Option { + self.store.clock_skew_ms() + } + + pub(crate) fn prefix(&self) -> &str { + &self.prefix + } + + fn key(&self, key: &str) -> String { + format!("{}{key}", self.prefix) + } + + /// The object and its version, or `None` when it does not exist. + pub async fn get(&self, key: &str) -> Result, Version)>> { + self.store.get(&self.key(key), None).await + } + + /// Write `body` if the guard holds. `Ok(None)` means the guard failed and + /// nothing was written; an error means the write may have landed. + pub async fn put_if(&self, key: &str, body: Vec, guard: Guard) -> Result> { + let body = Payload::Bytes(body.into()); + self.store.put(&self.key(key), body, Some(guard), &[]).await + } + + /// Write `body` unconditionally, with `x-amz-meta-*` metadata. + pub async fn put(&self, key: &str, body: Vec, metadata: &[(&str, String)]) -> Result<()> { + let key = self.key(key); + let body = Payload::Bytes(body.into()); + self.store.put(&key, body, None, metadata).await.map(drop) + } + + /// Writes the object to the file `to`: `false` when it does not exist. + pub async fn get_file(&self, key: &str, to: &Path) -> Result { + let got = self.store.get(&self.key(key), Some(to)).await?; + Ok(got.is_some()) + } + + /// Writes the file `from` unconditionally, with `x-amz-meta-*` metadata. + pub async fn put_file( + &self, + key: &str, + from: &Path, + metadata: &[(&str, String)], + ) -> Result<()> { + let body = Payload::File(from.to_owned()); + self.store + .put(&self.key(key), body, None, metadata) + .await + .map(drop) + } + + /// The keys under `prefix` in lexical order. + pub async fn list(&self, prefix: &str) -> Result> { + let listed = self.store.list(&self.key(prefix)).await?; + Ok(listed + .into_iter() + .filter_map(|entry| { + let key = entry.key.strip_prefix(&self.prefix)?.to_owned(); + Some(Entry { key }) + }) + .collect()) + } + + /// Idempotent: deleting an absent key succeeds. + pub async fn delete(&self, key: &str) -> Result<()> { + self.store.delete(&self.key(key)).await + } +} + +fn normalize(prefix: &str) -> String { + let prefix = prefix.trim_matches('/'); + if prefix.is_empty() { + String::new() + } else { + format!("{prefix}/") + } +} + +#[cfg(test)] +pub mod memory { + use std::cell::Cell; + use std::cell::RefCell; + use std::collections::BTreeMap; + use std::path::Path; + + use super::Entry; + use super::Guard; + use super::Payload; + use super::Store; + use super::Version; + use crate::Result; + + /// S3's semantics for the calls the cluster makes, in memory. + #[derive(Default)] + pub struct Memory { + objects: RefCell, Version)>>, + writes: Cell, + /// As GCS's S3 API treats `If-Match` on a PUT. + pub ignores_guards: Cell, + } + + #[async_trait::async_trait(?Send)] + impl Store for Memory { + async fn get(&self, key: &str, to: Option<&Path>) -> Result, Version)>> { + let got = self.objects.borrow().get(key).cloned(); + match (got, to) { + (Some((body, version)), Some(to)) => { + std::fs::write(to, body)?; + Ok(Some((Vec::new(), version))) + } + (got, _) => Ok(got), + } + } + + async fn put( + &self, + key: &str, + body: Payload, + guard: Option, + _: &[(&str, String)], + ) -> Result> { + let body = match body { + Payload::Bytes(bytes) => bytes.to_vec(), + Payload::File(path) => std::fs::read(path)?, + }; + let mut objects = self.objects.borrow_mut(); + let current = objects.get(key).map(|(_, version)| version); + let holds = match guard { + None => true, + Some(Guard::Absent) => current.is_none(), + Some(Guard::Match(version)) => current == Some(&version), + }; + if !holds && !self.ignores_guards.get() { + return Ok(None); + } + self.writes.set(self.writes.get() + 1); + let version = Version(format!("\"{}\"", self.writes.get())); + objects.insert(key.to_owned(), (body, version.clone())); + Ok(Some(version)) + } + + async fn list(&self, prefix: &str) -> Result> { + let objects = self.objects.borrow(); + let keys = objects.keys().filter(|key| key.starts_with(prefix)); + Ok(keys.map(|key| Entry { key: key.clone() }).collect()) + } + + async fn delete(&self, key: &str) -> Result<()> { + self.objects.borrow_mut().remove(key); + Ok(()) + } + } +} diff --git a/src/workerd/server/cluster/capture.rs b/src/workerd/server/cluster/capture.rs new file mode 100644 index 00000000000..dca996b49bb --- /dev/null +++ b/src/workerd/server/cluster/capture.rs @@ -0,0 +1,191 @@ +// Copyright (c) 2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +//! A Durable Object's commits, cut from the writes `SQLite` makes to its WAL +//! as workerd's VFS reports them: per transaction, a frame of +//! `frames.capnp` with the pages it wrote. +//! +//! `SQLite` writes a frame as its 24-byte header and then its page, at offsets +//! that follow the 32-byte WAL header, and the header of a transaction's +//! last frame carries the database's size, which marks the commit. A write +//! at offset 0 starts a new WAL, after a checkpoint has copied every +//! earlier frame into the database file. So the frames since the last +//! commit are buffered, and the write that completes a commit frame +//! completes one transaction. A rolled-back transaction's frames start +//! where the next one's do, which overwrites them. + +use std::collections::BTreeMap; +use std::path::Path; +use std::path::PathBuf; + +use crate::Error; +use crate::Result; +use crate::Sequence; +use crate::log::Body; +use crate::log::Cursor; +use crate::log::push; + +const WAL_HEADER: u64 = 32; +const FRAME_HEADER: usize = 24; + +pub struct Capture { + /// The database file. + path: PathBuf, + /// The session's frames not yet in the bucket, by commit, keyed by the + /// frame after it. + entries: BTreeMap>, + cursor: Cursor, + page_size: usize, + /// Where the next transaction's first frame goes: 0 before a WAL header. + start: u64, + /// What was written from `start` on. + frames: Vec, + /// Why the capture can no longer be trusted, if it cannot. + lost: Option, +} + +impl Capture { + /// Capture the database at `path`, which no connection has open yet, + /// starting from what it holds now. + #[cfg(test)] + pub fn open(path: &Path) -> Result { + let mut capture = Self::resume(path, Cursor::default()); + capture.restart()?; + Ok(capture) + } + + /// Continue a session at `cursor` over the database at `path`, which + /// holds exactly the session's log so far and no connection has open + /// yet. Without a lane, the next commit creates it from the file, since + /// a new database's first page reaches the file, not the WAL. + pub fn resume(path: &Path, cursor: Cursor) -> Self { + Self { + path: path.to_owned(), + entries: BTreeMap::new(), + cursor, + page_size: 0, + start: 0, + frames: Vec::new(), + lost: None, + } + } + + /// Start the session over from the database file, which holds every + /// commit: no WAL, or one a checkpoint truncated. + #[cfg(test)] + pub fn restart(&mut self) -> Result<()> { + self.entries.clear(); + self.cursor = Cursor { + session: self.cursor.session, + ..Cursor::default() + }; + let image = match std::fs::read(&self.path) { + Err(e) if e.kind() == std::io::ErrorKind::NotFound => Vec::new(), + image => image?, + }; + if !image.is_empty() { + let mut entry = Vec::new(); + push(&mut entry, &mut self.cursor, Body::NewLane); + push(&mut entry, &mut self.cursor, Body::Initialize(&image)); + self.entries.insert(self.cursor.sequence, entry); + } + Ok(()) + } + + /// The database's files were deleted: the session continues without the + /// lane, and the next commit creates it. + pub fn reset(&mut self) { + self.start = 0; + self.frames.clear(); + if !self.cursor.lane { + return; + } + let mut entry = Vec::new(); + push(&mut entry, &mut self.cursor, Body::DeleteLane); + self.entries.insert(self.cursor.sequence, entry); + } + + /// A write `SQLite` made to the WAL. + pub fn write(&mut self, offset: u64, data: &[u8]) { + if offset == 0 { + // A new WAL: a checkpoint copied the last one into the file. + if let Some(size) = data.get(8..12) { + self.page_size = u32::from_be_bytes([size[0], size[1], size[2], size[3]]) as usize; + self.start = WAL_HEADER; + self.frames.clear(); + } + return; + } + if self.start == 0 || offset < self.start { + self.lost = Some(format!("a WAL write at {offset}, before {}", self.start)); + return; + } + let at = usize::try_from(offset - self.start).unwrap_or(usize::MAX); + let end = at.saturating_add(data.len()); + if self.frames.len() < end { + self.frames.resize(end, 0); + } + self.frames[at..end].copy_from_slice(data); + let size = FRAME_HEADER + self.page_size; + if !end.is_multiple_of(size) { + return; + } + let header = &self.frames[end - size..end - size + FRAME_HEADER]; + let commit = u32::from_be_bytes([header[4], header[5], header[6], header[7]]); + if commit != 0 { + self.commit(end, commit); + } + } + + /// The frames before `end` are one committed transaction. + fn commit(&mut self, end: usize, commit: u32) { + let mut entry = Vec::new(); + if !self.cursor.lane { + match std::fs::read(&self.path) { + Ok(image) => { + push(&mut entry, &mut self.cursor, Body::NewLane); + push(&mut entry, &mut self.cursor, Body::Initialize(&image)); + } + Err(e) => self.lost = Some(format!("the new database: {e}")), + } + } + let mut pages = BTreeMap::new(); + for frame in self.frames[..end].chunks_exact(FRAME_HEADER + self.page_size) { + let pgno = u32::from_be_bytes([frame[0], frame[1], frame[2], frame[3]]); + pages.insert(pgno, &frame[FRAME_HEADER..]); + } + pages.retain(|&pgno, _| pgno <= commit); + push(&mut entry, &mut self.cursor, Body::Pages(commit, &pages)); + self.start += end as u64; + self.frames.drain(..end); + self.entries.insert(self.cursor.sequence, entry); + } + + /// The frame after the last commit, or why the capture cannot be + /// trusted. + pub fn last(&self) -> Result { + match &self.lost { + None => Ok(self.cursor.sequence), + Some(why) => Err(Error::Replica(format!("capture lost: {why}"))), + } + } + + pub fn cursor(&self) -> Cursor { + self.cursor + } + + /// The frames from `from` up to `to`, both after a commit. + pub fn batch(&self, from: Sequence, to: Sequence) -> Result> { + let mut entries = self.entries.range(from + 1..=to).peekable(); + if entries.peek().is_none() { + return Err(Error::Replica(format!("no captured frames {from}..{to}"))); + } + Ok(entries.flat_map(|(_, e)| e.iter().copied()).collect()) + } + + /// The bucket holds everything up to `to`. + pub fn forget(&mut self, to: Sequence) { + self.entries.retain(|&end, _| end > to); + } +} diff --git a/src/workerd/server/cluster/cluster.rs b/src/workerd/server/cluster/cluster.rs new file mode 100644 index 00000000000..6c1a1936792 --- /dev/null +++ b/src/workerd/server/cluster/cluster.rs @@ -0,0 +1,367 @@ +// Copyright (c) 2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +//! One instance's view of the cluster: what the server calls. Runs on the +//! server's loop thread, inside its tokio `current_thread` runtime. + +use std::cell::RefCell; +use std::collections::HashMap; +use std::path::Path; +use std::rc::Rc; +use std::time::Duration; +use std::time::SystemTime; +use std::time::UNIX_EPOCH; + +use cluster_logic::lease; +use cluster_logic::lease::Lease; +use cluster_logic::lease::Step; +use tokio::time::timeout; + +use crate::Bucket; +use crate::Claim; +use crate::Error; +use crate::Guard; +use crate::Ownership; +use crate::Registry; +use crate::Result; +use crate::Version; +use crate::hmac_sha256; +use crate::owner::Owners; +use crate::ownership::Owned; +use crate::s3::S3; + +/// The bucket layout's version, the last element of every key's prefix. +const FORMAT: &str = "v1"; + +/// Claims a new object retries when another node opened it first. +const FRESH_ATTEMPTS: usize = 3; + +pub struct Setup { + /// `s3:///`. + pub url: String, + /// `name=value` bucket options (see `S3::new`). + pub options: Vec, + /// This instance's identity, fresh per process. + pub node: String, + /// Where peers reach this instance: `host:port`. + pub address: String, + pub ttl_ms: u64, + /// The cluster secret: MACs every peer call. + pub key: Vec, +} + +/// Where a Durable Object runs. +pub enum Route { + /// Here, at this ownership epoch: take it over with `restore_actor`. + Owned(u64), + /// On another instance. `stale`: the record names a node that looks + /// dead, so the caller should `refresh_route` in the background. + Remote { + node: String, + address: String, + stale: bool, + }, +} + +pub struct Cluster { + registry: Rc, + ownership: Ownership, + owners: Owners, + /// Claims won but not yet taken over, for `restore_actor`. + claims: RefCell>, + bucket: Bucket, + key: Vec, + lease: RefCell, +} + +impl Cluster { + pub fn new(setup: Setup) -> Result> { + let options: Vec<_> = setup + .options + .iter() + .filter_map(|option| option.split_once('=')) + .map(|(key, value)| (key.to_owned(), value.to_owned())) + .collect(); + let (s3, prefix) = S3::new(&setup.url, &options)?; + let bucket = Bucket::new(s3, &format!("{prefix}/{FORMAT}")); + let registry = Rc::new(Registry::new( + bucket.clone(), + &setup.node, + &setup.address, + setup.ttl_ms, + )); + Ok(Rc::new(Self { + owners: Owners::new(bucket.clone()), + ownership: Ownership::new(bucket.clone(), registry.clone()), + bucket, + registry, + claims: RefCell::new(HashMap::new()), + key: setup.key, + lease: RefCell::new(Lease::default()), + })) + } + + pub fn node(&self) -> &str { + self.registry.node() + } + + pub async fn renew(&self, now_ms: u64) -> Result { + self.registry.renew(now_ms).await + } + + /// How far this node's wall clock is outside the bucket's, in ms; 0 + /// before any response. + pub fn clock_skew_ms(&self) -> i64 { + self.bucket.clock_skew_ms().unwrap_or(0) + } + + pub async fn deregister(&self) -> Result<()> { + self.registry.deregister().await + } + + /// Live peers' records, as the bucket lists them now. + pub async fn peers(&self, now_ms: u64) -> Result> { + self.registry.peers(now_ms).await + } + + pub async fn acquire_or_route(&self, ns: &str, actor: &str, now_ms: u64) -> Result { + let mut claim = self.ownership.acquire_or_route(ns, actor, now_ms).await?; + // Named open here, but not open here. Evicted here with the bucket + // holding the epoch: resume it (`take_over`); a stale owner that + // writes on cannot acknowledge, since every acknowledgement rereads + // own.json. Otherwise, dropped after a failure or evicted before its + // upload completed: recover it at the next epoch. + if let Claim::Owned(owned) = &claim + && owned.recover.is_none() + && !self.owners.holds(ns, actor) + && self.owners.dormant(ns, actor) != Some(owned.epoch) + { + let next = self.ownership.advance(ns, actor, owned.epoch).await?; + // Changed meanwhile, by another claim here: claim what it left. + claim = match next { + Some(next) => Claim::Owned(next), + None => self.ownership.acquire_or_route(ns, actor, now_ms).await?, + }; + } + // A new object opens here, where losing the race still routes. + for _ in 0..FRESH_ATTEMPTS { + let Claim::Owned(owned) = &claim else { break }; + if owned.recover.is_none_or(|from| from != 0) + || self + .owners + .open_fresh(&self.ownership, ns, actor, owned) + .await? + { + break; + } + claim = self.ownership.acquire_or_route(ns, actor, now_ms).await?; + } + Ok(match claim { + Claim::Owned(owned) => { + let epoch = owned.epoch; + let key = format!("{ns}/{actor}"); + self.claims.borrow_mut().insert(key, owned); + Route::Owned(epoch) + } + Claim::Route { + node, + address, + stale, + } => Route::Remote { + node, + address, + stale, + }, + }) + } + + /// Take a claimed object over: recover what earlier epochs committed + /// into the database at `path`, and open the claimed epoch. + pub async fn restore_actor(&self, ns: &str, actor: &str, epoch: u64, path: &str) -> Result<()> { + let claim = self.claims.borrow_mut().remove(&format!("{ns}/{actor}")); + let owned = claim.filter(|owned| owned.epoch == epoch).ok_or_else(|| { + crate::Error::Replica(format!("{ns}/{actor}: no claim at epoch {epoch}")) + })?; + self.owners + .take_over(&self.ownership, ns, actor, &owned, Path::new(path)) + .await + } + + /// Make an object's captured commits durable in the bucket. + pub async fn sync_actor(&self, ns: &str, actor: &str) -> Result<()> { + self.owners.commit(&self.ownership, ns, actor).await + } + + /// Stop capturing an object that left this node. + pub fn close_actor(&self, ns: &str, actor: &str) { + self.owners.retire(ns, actor); + } + + /// An owned object's database was deleted, before anything writes a + /// new one. + pub fn database_reset(&self, ns: &str, actor: &str) { + self.owners.database_reset(ns, actor); + } + + /// A write SQLite made to a WAL file in namespace `ns`'s storage. + pub fn wal_write(&self, ns: &str, file: &str, offset: u64, data: &[u8]) { + // `.sqlite-wal`, under whatever directory SQLite was given. + let name = file.rsplit('/').next().unwrap_or(file); + if let Some(actor) = name.strip_suffix(".sqlite-wal") { + self.owners.wal_write(ns, actor, offset, data); + } + } + + /// Compact owned objects' logs in the bucket. + pub async fn tier(&self) { + self.owners.tier().await; + } + + /// Keep an object workerd evicted, uploading what the bucket lacks. + pub async fn evict_actor(&self, ns: &str, actor: &str, epoch: u64) { + self.owners.evict(ns, actor, epoch).await; + } + + pub async fn refresh_route(&self, ns: &str, actor: &str, now_ms: u64) -> Result<()> { + self.ownership.refresh_route(ns, actor, now_ms).await + } + + /// The MAC a peer call carries. The cluster key is used only through + /// labelled derivations, so no two uses share a key: this, with each + /// message labelled, and the channel token key. + pub fn mac(&self, data: &[u8]) -> Vec { + hmac_sha256(&self.key, data) + } +} + +impl Cluster { + /// Keeps this node's lease, until the node must stop + /// serving: answers why. Runs for the life of the process. + pub async fn maintain(&self) -> String { + loop { + if self.lease.borrow().draining { + std::future::pending::<()>().await; + } + let observed = lease::Observed { + now_ms: now_ms(), + skew_ms: self.clock_skew_ms(), + }; + let mut step = self.lease.borrow().before_renew(&observed); + if step == Step::Renew { + let renew = timeout(ms(lease::RENEW_TIMEOUT_MS), self.renew(now_ms())); + let result = match renew.await { + Ok(Ok(until)) => Ok(until), + Ok(Err(e)) => Err(e.to_string()), + Err(_) => Err("renewal timed out".into()), + }; + // A drain began meanwhile, and its deregistration may have + // failed this renewal: what it found no longer matters. + if self.lease.borrow().draining { + continue; + } + step = self.lease.borrow_mut().renewed(now_ms(), result); + } + let delay = match step { + Step::Fence(reason) => return reason, + Step::Retry(reason) => { + tracing::warn!("renewing the cluster lease failed; retrying: {reason}"); + lease::RETRY_MS + } + Step::Renew => lease::INTERVAL_MS, + }; + // A stale listing only delays routing to a new peer: its + // failure is logged, never fatal. + match timeout(ms(lease::LIST_TIMEOUT_MS), self.peers(now_ms())).await { + Ok(Ok(_)) => {} + Ok(Err(e)) => tracing::warn!("listing the cluster failed: {e}"), + Err(_) => tracing::warn!("listing the cluster timed out"), + } + tokio::time::sleep(ms(delay)).await; + } + } + + /// Compacts owned objects' logs. Never returns. + pub async fn tier_loop(&self) { + loop { + tokio::time::sleep(TIER_INTERVAL).await; + self.tier().await; + } + } + + /// Stops claiming before the claim reaches the bucket, and parks the + /// lease loop; then `deregister`. + pub fn start_draining(&self) { + self.lease.borrow_mut().draining = true; + self.ownership.may_claim.set(false); + } + + /// The first renewal, before the node serves. + pub async fn join(&self) -> Result<()> { + check_conditional_writes(&self.bucket, self.node()).await?; + let until = self.renew(now_ms()).await?; + self.lease.borrow_mut().renewed(now_ms(), Ok(until)); + Ok(()) + } + + /// Whether this node's lease holds now: an actor database opens only + /// while it does. + pub fn lease_holds(&self) -> bool { + self.lease.borrow().holds(now_ms()) + } +} + +/// How often owned objects' logs are checked for compaction. +const TIER_INTERVAL: Duration = Duration::from_secs(2); + +const fn ms(ms: u64) -> Duration { + Duration::from_millis(ms) +} + +pub fn now_ms() -> u64 { + let since = SystemTime::now() + .duration_since(UNIX_EPOCH) + .unwrap_or_default(); + u64::try_from(since.as_millis()).unwrap_or(u64::MAX) +} + +/// Refuse a store that ignores write preconditions: fencing rests on +/// them. GCS's S3 API, for one, ignores `If-Match` on a PUT. +async fn check_conditional_writes(bucket: &Bucket, node: &str) -> Result<()> { + let key = format!("nodes/{node}.probe"); + let stale = Guard::Match(Version("\"0\"".into())); + let created = bucket.put_if(&key, vec![], Guard::Absent).await?; + let again = bucket.put_if(&key, vec![], Guard::Absent).await?; + let replaced = bucket.put_if(&key, vec![], stale).await?; + bucket.delete(&key).await?; + if created.is_none() || again.is_some() || replaced.is_some() { + return Err(Error::Config( + "the bucket ignores conditional writes (If-None-Match, If-Match)".into(), + )); + } + Ok(()) +} + +#[cfg(test)] +mod tests { + use futures::executor::block_on; + + use super::check_conditional_writes; + use crate::Bucket; + use crate::bucket::memory::Memory; + + #[test] + fn a_bucket_that_ignores_preconditions_is_refused() { + block_on(async { + assert!( + check_conditional_writes(&Bucket::memory(""), "n") + .await + .is_ok() + ); + let lax = Memory::default(); + lax.ignores_guards.set(true); + let lax = Bucket::new(lax, ""); + assert!(check_conditional_writes(&lax, "n").await.is_err()); + }); + } +} diff --git a/src/workerd/server/cluster/frames.capnp b/src/workerd/server/cluster/frames.capnp new file mode 100644 index 00000000000..12eb61f23c8 --- /dev/null +++ b/src/workerd/server/cluster/frames.capnp @@ -0,0 +1,172 @@ +# Copyright (c) 2026 Cloudflare, Inc. +# Licensed under the Apache 2.0 license found in the LICENSE file or at: +# https://opensource.org/licenses/Apache-2.0 + +@0xf699edadaece33b7; +# The format of a Durable Object's log in the bucket: the storage relay's +# (SRS's), field for field, so the two stay wire-compatible, and one lane +# type of workerd's own, `sqlitePages`, at new ordinals. Workerd leaves +# unset what it has no use for, noted below. Compression and encryption are +# not part of it. +# +# A log batch is a run of frames, each one message, that starts and ends at +# a commit. A snapshot is a `SnapshotHeader` and then each lane's content in +# lane order. + +using Cxx = import "/capnp/c++.capnp"; +$Cxx.namespace("workerd::server::cluster"); + +struct Frame { + # One change to an object's state. + + session @0 :UInt32; + # The ownership epoch that wrote the frame. + + sequence @1 :UInt32; + # Position within the session, from zero. A position between two frames + # is the sequence of the one after it. + + lane @2 :UInt32; + # A lane is numbered on first use, with the lowest free number. + + const unknownContinueFromSession :UInt32 = 0xffffffff; + + continueFromSession @10 :UInt32 = Frame.unknownContinueFromSession; + continueFromSequence @11 :UInt32; + # For a session's first frame: where the last non-empty session ended. + # Workerd leaves these unknown: own.json names the session a session + # recovered. + + union { + newLane @3 :LaneDescription; + + cloneLane :group { + # Unused by workerd. + name @13 :Text; + from @14 :UInt32; + } + + deleteLane @4 :Void; + + write :group { + # For a `sqlite` lane, bytes SQLite wrote to the WAL, at an offset + # that keeps growing across checkpoints, though SQLite sees each WAL + # start over at zero. Unused by workerd. + offset @5 :UInt64; + data @6 :Data; + } + + obsolete7 @7 :Void; + + consume :group { + # For a `sqlite` lane, a checkpoint copied the WAL's committed pages + # into the database: data before `offset` is not needed again. Unused + # by workerd. + offset @8 :UInt64; + } + + initialize @9 :Data; + # For a `sqlite` or `sqlitePages` lane, the whole database file, + # written before the WAL is in use. + + setReplicaInformation @12 :AnyPointer; + # Unused by workerd. + + pages :group { + # For a `sqlitePages` lane, the newest version of each page that the + # commits `first..=sequence` wrote, and the database's size after, in + # pages. A batch's commits merge to one such frame between other + # frames, so a page a hot object rewrites is in the log once per + # upload, not once per commit. + first @15 :UInt32; + commit @16 :UInt32; + list @17 :List(Page); + } + } +} + +struct Page { + number @0 :UInt32; + data @1 :Data; +} + +struct LaneDescription { + name @0 :Text; + # The lane's stable identifier; numbers differ between objects. + + union { + sqlite @1 :Void; + journal @2 :Void; + # Unused by workerd. + replicaInformation @3 :Void; + # Unused by workerd. + sqlitePages @4 :Void; + # Workerd's: a SQLite database whose writes are pages (`Frame.pages`). + } +} + +enum SnapshotJustification { + # Workerd writes only `clientRequested` and `coldStartMitigation`. + + unknown @0; + clientRequested @1; + # The log written since the last snapshot outgrew the database. + pointInTimeRecovery @2; + snapshotFallback @3; + lostAndFoundRecovery @4; + coldStartMitigation @5; + # Workerd's snapshot of the state a session starts from, so the next + # restore reads one file and no earlier session's log. + cleanShutdown @6; + test @7; + srsTools @8; + localOnly @9; +} + +struct SnapshotHeader { + # The whole state of an object at a session and frame; the log continues + # from `frame`. + + session @0 :UInt32; + frame @1 :UInt32; + + lanes @2 :List(Lane); + # Every lane, by number. + + struct Lane { + name @0 :Text; + + union { + deleted @1 :Void; + # A lane number freed while later ones are in use. + + sqlite :group { + # Also a `sqlitePages` lane's, with only `databaseSize` set: the + # database file, every committed page in it. + databaseSize @2 :UInt64; + walSize @3 :UInt64; + walBaseOffset @6 :UInt64; + walShiftOffset @7 :UInt64; + } + + journal :group { + # Unused by workerd, as is `replicaInformation`. + size @4 :UInt64; + offset @5 :UInt64; + } + + replicaInformation :group { + size @8 :UInt64; + } + } + } + + logBytesSinceLastSnapshot @3 :UInt64; + # The log bytes in the bucket since the previous snapshot. + + continueFromSession @4 :UInt32 = Frame.unknownContinueFromSession; + continueFromSequence @5 :UInt32; + # Unknown, as in `Frame`. + + justification @6 :SnapshotJustification; +} diff --git a/src/workerd/server/cluster/http-test.rs b/src/workerd/server/cluster/http-test.rs new file mode 100644 index 00000000000..ab8cde9e915 --- /dev/null +++ b/src/workerd/server/cluster/http-test.rs @@ -0,0 +1,35 @@ +// Copyright (c) 2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +use tokio::io::AsyncReadExt; +use tokio::io::AsyncWriteExt; +use tokio::net::TcpListener; + +use super::*; + +#[test] +fn a_body_held_in_memory_is_capped() { + let rt = tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap(); + rt.block_on(async { + let listener = TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + tokio::spawn(async move { + let (mut stream, _) = listener.accept().await.unwrap(); + let _ = stream.read(&mut [0; 4096]).await; + let size = MAX_BODY + 1; + let head = format!("HTTP/1.1 200 OK\r\ncontent-length: {size}\r\n\r\n"); + stream.write_all(head.as_bytes()).await.unwrap(); + let _ = stream.write_all(&vec![0; size]).await; + }); + let http = Http::new().unwrap(); + let wait = Duration::from_secs(10); + let url = format!("http://127.0.0.1:{port}/"); + let payload = Payload::Bytes(bytes::Bytes::new()); + let sent = http.send("GET".into(), url, vec![], payload, None, (wait, wait)); + assert_eq!(sent.await.err().as_deref(), Some("response body too large")); + }); +} diff --git a/src/workerd/server/cluster/http.rs b/src/workerd/server/cluster/http.rs new file mode 100644 index 00000000000..a90b71ee407 --- /dev/null +++ b/src/workerd/server/cluster/http.rs @@ -0,0 +1,297 @@ +// Copyright (c) 2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +//! The bucket's HTTP client: hyper's pooled HTTP/1.1 client over TCP, with +//! rustls and the platform's roots for `https` endpoints. + +use std::future::Future; +use std::future::poll_fn; +use std::io; +use std::io::Read; +use std::io::Write; +use std::path::Path; +use std::path::PathBuf; +use std::pin::Pin; +use std::sync::Arc; +use std::task::Context; +use std::task::Poll; +use std::time::Duration; + +use http::Uri; +use http_body::Body as _; +use http_body::Frame; +use hyper_util::client::legacy; +use hyper_util::client::legacy::connect::Connected; +use hyper_util::client::legacy::connect::Connection; +use hyper_util::rt::TokioExecutor; +use hyper_util::rt::TokioIo; +use rustls::ClientConfig; +use rustls::pki_types::ServerName; +use rustls_platform_verifier::ConfigVerifierExt; +use tokio::io::AsyncRead; +use tokio::io::AsyncWrite; +use tokio::io::ReadBuf; +use tokio::net::TcpStream; +use tokio_rustls::TlsConnector; +use tokio_rustls::client::TlsStream; + +/// A bucket response: only the parts the S3 client reads. +pub struct HttpResponse { + pub status: u16, + pub etag: String, + /// The `Date` header, the bucket's clock. + pub date: String, + /// Empty when a 2xx body went to a file. + pub body: Vec, +} + +/// What a request sends: bytes, or a file read in chunks on the blocking +/// pool. +#[derive(Clone)] +pub enum Payload { + Bytes(bytes::Bytes), + File(PathBuf), +} + +/// The chunks a file is sent and received in. +const CHUNK: usize = 256 << 10; +/// The most of a response body held in memory rather than written to a file. +const MAX_BODY: usize = 16 << 20; + +pub struct Http { + client: legacy::Client, +} + +impl Http { + pub fn new() -> Result { + let tls = Arc::new(ClientConfig::with_platform_verifier()?); + let client = legacy::Client::builder(TokioExecutor::new()).build(Connector(tls)); + Ok(Self { client }) + } + + /// `headers` alternates names and values. A 2xx body goes to the file + /// `to`, if any. `head` bounds the wait for the response's head, which + /// includes sending the body; `idle`, each wait for the body's next + /// chunk. + pub async fn send( + &self, + method: String, + url: String, + headers: Vec, + payload: Payload, + to: Option<&Path>, + (head, idle): (Duration, Duration), + ) -> Result { + let mut request = http::Request::builder().method(method.as_str()).uri(url); + for pair in headers.chunks(2) { + request = request.header(&pair[0], &pair[1]); + } + let body = match payload { + Payload::Bytes(bytes) => Body::Bytes(Some(bytes)), + Payload::File(path) => Body::file(&path).map_err(|e| e.to_string())?, + }; + let request = request.body(body).map_err(|e| e.to_string())?; + let response = tokio::time::timeout(head, self.client.request(request)) + .await + .map_err(|_| "timed out".to_owned())? + .map_err(|e| e.to_string())?; + let header = |name| { + let value = response.headers().get(name); + value.and_then(|v| v.to_str().ok()).unwrap_or("").to_owned() + }; + let (status, etag, date) = (response.status().as_u16(), header("etag"), header("date")); + let mut incoming = response.into_body(); + let to = to + .filter(|_| (200..300).contains(&status)) + .map(Path::to_owned); + let (tx, mut rx) = tokio::sync::mpsc::channel::(4); + let writer = to.map(|path| { + tokio::task::spawn_blocking(move || -> io::Result<()> { + let mut file = io::BufWriter::new(std::fs::File::create(path)?); + while let Some(chunk) = rx.blocking_recv() { + file.write_all(&chunk)?; + } + file.flush() + }) + }); + let mut body = Vec::new(); + loop { + let frame = poll_fn(|cx| Pin::new(&mut incoming).poll_frame(cx)); + let frame = tokio::time::timeout(idle, frame) + .await + .map_err(|_| "timed out reading the body".to_owned())?; + let Some(frame) = frame else { break }; + let chunk = frame + .map_err(|e| e.to_string())? + .into_data() + .unwrap_or_default(); + if writer.is_none() { + // Records, listings and errors: an endpoint sending more is broken. + if body.len() + chunk.len() > MAX_BODY { + return Err("response body too large".to_owned()); + } + body.extend(chunk); + } else if tx.send(chunk).await.is_err() { + break; + } + } + drop(tx); + if let Some(writer) = writer { + writer + .await + .map_err(|e| e.to_string())? + .map_err(|e| e.to_string())?; + } + Ok(HttpResponse { + status, + etag, + date, + body, + }) + } +} + +/// A request body: bytes in one piece, or a file's chunks as a blocking +/// task reads them. Either way its exact size is known, which S3 needs. +enum Body { + Bytes(Option), + File { + chunks: tokio::sync::mpsc::Receiver>, + size: u64, + }, +} + +impl Body { + fn file(path: &Path) -> io::Result { + let mut file = std::fs::File::open(path)?; + let size = file.metadata()?.len(); + let (tx, chunks) = tokio::sync::mpsc::channel(4); + tokio::task::spawn_blocking(move || { + loop { + let mut chunk = vec![0; CHUNK]; + match file.read(&mut chunk) { + Ok(0) => return, + Ok(n) => chunk.truncate(n), + Err(e) => return drop(tx.blocking_send(Err(e))), + } + if tx.blocking_send(Ok(chunk.into())).is_err() { + return; + } + } + }); + Ok(Self::File { chunks, size }) + } +} + +impl http_body::Body for Body { + type Data = bytes::Bytes; + type Error = io::Error; + + fn poll_frame( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + ) -> Poll, Self::Error>>> { + match self.get_mut() { + Self::Bytes(bytes) => Poll::Ready(bytes.take().map(|data| Ok(Frame::data(data)))), + Self::File { chunks, .. } => { + chunks.poll_recv(cx).map(|c| c.map(|c| c.map(Frame::data))) + } + } + } + + fn size_hint(&self) -> http_body::SizeHint { + match self { + Self::Bytes(bytes) => { + http_body::SizeHint::with_exact(bytes.as_ref().map_or(0, |b| b.len() as u64)) + } + Self::File { size, .. } => http_body::SizeHint::with_exact(*size), + } + } +} + +#[derive(Clone)] +struct Connector(Arc); + +impl tower_service::Service for Connector { + type Response = TokioIo; + type Error = io::Error; + type Future = Pin> + Send>>; + + fn poll_ready(&mut self, _: &mut Context<'_>) -> Poll> { + Poll::Ready(Ok(())) + } + + fn call(&mut self, uri: Uri) -> Self::Future { + let tls = self.0.clone(); + Box::pin(async move { + let invalid = || io::Error::new(io::ErrorKind::InvalidInput, uri.to_string()); + let host = uri.host().ok_or_else(invalid)?.to_owned(); + let https = uri.scheme_str() == Some("https"); + let port = uri.port_u16().unwrap_or(if https { 443 } else { 80 }); + let tcp = TcpStream::connect((host.as_str(), port)).await?; + tcp.set_nodelay(true)?; + if !https { + return Ok(TokioIo::new(Stream::Tcp(tcp))); + } + let name = ServerName::try_from(host).map_err(|_| invalid())?; + let tls = TlsConnector::from(tls).connect(name, tcp).await?; + Ok(TokioIo::new(Stream::Tls(Box::new(tls)))) + }) + } +} + +enum Stream { + Tcp(TcpStream), + Tls(Box>), +} + +impl Connection for Stream { + fn connected(&self) -> Connected { + Connected::new() + } +} + +impl AsyncRead for Stream { + fn poll_read( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &mut ReadBuf<'_>, + ) -> Poll> { + match self.get_mut() { + Self::Tcp(s) => Pin::new(s).poll_read(cx, buf), + Self::Tls(s) => Pin::new(s).poll_read(cx, buf), + } + } +} + +impl AsyncWrite for Stream { + fn poll_write( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + buf: &[u8], + ) -> Poll> { + match self.get_mut() { + Self::Tcp(s) => Pin::new(s).poll_write(cx, buf), + Self::Tls(s) => Pin::new(s).poll_write(cx, buf), + } + } + + fn poll_flush(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + match self.get_mut() { + Self::Tcp(s) => Pin::new(s).poll_flush(cx), + Self::Tls(s) => Pin::new(s).poll_flush(cx), + } + } + + fn poll_shutdown(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll> { + match self.get_mut() { + Self::Tcp(s) => Pin::new(s).poll_shutdown(cx), + Self::Tls(s) => Pin::new(s).poll_shutdown(cx), + } + } +} + +#[cfg(test)] +#[path = "http-test.rs"] +mod tests; diff --git a/src/workerd/server/cluster/lib.rs b/src/workerd/server/cluster/lib.rs new file mode 100644 index 00000000000..e8ca3a84d8d --- /dev/null +++ b/src/workerd/server/cluster/lib.rs @@ -0,0 +1,134 @@ +// Copyright (c) 2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +//! A Durable Object cluster whose shared state is an S3-compatible bucket +//! (S3, R2, `MinIO`) instead of a shared filesystem. +//! +//! Everything lives under `v1/` in the configured prefix, so a later format +//! can tell an old bucket from its own: +//! +//! - `nodes/.json`: one per running instance, with its address and a +//! lease expiry the instance renews. A peer is live while its lease is. +//! - `do///own.json`: one per Durable Object, naming its +//! owner and an ownership epoch. A claim is a conditional write; taking an +//! object over from an owner whose lease expired bumps the epoch. +//! - `do///{log,snapshot}//..`: the object's log +//! (`log.rs`), a session per epoch. A takeover restores the last opened +//! epoch's session and starts the next from a snapshot of it, so a late +//! write from the old owner lands where nothing reads it. +//! +//! `` is a hash of the namespace's `uniqueKey`, and `` the +//! actor ID's hex, so no key's length depends on a name. +//! +//! The clock is an argument everywhere (`now_ms`), so a test decides time. + +#![deny( + clippy::expect_used, + clippy::panic, + clippy::unreachable, + clippy::unwrap_used +)] + +mod bucket; +mod capture; +pub mod cluster; +mod http; +mod log; +#[cfg(test)] +mod log_tests; +mod owner; +mod ownership; +mod registry; +pub mod s3; + +pub use bucket::Bucket; +pub use bucket::Guard; +pub use bucket::Store; +pub use bucket::Version; +pub use cluster::Cluster; +pub use cluster::Route; +pub use cluster::Setup; +pub use cluster::now_ms; +pub use owner::Owners; +pub use ownership::Claim; +pub use ownership::Ownership; +pub use registry::NodeRecord; +pub use registry::Registry; + +#[derive(Debug, thiserror::Error)] +pub enum Error { + /// The bucket URL or an option is missing or malformed. + #[error("bucket config: {0}")] + Config(String), + /// A claim or a commit's capture could not proceed. + #[error("replica: {0}")] + Replica(String), + /// A bucket request failed or answered with an unexpected status. + #[error("bucket: {0}")] + Http(String), + #[error("record: {0}")] + Codec(#[from] serde_json::Error), + /// Another writer kept winning the conditional write. + #[error("{0} is contended")] + Contended(String), + /// Someone else rewrote this node's own record: this process no longer + /// holds its identity and must stop serving. + #[error("node record {0} was rewritten by another writer")] + Fenced(String), + #[error("io: {0}")] + Io(#[from] std::io::Error), + /// A log or snapshot does not decode. + #[error("log: {0}")] + Log(#[from] capnp::Error), + #[error("log: {0}")] + Schema(#[from] capnp::NotInSchema), +} + +pub type Result = std::result::Result; + +/// An ownership epoch: each takeover of an object bumps it. +pub type Epoch = u64; +/// A position in an epoch's session of the log: the sequence of the frame +/// after it. +pub type Sequence = u32; + +/// Where an object's records are: `do//`, the namespace +/// as the first 16 bytes of a labelled SHA-256 of its tenant, empty for +/// now, and `uniqueKey`, in base64url. +pub(crate) fn object_prefix(ns: &str, actor: &str) -> String { + use base64::Engine; + let mut data = Vec::new(); + for part in ["workerd cluster namespace v1", "", ns] { + data.extend((part.len() as u64).to_be_bytes()); + data.extend(part.as_bytes()); + } + let hash = base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(&sha256(&data)[..16]); + format!("do/{hash}/{actor}") +} + +pub(crate) fn sha256(data: &[u8]) -> Vec { + ring::digest::digest(&ring::digest::SHA256, data) + .as_ref() + .to_vec() +} + +pub(crate) fn hmac_sha256(key: &[u8], data: &[u8]) -> Vec { + let key = ring::hmac::Key::new(ring::hmac::HMAC_SHA256, key); + ring::hmac::sign(&key, data).as_ref().to_vec() +} + +pub fn hex(bytes: &[u8]) -> String { + use std::fmt::Write; + bytes.iter().fold(String::new(), |mut out, byte| { + let _ = write!(out, "{byte:02x}"); + out + }) +} + +pub fn unhex(text: &str) -> Option> { + (0..text.len()) + .step_by(2) + .map(|i| u8::from_str_radix(text.get(i..i + 2)?, 16).ok()) + .collect() +} diff --git a/src/workerd/server/cluster/log.rs b/src/workerd/server/cluster/log.rs new file mode 100644 index 00000000000..183f58e3e58 --- /dev/null +++ b/src/workerd/server/cluster/log.rs @@ -0,0 +1,644 @@ +// Copyright (c) 2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +//! An object's log in the bucket (`frames.capnp`), laid out as the storage +//! relay's: one `sqlitePages` lane, the database, whose frames are the +//! pages each commit wrote. +//! +//! Under `do///`, each file gzipped: +//! +//! - `log//`: a batch, the frames of a run of commits up to +//! frame `end`, each run of page frames merged to one. +//! - `snapshot//`: a snapshot, the log continuing from +//! `frame`. +//! +//! Numbers are length-prefixed hex, sessions and snapshot frames +//! descending, so a listing comes newest first. A session's state is its +//! newest snapshot and the batches after it; a frame a batch before +//! already applied is skipped by its sequence. + +use std::collections::BTreeMap; +use std::io::BufReader; +use std::io::Read; +use std::io::Seek; +use std::io::SeekFrom; +use std::io::Write; +use std::path::Path; +use std::path::PathBuf; + +use frames_capnp::SnapshotJustification; +use frames_capnp::frame; +use frames_capnp::snapshot_header; +use futures::StreamExt; +use futures::TryStreamExt; +use futures::stream; + +use crate::Bucket; +use crate::Error; +use crate::Result; +use crate::Sequence; + +/// The database's lane. +const LANE: &str = "db"; +/// Downloads in flight during a restore, to hide the store's round trips. +const RESTORE_CONCURRENCY: usize = 16; + +/// Where a session's next frame goes. +#[derive(Clone, Copy, Debug, Default, PartialEq, Eq)] +pub struct Cursor { + pub session: u32, + pub sequence: Sequence, + /// Whether the database lane exists. + pub lane: bool, +} + +/// Pages by number. +pub type Pages<'a> = BTreeMap; + +#[derive(Clone, Copy)] +pub enum Body<'a> { + NewLane, + DeleteLane, + Initialize(&'a [u8]), + /// A commit's pages, and the database's size after, in pages. + Pages(u32, &'a Pages<'a>), +} + +/// Appends a frame to `out` at `cursor`, and moves it past. +pub fn push(out: &mut Vec, cursor: &mut Cursor, body: Body) { + match body { + Body::NewLane => cursor.lane = true, + Body::DeleteLane => cursor.lane = false, + _ => {} + } + let (session, sequence) = (cursor.session, cursor.sequence); + frame_at(out, session, sequence..=sequence, body); + cursor.sequence += 1; +} + +/// Appends a frame covering `sequences` to `out`. +fn frame_at( + out: &mut Vec, + session: u32, + sequences: std::ops::RangeInclusive, + body: Body, +) { + let mut message = capnp::message::Builder::new_default(); + let mut f = message.init_root::(); + f.set_session(session); + f.set_sequence(*sequences.end()); + match body { + Body::NewLane => { + let mut lane = f.init_new_lane(); + lane.set_name(LANE); + lane.set_sqlite_pages(()); + } + Body::DeleteLane => f.set_delete_lane(()), + Body::Initialize(image) => f.set_initialize(image), + Body::Pages(commit, pages) => { + let mut group = f.init_pages(); + group.set_first(*sequences.start()); + group.set_commit(commit); + let mut list = group.init_list(pages.len() as u32); + for (i, (&number, data)) in pages.iter().enumerate() { + let mut page = list.reborrow().get(i as u32); + page.set_number(number); + page.set_data(data); + } + } + } + let _ = capnp::serialize::write_message(&mut *out, &message); +} + +/// `batch`, a run of frames, with each run of page frames merged to one +/// with the newest version of each page. +pub fn merge(batch: &[u8]) -> Result> { + let mut messages = Vec::new(); + let mut r = batch; + while let Some(message) = capnp::serialize::try_read_message(&mut r, options())? { + messages.push(message); + } + let mut out = Vec::new(); + // The run's session, first and last sequence, size, and pages. + let mut run: Option<(u32, Sequence, Sequence, u32, Pages)> = None; + let flush = |out: &mut Vec, run: &mut Option<(u32, Sequence, Sequence, u32, Pages)>| { + if let Some((session, first, last, commit, mut pages)) = run.take() { + pages.retain(|&number, _| number <= commit); + frame_at(out, session, first..=last, Body::Pages(commit, &pages)); + } + }; + for message in &messages { + let f = message.get_root::()?; + let (session, sequence) = (f.get_session(), f.get_sequence()); + if let frame::Pages(group) = f.which()? { + let first = group.get_first(); + let r = run.get_or_insert_with(|| (session, first, sequence, 0, BTreeMap::new())); + (r.2, r.3) = (sequence, group.get_commit()); + for page in group.get_list()? { + r.4.insert(page.get_number(), page.get_data()?); + } + } else { + flush(&mut out, &mut run); + let mut copy = capnp::message::Builder::new_default(); + copy.set_root(f)?; + capnp::serialize::write_message(&mut out, ©)?; + } + } + flush(&mut out, &mut run); + Ok(out) +} + +/// Gzips `bytes`, as every file in the bucket is. +pub fn compress(bytes: &[u8]) -> Result> { + let mut encoder = encoder(Vec::new()); + encoder.write_all(bytes)?; + Ok(encoder.finish()?) +} + +/// Gzip ends with the content's CRC-32 and length, which the decoder +/// checks: a corrupt or truncated copy fails to replay rather than +/// replaying wrong or short. +fn encoder(w: W) -> flate2::write::GzEncoder { + flate2::write::GzEncoder::new(w, flate2::Compression::fast()) +} + +/// Bounds a message, so that a corrupt one fails rather than allocating +/// whatever its segment table claims: 1 GiB, far more than a batch holds. +fn options() -> capnp::message::ReaderOptions { + *capnp::message::ReaderOptions::new().traversal_limit_in_words(Some(1 << 27)) +} + +/// The database lane, replayed into a file. +pub struct Replay { + db: std::fs::File, + live: bool, + /// The next frame, by session and sequence. + next: Option<(u32, Sequence)>, + /// The furthest end of a batch skipped for starting past `next`, which + /// a later batch must reach. + skipped: Option<(u32, Sequence)>, + log_bytes: u64, +} + +impl Replay { + pub fn new(db: &Path) -> Result { + Ok(Self { + db: std::fs::File::options() + .read(true) + .write(true) + .create(true) + .truncate(true) + .open(db)?, + live: false, + next: None, + skipped: None, + log_bytes: 0, + }) + } + + /// Applies `f`, the local copy at `path`. A batch that does not reach + /// back to the next frame is skipped: a later, wider one must cover it. + pub fn apply(&mut self, f: &File, path: &Path) -> Result<()> { + let file = std::fs::File::open(path)?; + let size = file.metadata()?.len(); + let mut r = BufReader::new(flate2::read::GzDecoder::new(file)); + if f.snapshot { + return self.snapshot(&mut r); + } + let end = (f.session, f.frame); + while let Some(message) = capnp::serialize::try_read_message(&mut r, options())? { + let f = message.get_root::()?; + let (session, last) = (f.get_session(), f.get_sequence()); + let first = match f.which()? { + frame::Pages(group) => group.get_first(), + _ => last, + }; + let next = self.next.unwrap_or((session, 0)); + if (session, last) < next { + continue; + } + if (session, first) > next { + self.skipped = self.skipped.max(Some(end)); + return Ok(()); + } + // Pages a frame before already applied are rewritten with the + // same or newer versions. + self.next = Some((session, last + 1)); + self.frame(f)?; + } + self.log_bytes += size; + Ok(()) + } + + fn snapshot(&mut self, r: &mut impl Read) -> Result<()> { + let message = capnp::serialize::read_message(&mut *r, options())?; + let header = message.get_root::()?; + self.next = Some((header.get_session(), header.get_frame())); + (self.live, self.log_bytes) = (false, 0); + self.db.set_len(0)?; + self.db.seek(SeekFrom::Start(0))?; + let lanes = header.get_lanes()?; + if lanes.len() > 1 { + return Err(Error::Replica("a snapshot of more than one lane".into())); + } + for lane in lanes { + if let snapshot_header::lane::Sqlite(sqlite) = lane.which()? { + let size = sqlite.get_database_size(); + if std::io::copy(&mut r.take(size), &mut self.db)? != size { + return Err(Error::Replica( + "a snapshot shorter than its database".into(), + )); + } + self.live = true; + } + } + // Read to the end, where the decoder checks the checksum. + if std::io::copy(r, &mut std::io::sink())? != 0 { + return Err(Error::Replica("a snapshot longer than its lanes".into())); + } + Ok(()) + } + + fn frame(&mut self, f: frame::Reader) -> Result<()> { + if f.get_lane() != 0 { + return Err(Error::Replica(format!("lane {}", f.get_lane()))); + } + match f.which()? { + frame::NewLane(_) | frame::DeleteLane(()) => { + self.db.set_len(0)?; + self.live = matches!(f.which()?, frame::NewLane(_)); + } + frame::Initialize(image) => { + self.db.set_len(0)?; + self.db.seek(SeekFrom::Start(0))?; + self.db.write_all(image?)?; + } + frame::Pages(group) => { + let mut page_size = 0; + for page in group.get_list()? { + let data = page.get_data()?; + page_size = data.len() as u64; + let at = u64::from(page.get_number() - 1) * page_size; + self.db.seek(SeekFrom::Start(at))?; + self.db.write_all(data)?; + } + if page_size > 0 { + self.db.set_len(u64::from(group.get_commit()) * page_size)?; + } + } + _ => return Err(Error::Replica("an unsupported frame".into())), + } + Ok(()) + } + + /// Fails if a skipped batch's frames were never replayed: a batch + /// between is missing. + fn check_gaps(&self) -> Result<()> { + match self.skipped { + Some(end) if self.next.is_none_or(|next| next < end) => Err(Error::Replica(format!( + "the log is missing frames before {}", + end.1 + ))), + _ => Ok(()), + } + } + + /// The replayed database, whole, and the frame the log continues from. + pub fn finish(self) -> Result { + self.check_gaps()?; + self.db.sync_data()?; + Ok(self.next.map_or(0, |n| n.1)) + } + + /// Writes the replayed state as a snapshot into `dir`. + pub fn write_snapshot(mut self, dir: &Path) -> Result { + self.check_gaps()?; + let (session, frame) = self.next.unwrap_or_default(); + let size = self.db.seek(SeekFrom::End(0))?; + self.db.seek(SeekFrom::Start(0))?; + let header = Header { + session, + frame, + justification: SnapshotJustification::ClientRequested, + log_bytes: self.log_bytes, + }; + write_snapshot(dir, &header, self.live.then_some(size), |w| { + Ok(std::io::copy(&mut self.db, w).map(drop)?) + }) + } +} + +struct Header { + session: u32, + frame: Sequence, + justification: SnapshotJustification, + log_bytes: u64, +} + +/// Writes a snapshot into `dir`: the database lane, if it exists, of `size` +/// bytes that `payload` writes. +fn write_snapshot( + dir: &Path, + h: &Header, + size: Option, + payload: impl FnOnce(&mut dyn Write) -> Result<()>, +) -> Result { + let mut message = capnp::message::Builder::new_default(); + let mut header = message.init_root::(); + header.set_session(h.session); + header.set_frame(h.frame); + header.set_justification(h.justification); + header.set_log_bytes_since_last_snapshot(h.log_bytes); + let mut lanes = header.init_lanes(u32::from(size.is_some())); + if let Some(size) = size { + let mut lane = lanes.reborrow().get(0); + lane.set_name(LANE); + lane.init_sqlite().set_database_size(size); + } + let file = File { + snapshot: true, + session: h.session, + frame: h.frame, + size: 0, + }; + let out = dir.join(file.name()); + let tmp = with_suffix(&out, ".tmp"); + let mut encoder = encoder(std::io::BufWriter::new(std::fs::File::create(&tmp)?)); + capnp::serialize::write_message(&mut encoder, &message)?; + payload(&mut encoder)?; + let mut w = encoder.finish()?; + w.flush()?; + drop(w); + std::fs::rename(&tmp, &out)?; + let size = std::fs::metadata(&out)?.len(); + Ok(File { size, ..file }) +} + +/// Writes the database at `path`, which no connection has open, as the +/// snapshot `session` starts from into `dir`, or `None` for an empty +/// database. Closing the last connection checkpointed the WAL into the +/// file. +pub fn snapshot_file(path: &Path, dir: &Path, session: u32) -> Result> { + let mut db = match std::fs::File::open(path) { + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(None), + db => db?, + }; + let size = db.metadata()?.len(); + if size < 100 { + return Ok(None); + } + let header = Header { + session, + frame: 0, + justification: SnapshotJustification::ColdStartMitigation, + log_bytes: 0, + }; + let file = write_snapshot(dir, &header, Some(size), |w| { + Ok(std::io::copy(&mut db, w).map(drop)?) + })?; + Ok(Some(file)) +} + +/// Replays `files`, a plan with local copies, into a snapshot in `dir`. +pub fn compact(files: &[(File, PathBuf)], dir: &Path) -> Result { + let db = dir.join("compact.db"); + let mut replay = Replay::new(&db)?; + for (f, path) in files { + replay.apply(f, path)?; + } + let snapshot = replay.write_snapshot(dir); + let _ = std::fs::remove_file(&db); + snapshot +} + +/// One file of a session's log: a batch up to frame `frame`, or a snapshot +/// the log continues from at `frame`. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct File { + pub snapshot: bool, + pub session: u32, + pub frame: Sequence, + pub size: u64, +} + +impl File { + /// This node's copy's name. + pub fn name(&self) -> String { + let kind = if self.snapshot { "snapshot" } else { "log" }; + format!("{kind}-{:08x}-{:08x}", self.session, self.frame) + } + + fn key(&self) -> String { + let session = descending(self.session.into()); + if self.snapshot { + format!("snapshot/{session}/{}", descending(self.frame.into())) + } else { + format!("log/{session}/{}", ascending(self.frame.into())) + } + } +} + +/// Hex, prefixed with its length so names sort as numbers: `a0`, `a1`, .., +/// `bff`. +fn ascending(v: u64) -> String { + let hex = format!("{v:x}"); + format!("{}{hex}", char::from(b'a' + hex.len() as u8 - 1)) +} + +/// Likewise, sorting in reverse: `zf`, `ze`, .., `y00`. +fn descending(v: u64) -> String { + let hex = format!("{v:x}"); + let digits: String = hex.bytes().map(|d| char::from(flip(d))).collect(); + format!("{}{digits}", char::from(b'z' + 1 - hex.len() as u8)) +} + +/// A hex digit's complement, `0` for `f`. +fn flip(digit: u8) -> u8 { + let value = if digit > b'9' { + digit - b'a' + 10 + } else { + digit - b'0' + }; + b"fedcba9876543210"[usize::from(value)] +} + +fn parse(name: &str, ascending: bool) -> Option { + let (&prefix, digits) = name.as_bytes().split_first()?; + let len = if ascending { + prefix.checked_sub(b'a')? + 1 + } else { + (b'z' + 1).checked_sub(prefix)? + }; + let valid = |d: &u8| d.is_ascii_digit() || (b'a'..=b'f').contains(d); + if digits.len() != usize::from(len) || !digits.iter().all(valid) { + return None; + } + let digits: String = digits + .iter() + .map(|&d| char::from(if ascending { d } else { flip(d) })) + .collect(); + u64::from_str_radix(&digits, 16).ok() +} + +/// What a restore of `files`, a session's, reads, in order: the newest +/// snapshot, then each batch past it. +pub fn plan(files: &[File]) -> Vec { + let snapshot = files.iter().filter(|f| f.snapshot).max_by_key(|f| f.frame); + let from = snapshot.map_or(0, |f| f.frame); + let mut batches: Vec<_> = files + .iter() + .filter(|f| !f.snapshot && f.frame > from) + .copied() + .collect(); + batches.sort_by_key(|f| f.frame); + snapshot.into_iter().copied().chain(batches).collect() +} + +/// An object's log in the bucket. +#[derive(Clone)] +pub struct Log { + bucket: Bucket, +} + +impl Log { + pub fn new(bucket: Bucket) -> Self { + Self { bucket } + } + + /// The files of `session`. + pub async fn files(&self, session: u32) -> Result> { + self.list(&format!("/{}/", descending(session.into()))) + .await + } + + /// The files of every session before `session`. + pub async fn older(&self, session: u32) -> Result> { + let files = self.list("/").await?; + Ok(files.into_iter().filter(|f| f.session < session).collect()) + } + + /// The files under ``, for both kinds. + async fn list(&self, under: &str) -> Result> { + let mut files = Vec::new(); + for snapshot in [false, true] { + let kind = if snapshot { "snapshot" } else { "log" }; + for entry in self.bucket.list(&format!("{kind}{under}")).await? { + let mut parts = entry.key.rsplit('/'); + let number = |p: Option<&str>, asc| { + p.and_then(|p| parse(p, asc)) + .and_then(|n| u32::try_from(n).ok()) + }; + let frame = number(parts.next(), !snapshot); + let session = number(parts.next(), false); + if let (Some(frame), Some(session)) = (frame, session) { + files.push(File { + snapshot, + session, + frame, + size: 0, + }); + } + } + } + Ok(files) + } + + pub async fn get(&self, f: &File, to: &Path) -> Result<()> { + if self.bucket.get_file(&f.key(), to).await? { + Ok(()) + } else { + Err(Error::Replica(format!("{} is missing", f.key()))) + } + } + + pub async fn put(&self, f: &File, from: &Path) -> Result<()> { + self.bucket.put_file(&f.key(), from, &[]).await + } + + pub async fn delete(&self, f: &File) -> Result<()> { + self.bucket.delete(&f.key()).await + } +} + +/// Restores `session` of the log into `output`, which must not exist: the +/// plan's files spooled to `.restore/`, then replayed on the +/// blocking pool. The frame the log continues from, 0 for an empty log, +/// which leaves no file. +pub async fn restore(log: &Log, session: u32, output: &Path) -> Result { + let plan = plan(&log.files(session).await?); + if plan.is_empty() { + return Ok(0); + } + let spool = with_suffix(output, ".restore"); + std::fs::create_dir_all(&spool)?; + let files: Vec<_> = (0..plan.len()) + .map(|i| (plan[i], spool.join(i.to_string()))) + .collect(); + let downloaded = stream::iter(&files) + .map(|(f, path)| log.get(f, path)) + .buffer_unordered(RESTORE_CONCURRENCY) + .try_collect::<()>() + .await; + let output = output.to_owned(); + let result = match downloaded { + Ok(()) => { + crate::owner::blocking(move || { + let mut replay = Replay::new(&output)?; + for (f, path) in &files { + replay.apply(f, path)?; + } + replay.finish() + }) + .await + } + Err(e) => Err(e), + }; + let _ = std::fs::remove_dir_all(&spool); + result +} + +pub fn with_suffix(path: &Path, suffix: &str) -> PathBuf { + let mut path = path.as_os_str().to_owned(); + path.push(suffix); + path.into() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn names_sort_as_their_numbers() { + let values = [0, 1, 0xf, 0x10, 0xff, 0x100, 0xffff_ffff]; + let up: Vec<_> = values.iter().map(|&v| ascending(v)).collect(); + let down: Vec<_> = values.iter().map(|&v| descending(v)).collect(); + assert_eq!(up[..3], ["a0", "a1", "af"]); + assert_eq!(down[..3], ["zf", "ze", "z0"]); + assert!(up.is_sorted()); + assert!(down.iter().rev().is_sorted()); + for (v, (up, down)) in values.iter().zip(up.iter().zip(&down)) { + assert_eq!(parse(up, true), Some(*v)); + assert_eq!(parse(down, false), Some(*v)); + } + assert_eq!(parse("b1", true), None); + } + + fn file(snapshot: bool, frame: Sequence) -> File { + File { + snapshot, + session: 1, + frame, + size: 0, + } + } + + #[test] + fn a_plan_is_the_newest_snapshot_and_the_batches_past_it() { + let (old, new) = (file(true, 0), file(true, 4)); + let files = [file(false, 9), file(false, 4), new, old, file(false, 6)]; + assert_eq!(plan(&files), [new, file(false, 6), file(false, 9)]); + assert_eq!(plan(&files[..2]), [file(false, 4), file(false, 9)]); + assert_eq!(plan(&[]), []); + } +} diff --git a/src/workerd/server/cluster/log_tests.rs b/src/workerd/server/cluster/log_tests.rs new file mode 100644 index 00000000000..fd5a7d5846a --- /dev/null +++ b/src/workerd/server/cluster/log_tests.rs @@ -0,0 +1,675 @@ +// Copyright (c) 2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +//! The capture against real `SQLite`: a test VFS wraps the default one and +//! reports WAL writes to a `Capture`, as workerd's VFS does, and whatever +//! the capture cut must restore to what the application sees. + +use std::cell::RefCell; +use std::ffi::CString; +use std::ffi::c_char; +use std::ffi::c_int; +use std::ffi::c_void; +use std::path::Path; +use std::path::PathBuf; +use std::rc::Rc; +use std::sync::OnceLock; + +use crate::Bucket; +use crate::Error; +use crate::Sequence; +use crate::capture::Capture; +use crate::log::File as LogFile; +use crate::log::Log; +use crate::log::restore; +use crate::owner::Owners; +use crate::ownership::Claim; +use crate::ownership::Ownership; +use crate::registry::Registry; + +/// `sqlite3_vfs`, with the members the wrapper replaces typed. +#[repr(C)] +#[derive(Clone, Copy)] +struct Vfs { + version: c_int, + file_size: c_int, + max_path: c_int, + next: *mut Self, + name: *const c_char, + app_data: *mut c_void, + open: unsafe extern "C" fn(*mut Self, *const c_char, *mut File, c_int, *mut c_int) -> c_int, + rest: [*const c_void; 15], +} + +/// `sqlite3_io_methods`, likewise. +#[repr(C)] +#[derive(Clone, Copy)] +struct Methods { + version: c_int, + close: *const c_void, + read: *const c_void, + write: unsafe extern "C" fn(*mut File, *const c_void, c_int, i64) -> c_int, + rest: [*const c_void; 15], +} + +/// A file's own method table, followed by the method it replaces and +/// whether the file is a WAL. It sits past the wrapped file's state. +#[repr(C)] +struct Patched { + methods: Methods, + wrapped_write: unsafe extern "C" fn(*mut File, *const c_void, c_int, i64) -> c_int, + is_wal: bool, +} + +#[repr(C)] +struct File { + methods: *const Methods, +} + +unsafe extern "C" { + fn sqlite3_vfs_find(name: *const c_char) -> *mut Vfs; + fn sqlite3_vfs_register(vfs: *mut Vfs, default: c_int) -> c_int; + fn sqlite3_open_v2( + path: *const c_char, + db: *mut *mut c_void, + flags: c_int, + vfs: *const c_char, + ) -> c_int; + fn sqlite3_exec( + db: *mut c_void, + sql: *const c_char, + cb: *const c_void, + arg: *mut c_void, + err: *mut *mut c_char, + ) -> c_int; + fn sqlite3_prepare_v2( + db: *mut c_void, + sql: *const c_char, + n: c_int, + stmt: *mut *mut c_void, + tail: *mut *const c_char, + ) -> c_int; + fn sqlite3_step(stmt: *mut c_void) -> c_int; + fn sqlite3_column_int64(stmt: *mut c_void, col: c_int) -> i64; + fn sqlite3_finalize(stmt: *mut c_void) -> c_int; + fn sqlite3_close(db: *mut c_void) -> c_int; +} + +/// Just enough of `SQLite` for the tests: run SQL, read one row of integers. +struct Connection(*mut c_void); + +impl Connection { + fn open(path: &str) -> Self { + let path = CString::new(path).unwrap(); + let mut db = std::ptr::null_mut(); + // OPEN_READWRITE | OPEN_CREATE | OPEN_URI. + // SAFETY: the arguments are valid for the call. + let rc = unsafe { sqlite3_open_v2(path.as_ptr(), &raw mut db, 0x46, std::ptr::null()) }; + assert_eq!(rc, 0); + Self(db) + } + + fn execute(&self, sql: &str) { + let sql = CString::new(sql).unwrap(); + // SAFETY: `self.0` is an open connection. + let rc = unsafe { + sqlite3_exec( + self.0, + sql.as_ptr(), + std::ptr::null(), + std::ptr::null_mut(), + std::ptr::null_mut(), + ) + }; + assert_eq!(rc, 0, "{sql:?}"); + } + + fn row(&self, sql: &str, columns: c_int) -> Vec { + let sql = CString::new(sql).unwrap(); + let mut stmt = std::ptr::null_mut(); + // SAFETY: `self.0` is an open connection and `stmt` is finalized. + unsafe { + assert_eq!( + sqlite3_prepare_v2( + self.0, + sql.as_ptr(), + -1, + &raw mut stmt, + std::ptr::null_mut() + ), + 0 + ); + assert_eq!(sqlite3_step(stmt), 100); + let row = (0..columns) + .map(|c| sqlite3_column_int64(stmt, c)) + .collect(); + sqlite3_finalize(stmt); + row + } + } +} + +impl Drop for Connection { + fn drop(&mut self) { + // SAFETY: `self.0` is an open connection, closed once. + unsafe { sqlite3_close(self.0) }; + } +} + +const SQLITE_OPEN_WAL: c_int = 0x0008_0000; + +thread_local! { + static CAPTURE: RefCell> = const { RefCell::new(None) }; +} + +/// The wrapped VFS, and where a file's `Patched` sits. +struct Wrapped { + vfs: usize, + patched_at: usize, +} + +fn wrapped() -> &'static Wrapped { + static WRAPPED: OnceLock = OnceLock::new(); + // SAFETY: the default VFS lives for the process, and the copy + // registered here is leaked, as `SQLite` requires. + WRAPPED.get_or_init(|| unsafe { + let native = sqlite3_vfs_find(std::ptr::null()); + let patched_at = usize::try_from((*native).file_size) + .unwrap() + .next_multiple_of(8); + let mut vfs = *native; + vfs.name = c"capture-test".as_ptr(); + vfs.file_size = c_int::try_from(patched_at + size_of::()).unwrap(); + vfs.open = open; + assert_eq!(sqlite3_vfs_register(Box::leak(Box::new(vfs)), 0), 0); + Wrapped { + vfs: native as usize, + patched_at, + } + }) +} + +unsafe extern "C" fn open( + _: *mut Vfs, + name: *const c_char, + file: *mut File, + flags: c_int, + out: *mut c_int, +) -> c_int { + // SAFETY: `SQLite` passes a file of the wrapper's size, so the + // patched table lives as long as the file. + unsafe { + let native = wrapped().vfs as *mut Vfs; + let rc = ((*native).open)(native, name, file, flags, out); + if rc == 0 && !(*file).methods.is_null() { + let methods = *(*file).methods; + let patched: *mut Patched = file.cast::().add(wrapped().patched_at).cast(); + patched.write(Patched { + methods: Methods { write, ..methods }, + wrapped_write: methods.write, + is_wal: flags & SQLITE_OPEN_WAL != 0, + }); + (*file).methods = patched.cast_const().cast(); + } + rc + } +} + +unsafe extern "C" fn write(file: *mut File, data: *const c_void, n: c_int, offset: i64) -> c_int { + // SAFETY: `data` holds `n` bytes, and the file was opened by `open`. + unsafe { + let patched = &*(*file).methods.cast::(); + let rc = (patched.wrapped_write)(file, data, n, offset); + if rc == 0 && patched.is_wal { + let data = std::slice::from_raw_parts(data.cast::(), usize::try_from(n).unwrap()); + let offset = u64::try_from(offset).unwrap(); + CAPTURE.with_borrow_mut(|c| c.as_mut().unwrap().write(offset, data)); + } + rc + } +} + +fn scratch(name: &str) -> PathBuf { + let root = std::env::var_os("TEST_TMPDIR").map_or_else(std::env::temp_dir, PathBuf::from); + let dir = root.join(format!("capture-{name}-{}", std::process::id())); + let _ = std::fs::remove_dir_all(&dir); + std::fs::create_dir_all(&dir).unwrap(); + dir +} + +/// The application's connection, through the capturing VFS, with the +/// capture already watching. +fn open_app(path: &Path) -> Connection { + wrapped(); + CAPTURE.set(Some(Capture::open(path).unwrap())); + let app = Connection::open(&format!("file:{}?vfs=capture-test", path.display())); + app.execute("PRAGMA journal_mode=WAL; CREATE TABLE t (id INTEGER PRIMARY KEY, v TEXT)"); + app +} + +fn insert(app: &Connection, from: i64, to: i64) { + let mut sql = String::from("BEGIN;"); + for id in from..to { + sql += &format!( + "INSERT INTO t (id, v) VALUES ({id}, '{}');", + "x".repeat(200) + ); + } + sql += "COMMIT;"; + app.execute(&sql); +} + +/// The table as one comparable value: row count, id sum, and value bytes. +fn rows(conn: &Connection) -> (i64, i64, i64) { + let r = conn.row( + "SELECT count(*), coalesce(sum(id), 0), coalesce(sum(length(v)), 0) FROM t", + 3, + ); + (r[0], r[1], r[2]) +} + +fn runtime() -> tokio::runtime::Runtime { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap() +} + +/// The frame after the capture's last commit. +fn last() -> Sequence { + CAPTURE.with_borrow(|c| c.as_ref().unwrap().last().unwrap()) +} + +/// The captured frames `from..to`, merged. +fn merged(from: Sequence, to: Sequence) -> Vec { + let bytes = CAPTURE.with_borrow(|c| c.as_ref().unwrap().batch(from, to).unwrap()); + crate::log::merge(&bytes).unwrap() +} + +/// The captured frames `from..to` as one batch in `dir`. +fn batch(dir: &Path, from: Sequence, to: Sequence) -> (LogFile, PathBuf) { + let bytes = merged(from, to); + let path = dir.join(format!("{from}-{to}")); + std::fs::write(&path, crate::log::compress(&bytes).unwrap()).unwrap(); + let file = LogFile { + snapshot: false, + session: 0, + frame: to, + size: 0, + }; + (file, path) +} + +fn log(store: &Bucket) -> Log { + Log::new(store.at("cell")) +} + +/// Upload the captured frames `from..to` as one batch, as a commit does. +fn upload(store: &Bucket, from: Sequence, to: Sequence) { + let dir = scratch(&format!("upload-{from}-{to}")); + let (file, path) = batch(&dir, from, to); + runtime().block_on(log(store).put(&file, &path)).unwrap(); +} + +/// Upload a snapshot at `to` of the captured frames, as compaction does. +fn upload_snapshot(store: &Bucket, to: Sequence) { + let dir = scratch(&format!("snapshot-{to}")); + let files = [batch(&dir, 0, to)]; + let snapshot = crate::log::compact(&files, &dir).unwrap(); + let path = dir.join(snapshot.name()); + runtime() + .block_on(log(store).put(&snapshot, &path)) + .unwrap(); +} + +fn restored_rows(store: &Bucket, out: &Path) -> (i64, i64, i64) { + runtime().block_on(restore(&log(store), 0, out)).unwrap(); + rows(&Connection::open(out.to_str().unwrap())) +} + +#[test] +fn restore_reproduces_the_application_rows() { + let dir = scratch("roundtrip"); + let app = open_app(&dir.join("app.db")); + let store = Bucket::memory(""); + insert(&app, 0, 100); + let shipped = last(); + upload(&store, 0, shipped); + insert(&app, 100, 250); + app.execute("DELETE FROM t WHERE id % 7 = 0"); + upload(&store, shipped, last()); + let want = rows(&app); + assert_eq!(want.0, 250 - 36); + assert_eq!(restored_rows(&store, &dir.join("restored.db")), want); +} + +#[test] +fn restore_reads_the_newest_snapshot_and_what_extends_it() { + let dir = scratch("snapshot"); + let app = open_app(&dir.join("app.db")); + let store = Bucket::memory(""); + insert(&app, 0, 100); + let a = last(); + upload(&store, 0, a); + upload_snapshot(&store, a); + insert(&app, 100, 200); + let b = last(); + upload(&store, a, b); + upload_snapshot(&store, b); + app.execute("DELETE FROM t WHERE id % 3 = 0"); + upload(&store, b, last()); + let files = runtime().block_on(log(&store).files(0)).unwrap(); + let plan: Vec<_> = crate::log::plan(&files) + .iter() + .map(|f| (f.snapshot, f.frame)) + .collect(); + assert_eq!(plan, [(true, b), (false, last())]); + assert_eq!(restored_rows(&store, &dir.join("restored.db")), rows(&app)); +} + +#[test] +fn a_snapshot_inside_a_wal_carries_it_across_checkpoints() { + let dir = scratch("snapshot-wal"); + let app = open_app(&dir.join("app.db")); + let store = Bucket::memory(""); + app.execute("PRAGMA wal_autocheckpoint=8"); + for from in (0..400).step_by(10) { + insert(&app, from, from + 10); + } + let a = last(); + upload_snapshot(&store, a); + for from in (400..800).step_by(10) { + insert(&app, from, from + 10); + } + upload(&store, a, last()); + assert_eq!(restored_rows(&store, &dir.join("restored.db")), rows(&app)); +} + +#[test] +fn a_snapshot_compacts_from_the_last_one_inside_a_wal() { + // No checkpoint: the second snapshot checkpoints a WAL that continues + // past the first, from a frame, not a WAL header. + let dir = scratch("resnapshot"); + let app = open_app(&dir.join("app.db")); + let store = Bucket::memory(""); + insert(&app, 0, 100); + let a = last(); + let files = [batch(&dir, 0, a)]; + let first = crate::log::compact(&files, &dir).unwrap(); + insert(&app, 100, 200); + let b = last(); + let files = [(first, dir.join(first.name())), batch(&dir, a, b)]; + let second = crate::log::compact(&files, &dir).unwrap(); + assert_eq!(second.frame, b); + runtime() + .block_on(log(&store).put(&second, &dir.join(second.name()))) + .unwrap(); + insert(&app, 200, 300); + upload(&store, b, last()); + assert_eq!(restored_rows(&store, &dir.join("restored.db")), rows(&app)); +} + +#[test] +fn a_batch_keeps_one_copy_of_a_page_its_commits_rewrote() { + let dir = scratch("dedup"); + let app = open_app(&dir.join("app.db")); + let store = Bucket::memory(""); + insert(&app, 0, 10); + let a = last(); + for i in 0..100 { + app.execute(&format!("UPDATE t SET v = '{i}' WHERE id = 1")); + } + let raw = CAPTURE.with_borrow(|c| c.as_ref().unwrap().batch(a, last()).unwrap()); + let merged = merged(a, last()); + assert!( + merged.len() * 20 < raw.len(), + "{} of {}", + merged.len(), + raw.len() + ); + upload(&store, 0, a); + upload(&store, a, last()); + assert_eq!(restored_rows(&store, &dir.join("restored.db")), rows(&app)); +} + +#[test] +fn a_corrupt_file_fails_to_restore() { + let dir = scratch("corrupt"); + let app = open_app(&dir.join("app.db")); + let store = Bucket::memory(""); + insert(&app, 0, 100); + let (file, path) = batch(&dir, 0, last()); + let mut bytes = std::fs::read(&path).unwrap(); + let mid = bytes.len() / 2; + bytes[mid] ^= 1; + std::fs::write(&path, bytes).unwrap(); + runtime().block_on(log(&store).put(&file, &path)).unwrap(); + let restored = runtime().block_on(restore(&log(&store), 0, &dir.join("restored.db"))); + assert!(restored.is_err(), "{restored:?}"); +} + +#[test] +fn a_truncated_file_fails_to_restore() { + // Cut anywhere, including before the trailer that holds the checksum, + // or empty. + let dir = scratch("truncated"); + let app = open_app(&dir.join("app.db")); + insert(&app, 0, 100); + let (file, path) = batch(&dir, 0, last()); + let bytes = std::fs::read(&path).unwrap(); + let len = bytes.len(); + for cut in [&bytes[..len - 1], &bytes[..len - 8], &bytes[..len / 2], &[]] { + let store = Bucket::memory(""); + std::fs::write(&path, cut).unwrap(); + runtime().block_on(log(&store).put(&file, &path)).unwrap(); + let out = dir.join("restored.db"); + let _ = std::fs::remove_file(&out); + let restored = runtime().block_on(restore(&log(&store), 0, &out)); + assert!(restored.is_err(), "{restored:?}"); + } +} + +#[test] +fn a_rolled_back_transaction_leaves_nothing() { + let dir = scratch("rollback"); + let app = open_app(&dir.join("app.db")); + let store = Bucket::memory(""); + insert(&app, 0, 10); + let wal = || std::fs::metadata(dir.join("app.db-wal")).unwrap().len(); + let before = wal(); + // A small cache spills the doomed transaction's frames into the WAL. + app.execute("PRAGMA cache_size=2; BEGIN;"); + app.execute( + "WITH RECURSIVE s(x) AS (SELECT 1000 UNION ALL SELECT x + 1 FROM s WHERE x < 3000) \ + INSERT INTO t SELECT x, hex(randomblob(100)) FROM s", + ); + assert!(wal() > before + 100 * 4096, "no frames spilled"); + app.execute("ROLLBACK;"); + insert(&app, 10, 20); + upload(&store, 0, last()); + assert_eq!(rows(&app).0, 20); + assert_eq!(restored_rows(&store, &dir.join("restored.db")), rows(&app)); +} + +#[test] +fn checkpoints_and_a_restart_keep_the_log_whole() { + let dir = scratch("checkpoint"); + let app = open_app(&dir.join("app.db")); + let store = Bucket::memory(""); + // Frequent checkpoints: the WAL starts over many times. + app.execute("PRAGMA wal_autocheckpoint=8"); + for from in (0..600).step_by(50) { + insert(&app, from, from + 50); + } + app.execute("PRAGMA wal_checkpoint(TRUNCATE)"); + insert(&app, 600, 650); + upload(&store, 0, last()); + assert_eq!(restored_rows(&store, &dir.join("one.db")), rows(&app)); + + // A new log from the live database, then more commits, shrinking at + // the end. + app.execute("PRAGMA wal_checkpoint(TRUNCATE)"); + CAPTURE.with_borrow_mut(|c| c.as_mut().unwrap().restart().unwrap()); + insert(&app, 650, 700); + app.execute("DELETE FROM t WHERE id > 20; VACUUM;"); + insert(&app, 5000, 5010); + let store = Bucket::memory(""); + upload(&store, 0, last()); + assert_eq!(rows(&app).0, 31); + assert_eq!(restored_rows(&store, &dir.join("two.db")), rows(&app)); +} + +#[test] +fn a_restart_after_the_database_closed_reads_the_file() { + let dir = scratch("closed"); + let app = open_app(&dir.join("app.db")); + insert(&app, 0, 50); + let want = rows(&app); + // Closing the last connection checkpoints the WAL and deletes it, so + // the committed pages the capture knows of are in the database file. + drop(app); + assert!(!dir.join("app.db-wal").exists()); + CAPTURE.with_borrow_mut(|c| c.as_mut().unwrap().restart().unwrap()); + // Reopened, before its first write: an empty WAL. + let app = Connection::open(&format!( + "file:{}?vfs=capture-test", + dir.join("app.db").display() + )); + assert_eq!(rows(&app), want); + assert_eq!(std::fs::metadata(dir.join("app.db-wal")).unwrap().len(), 0); + CAPTURE.with_borrow_mut(|c| c.as_mut().unwrap().restart().unwrap()); + let store = Bucket::memory(""); + upload(&store, 0, last()); + assert_eq!(restored_rows(&store, &dir.join("restored.db")), want); +} + +#[test] +fn a_batch_the_log_already_covers_is_not_applied() { + // Tiering and a bucket-acknowledged commit upload concurrently, so a + // stale range can land beside a newer, wider one (lost writes on the + // fleet): the third batch is inside the second, and must not override + // it. + let dir = scratch("covered"); + let app = open_app(&dir.join("app.db")); + let store = Bucket::memory(""); + let mut ends = Vec::new(); + for from in (0..120).step_by(10) { + insert(&app, from, from + 10); + ends.push(last()); + } + upload(&store, 0, ends[0]); + upload(&store, ends[0], ends[11]); + upload(&store, ends[1], ends[7]); + assert_eq!(restored_rows(&store, &dir.join("restored.db")), rows(&app)); +} + +#[test] +fn a_missing_batch_fails_to_restore_or_compact() { + // The batch after a lost one starts past the frame the log reached, + // and no wider one covers it. + let dir = scratch("missing"); + let app = open_app(&dir.join("app.db")); + let store = Bucket::memory(""); + let mut ends = Vec::new(); + for from in (0..30).step_by(10) { + insert(&app, from, from + 10); + ends.push(last()); + } + upload(&store, 0, ends[0]); + upload(&store, ends[1], ends[2]); + let out = dir.join("restored.db"); + let restored = runtime().block_on(restore(&log(&store), 0, &out)); + assert!(restored.is_err(), "{restored:?}"); + // Nor does compaction snapshot it, then delete the batches. + let files = [batch(&dir, 0, ends[0]), batch(&dir, ends[1], ends[2])]; + let compacted = crate::log::compact(&files, &dir); + assert!(compacted.is_err(), "{compacted:?}"); +} + +#[test] +fn a_deleted_database_restores_empty_then_as_recreated() { + // A facet's delete() or a deleteAll() removes the file; the new + // database's first page then reaches the file, not the WAL. + let dir = scratch("reset"); + let path = dir.join("app.db"); + let app = open_app(&path); + insert(&app, 0, 50); + drop(app); + std::fs::remove_file(&path).unwrap(); + CAPTURE.with_borrow_mut(|c| c.as_mut().unwrap().reset()); + let deleted = last(); + let store = Bucket::memory(""); + upload(&store, 0, deleted); + let out = dir.join("empty.db"); + runtime().block_on(restore(&log(&store), 0, &out)).unwrap(); + assert_eq!(std::fs::metadata(&out).unwrap().len(), 0); + + let app = Connection::open(&format!("file:{}?vfs=capture-test", path.display())); + app.execute("PRAGMA journal_mode=WAL; CREATE TABLE t (id INTEGER PRIMARY KEY, v TEXT)"); + insert(&app, 100, 110); + upload(&store, deleted, last()); + assert_eq!(rows(&app).0, 10); + assert_eq!(restored_rows(&store, &dir.join("split.db")), rows(&app)); + // As one batch. + let store = Bucket::memory(""); + upload(&store, 0, last()); + assert_eq!(restored_rows(&store, &dir.join("merged.db")), rows(&app)); +} + +#[test] +fn a_superseded_owners_commit_is_not_acknowledged() { + // a owns x and commits while b takes x over: the upload lands in a's + // epoch, but own.json no longer names it, so a must not acknowledge. + let dir = scratch("superseded"); + let path = dir.join("x.db"); + let store = Bucket::memory("").at("cluster"); + let node = |name: &str| { + let registry = Rc::new(Registry::new(store.clone(), name, name, 10_000)); + let ownership = Ownership::new(store.clone(), registry.clone()); + (registry, Owners::new(store.clone()), ownership) + }; + let ((a, a_owners, a_own), (b, b_owners, b_own)) = (node("a"), node("b")); + let claim = + |own: &Ownership, now| match runtime().block_on(own.acquire_or_route("ns", "x", now)) { + Ok(Claim::Owned(owned)) => owned, + other => panic!("{other:?}"), + }; + runtime().block_on(a.renew(0)).unwrap(); + let owned = claim(&a_own, 0); + let rt = runtime(); + assert!( + rt.block_on(a_owners.open_fresh(&a_own, "ns", "x", &owned)) + .unwrap() + ); + rt.block_on(a_owners.take_over(&a_own, "ns", "x", &owned, &path)) + .unwrap(); + + let app = Connection::open(path.to_str().unwrap()); + app.execute("PRAGMA journal_mode=WAL; CREATE TABLE t (id INTEGER PRIMARY KEY, v TEXT)"); + insert(&app, 0, 10); + let wal = std::fs::read(dir.join("x.db-wal")).unwrap(); + a_owners.wal_write("ns", "x", 0, &wal[..32]); + a_owners.wal_write("ns", "x", 32, &wal[32..]); + + // a stops renewing; b takes x over at epoch 2. + rt.block_on(b.renew(9_000)).unwrap(); + let owned = claim(&b_own, 11_000); + assert_eq!(owned.epoch, 2); + rt.block_on(b_owners.take_over(&b_own, "ns", "x", &owned, &dir.join("b.db"))) + .unwrap(); + + let commit = rt.block_on(a_owners.commit(&a_own, "ns", "x")); + assert!( + matches!(&commit, Err(Error::Contended(m)) if m.contains("superseded")), + "{commit:?}" + ); + + // b's epoch is open, so no restore reads a's session: b deletes it, + // with the upload a made after b took over. + let x = format!("{}{}", store.prefix(), crate::object_prefix("ns", "x")); + let x = Log::new(store.at(&x)); + assert!(!rt.block_on(x.older(2)).unwrap().is_empty()); + rt.block_on(b_owners.tier()); + assert!(rt.block_on(x.older(2)).unwrap().is_empty()); +} diff --git a/src/workerd/server/cluster/logic/BUILD.bazel b/src/workerd/server/cluster/logic/BUILD.bazel new file mode 100644 index 00000000000..7bdc8bd8448 --- /dev/null +++ b/src/workerd/server/cluster/logic/BUILD.bazel @@ -0,0 +1,8 @@ +load("//:build/wd_rust_crate.bzl", "wd_rust_crate") + +# The bucket cluster's decision core. std only: nothing here may do I/O or +# read a clock, so it has no dependencies to do them with. +wd_rust_crate( + name = "cluster-logic", + visibility = ["//src/workerd/server:__subpackages__"], +) diff --git a/src/workerd/server/cluster/logic/lease.rs b/src/workerd/server/cluster/logic/lease.rs new file mode 100644 index 00000000000..660f920f99e --- /dev/null +++ b/src/workerd/server/cluster/logic/lease.rs @@ -0,0 +1,145 @@ +// Copyright (c) 2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +//! An instance's lease in the bucket: when to renew and when to stop +//! serving (fence). +//! +//! The executor runs one round every `INTERVAL_MS` (`RETRY_MS` after a +//! failed renewal): it asks `before_renew`, renews if told to and reports +//! the result to `renewed`. + +/// A node's lease. A peer whose record has not been renewed for a full +/// lease is dead, so the lease spans several rounds. +pub const LEASE_MS: u64 = 20_000; +/// Renewals every 3 s, each bounded at 4 s, a failed one retried every +/// second. +/// +/// A node rides out about 8 s of bucket trouble and fences while its +/// lease holds. R2 stalls writes ~5.5 s at every 5-minute mark; a 2 s +/// bound cancelled them and fenced every node. +pub const INTERVAL_MS: u64 = 3_000; +pub const RENEW_TIMEOUT_MS: u64 = 4_000; +pub const LIST_TIMEOUT_MS: u64 = 2_000; +pub const RETRY_MS: u64 = 1_000; +/// Peers judge this node's lease by their own clocks; one off from the +/// bucket's clock by more than this exits. +pub const MAX_SKEW_MS: i64 = 2_000; +/// Peers judge a lease by their own clocks, each within `MAX_SKEW_MS` of +/// the bucket's, so the owner stops serving this long before it expires. +const SAFETY_MS: u64 = 2 * MAX_SKEW_MS as u64; + +/// What the executor observed before a round's renewal. +pub struct Observed { + pub now_ms: u64, + /// This node's clock less the bucket's. + pub skew_ms: i64, +} + +#[derive(Debug, PartialEq, Eq)] +pub enum Step { + /// Renew now. + Renew, + /// Stop serving: the process exits before peers may take over. + Fence(String), + /// Try again after `RETRY_MS`; the lease still holds. + Retry(String), +} + +#[derive(Default)] +pub struct Lease { + /// The lease expires at this time; 0 before the first renewal. + until_ms: u64, + pub draining: bool, +} + +impl Lease { + pub fn before_renew(&self, o: &Observed) -> Step { + if o.skew_ms.abs() > MAX_SKEW_MS { + return Step::Fence(format!("clock off from the bucket's by {} ms", o.skew_ms)); + } + // A node stalled past its lease (a paused process, a blocked loop) + // may have been taken over: it must not renew and serve on. + if self.until_ms != 0 && !self.holds(o.now_ms + RENEW_TIMEOUT_MS) { + return Step::Fence("the lease lapsed before it could be renewed".into()); + } + Step::Renew + } + + /// The renewal's outcome: the new expiry, or why it failed. `Renew` + /// here means the round succeeded. + pub fn renewed(&mut self, now_ms: u64, result: Result) -> Step { + match result { + Ok(until_ms) => { + self.until_ms = until_ms; + Step::Renew + } + // Fence while the lease still holds, so the node is gone before + // a peer may take over. + Err(e) if !self.holds(now_ms + RENEW_TIMEOUT_MS) => Step::Fence(e), + Err(e) => Step::Retry(e), + } + } + + /// Whether the lease holds at `now_ms`, with peers' clocks allowed + /// for: an actor serves only while it does. + pub fn holds(&self, now_ms: u64) -> bool { + now_ms + SAFETY_MS < self.until_ms + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn seen(now_ms: u64) -> Observed { + Observed { now_ms, skew_ms: 0 } + } + + #[test] + fn renews_and_holds_until_the_safety_margin() { + let mut lease = Lease::default(); + assert_eq!(lease.before_renew(&seen(0)), Step::Renew); + assert_eq!(lease.renewed(0, Ok(LEASE_MS)), Step::Renew); + let last = LEASE_MS - SAFETY_MS; + assert!(lease.holds(last - 1) && !lease.holds(last)); + } + + #[test] + fn a_failed_renewal_retries_until_the_lease_nearly_lapses() { + let mut lease = Lease::default(); + lease.renewed(0, Ok(LEASE_MS)); + assert!(matches!( + lease.renewed(5_000, Err("x".into())), + Step::Retry(_) + )); + let last = LEASE_MS - SAFETY_MS - RENEW_TIMEOUT_MS; + assert!(matches!( + lease.renewed(last - 1, Err("x".into())), + Step::Retry(_) + )); + assert!(matches!( + lease.renewed(last, Err("x".into())), + Step::Fence(_) + )); + } + + #[test] + fn a_node_stalled_past_its_lease_fences_instead_of_renewing() { + let mut lease = Lease::default(); + lease.renewed(0, Ok(LEASE_MS)); + assert_eq!(lease.before_renew(&seen(INTERVAL_MS)), Step::Renew); + let step = lease.before_renew(&seen(LEASE_MS + 1)); + assert!(matches!(step, Step::Fence(_))); + } + + #[test] + fn skew_fences_at_once() { + let lease = Lease::default(); + let o = Observed { + skew_ms: MAX_SKEW_MS + 1, + ..seen(0) + }; + assert!(matches!(lease.before_renew(&o), Step::Fence(_))); + } +} diff --git a/src/workerd/server/cluster/logic/lib.rs b/src/workerd/server/cluster/logic/lib.rs new file mode 100644 index 00000000000..9ba432a6a55 --- /dev/null +++ b/src/workerd/server/cluster/logic/lib.rs @@ -0,0 +1,16 @@ +// Copyright (c) 2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +//! The bucket cluster's decisions, with no I/O: no async, clocks, +//! randomness, locks or dependencies beyond std. Time is an argument. The +//! executor (`cluster`) performs what these types decide. + +#![deny( + clippy::expect_used, + clippy::panic, + clippy::unreachable, + clippy::unwrap_used +)] + +pub mod lease; diff --git a/src/workerd/server/cluster/owner.rs b/src/workerd/server/cluster/owner.rs new file mode 100644 index 00000000000..c903eb24bb0 --- /dev/null +++ b/src/workerd/server/cluster/owner.rs @@ -0,0 +1,693 @@ +// Copyright (c) 2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +//! The owner's side of an object: taking it over, the commit path, and +//! keeping its log in the bucket compact. +//! +//! A commit is acknowledged once the bucket holds it and own.json still +//! names the epoch after the upload. Each epoch writes its own session of +//! the object's log, starting with a snapshot of the state it inherited; a +//! takeover restores the last opened epoch's session. + +use std::cell::Cell; +use std::cell::RefCell; +use std::collections::HashMap; +use std::path::Path; +use std::path::PathBuf; +use std::rc::Rc; + +use futures::StreamExt; +use futures::stream; + +use crate::Bucket; +use crate::Epoch; +use crate::Error; +use crate::Ownership; +use crate::Result; +use crate::Sequence; +use crate::capture::Capture; +use crate::log::Cursor; +use crate::log::File; +use crate::log::Log; +use crate::log::plan; +use crate::ownership::Owned; + +/// Objects compacting at once, and deletes in flight for one compaction. +const TIER_CONCURRENCY: usize = 64; +/// Evictions a node uploads at once. +const EVICT_CONCURRENCY: usize = 32; +/// Evicted objects whose database file this node remembers as current. +const DORMANT_MAX: usize = 1 << 16; + +/// An owned object, its capture, and how far its commits are acknowledged. +struct Slot { + object: tokio::sync::Mutex, + /// Outside the lock: `SQLite` reports WAL writes synchronously, and may + /// while an upload holds the lock. + capture: RefCell, + /// Up to where commits are in the bucket, by epoch, then frame. + acked: tokio::sync::watch::Sender<(Epoch, Sequence)>, + /// Whether a commit is uploading on behalf of the others. + pushing: Cell, +} + +impl Slot { + fn opened(object: Object, capture: Capture) -> Rc { + let acked = (object.epoch, object.tiered); + Rc::new(Self { + object: tokio::sync::Mutex::new(object), + capture: RefCell::new(capture), + acked: tokio::sync::watch::Sender::new(acked), + pushing: Cell::new(false), + }) + } + + /// The bucket holds everything up to `through`. + fn tiered(&self, object: &mut Object, through: Sequence, file: Option) { + object.tiered = object.tiered.max(through); + object.files.extend(file); + self.capture.borrow_mut().forget(through); + } + + fn ack(&self, epoch: Epoch, through: Sequence) { + self.acked.send_if_modified(|acked| { + let more = (epoch, through) > *acked; + *acked = (*acked).max((epoch, through)); + more + }); + } +} + +/// The uploading commit's turn. Ending it, even by cancellation, wakes the +/// waiting commits so one of them takes the next. +struct Pushing<'a>(&'a Slot); + +impl Drop for Pushing<'_> { + fn drop(&mut self) { + self.0.pushing.set(false); + self.0.acked.send_modify(|_| {}); + } +} + +/// An object this node owns. +#[derive(Clone)] +struct Object { + ns: String, + actor: String, + epoch: Epoch, + replica: Replica, + /// How far the epoch's session in the bucket is contiguous. + tiered: Sequence, + /// Where the capture was at eviction. + cursor: Cursor, + /// The files of this epoch's session. Only this owner writes it, so + /// compaction deletes from here rather than listing it. + files: Vec, + /// Whether earlier sessions may remain in the bucket, which no restore + /// reads once this epoch opened. + prune: bool, +} + +pub struct Owners { + bucket: Bucket, + objects: RefCell>>, + /// New objects' first epochs, opened at the claim, until their databases + /// open. + opened: RefCell>, + /// Objects workerd closed, until their eviction. + retiring: RefCell>>, + /// Evictions in flight. Objects written together go idle together, and + /// a node's lease renewals must not wait behind all their uploads. + evictions: tokio::sync::Semaphore, + /// Objects evicted with the bucket holding their epoch's session: this + /// node's database file is that session's state. + dormant: RefCell>, +} + +fn object_key(ns: &str, actor: &str) -> String { + format!("{ns}/{actor}") +} + +impl Owners { + pub fn new(bucket: Bucket) -> Self { + Self { + bucket, + objects: RefCell::new(HashMap::new()), + opened: RefCell::new(HashMap::new()), + retiring: RefCell::new(HashMap::new()), + evictions: tokio::sync::Semaphore::new(EVICT_CONCURRENCY), + dormant: RefCell::new(HashMap::new()), + } + } + + fn client(&self, ns: &str, actor: &str) -> Log { + let path = format!( + "{}{}", + self.bucket.prefix(), + crate::object_prefix(ns, actor) + ); + Log::new(self.bucket.at(&path)) + } + + /// Take the object over at `owned.epoch` and open the epoch, leaving its + /// database at `path` ready for workerd to open. + pub async fn take_over( + &self, + ownership: &Ownership, + ns: &str, + actor: &str, + owned: &Owned, + path: &Path, + ) -> Result<()> { + let key = object_key(ns, actor); + if owned.recover.is_none() && !self.opened.borrow().contains_key(&key) { + if self.objects.borrow().contains_key(&key) { + return Ok(()); + } + // Woken where it was evicted, own.json unchanged: continue the + // epoch's session over the file, with no write to the bucket. A + // dormant entry forgotten since the claim fails this attempt; + // the retry recovers at the next epoch. + let dormant = self.dormant.borrow_mut().remove(&key); + let Some(object) = dormant.filter(|o| o.epoch == owned.epoch && path.exists()) else { + return Err(Error::Contended(format!( + "{key}: not open here, nor dormant" + ))); + }; + let capture = Capture::resume(path, object.cursor); + tracing::info!( + "cluster: {ns}/{actor} epoch {} resumes at {}", + object.epoch, + object.tiered + ); + self.objects + .borrow_mut() + .insert(key, Slot::opened(object, capture)); + return Ok(()); + } + // Woken here from the epoch this node evicted: its database file is + // what the bucket holds, so the restore is skipped. The entry stays + // until the epoch opens, so a concurrent claim here also finds the + // object dormant. + let dormant = self.dormant.borrow().get(&key).map(|o| o.epoch); + let recover = owned.recover.filter(|&from| from != 0); + let local = path.exists() && recover.is_some_and(|from| dormant == Some(from)); + if !local { + remove_database(path)?; + if let Some(recover) = recover { + // Nothing in the bucket: an object that committed nothing. + let log = self.client(ns, actor); + let end = crate::log::restore(&log, recover as u32, path).await?; + tracing::info!( + "cluster: {ns}/{actor} epoch {} recovers {recover}: bucket up to {end}", + owned.epoch + ); + } + } + // The state the epoch inherits is its session's snapshot at frame + // 0, written from the file on the blocking pool; a new object has + // none. + let cache = open_cache(path, owned.epoch)?; + let session = owned.epoch as u32; + let snapshot = if self.opened.borrow().contains_key(&key) { + None + } else { + let (path, cache) = (path.to_owned(), cache.clone()); + blocking(move || crate::log::snapshot_file(&path, &cache, session)).await? + }; + let cursor = Cursor { + session, + lane: snapshot.is_some(), + ..Cursor::default() + }; + let capture = Capture::resume(path, cursor); + let object = self + .open_epoch(ownership, ns, actor, owned, cache, snapshot) + .await?; + self.dormant.borrow_mut().remove(&key); + self.objects + .borrow_mut() + .insert(key, Slot::opened(object, capture)); + Ok(()) + } + + /// Open a new object's first epoch at its claim, before a request + /// reaches it: create own.json, open, in one conditional write. A + /// claimant that loses the write answers `false` and routes to the + /// winner. Nothing to snapshot: an epoch that committed nothing restores + /// as an empty database. + pub async fn open_fresh( + &self, + ownership: &Ownership, + ns: &str, + actor: &str, + owned: &Owned, + ) -> Result { + if !ownership.open(ns, actor, owned.epoch).await? { + return Ok(false); + } + tracing::info!("cluster: {ns}/{actor} epoch {} opens new", owned.epoch); + self.opened + .borrow_mut() + .insert(object_key(ns, actor), owned.epoch); + Ok(true) + } + + /// Open a claimed epoch over the recovered state. Its session starts + /// with the state's snapshot (none for an empty database), which the + /// bucket holds before the epoch opens. + async fn open_epoch( + &self, + ownership: &Ownership, + ns: &str, + actor: &str, + owned: &Owned, + cache: PathBuf, + snapshot: Option, + ) -> Result { + let mut object = Object { + ns: ns.to_owned(), + actor: actor.to_owned(), + epoch: owned.epoch, + replica: Replica { + client: self.client(ns, actor), + cache, + session: owned.epoch as u32, + }, + tiered: 0, + cursor: Cursor::default(), + files: Vec::new(), + prune: false, + }; + if self + .opened + .borrow_mut() + .remove(&object_key(ns, actor)) + .is_some() + { + // A new object, opened at its claim. + return Ok(object); + } + if let Some(snapshot) = snapshot { + object.files.push(upload(&object.replica, snapshot).await?); + } + if !ownership.open(ns, actor, owned.epoch).await? { + return Err(Error::Contended(format!( + "{ns}/{actor}: epoch {} was superseded before it opened", + owned.epoch + ))); + } + tracing::info!("cluster: {ns}/{actor} epoch {} open", owned.epoch); + object.prune = true; + Ok(object) + } + + /// Capture every commit since the last call and make it durable in the + /// bucket. + /// + /// Commits upload as a group. Each captures its own transactions, then + /// one at a time uploads everything captured so far on behalf of all, + /// so a hot object pays an upload per group, not per transaction. + pub async fn commit(&self, ownership: &Ownership, ns: &str, actor: &str) -> Result<()> { + let slot = self.objects.borrow().get(&object_key(ns, actor)).cloned(); + let slot = + slot.ok_or_else(|| Error::Contended(format!("{ns}/{actor} is not open here")))?; + let target = { + let guard = slot.object.lock().await; + let target = slot.capture.borrow().last()?; + (guard.epoch, target) + }; + let mut acked = slot.acked.subscribe(); + loop { + if *acked.borrow_and_update() >= target { + return Ok(()); + } + if slot.pushing.replace(true) { + let _ = acked.changed().await; + continue; + } + let _pushing = Pushing(&slot); + self.push(ownership, &slot).await?; + } + } + + /// Upload everything captured, and acknowledge it if own.json still + /// names the epoch after the upload. + async fn push(&self, ownership: &Ownership, slot: &Slot) -> Result<()> { + let (ns, actor, epoch, round, staged, replica) = { + let guard = slot.object.lock().await; + let round = slot.capture.borrow().last()?; + let staged = stage(&slot.capture.borrow(), guard.tiered, round)?; + let (ns, actor) = (guard.ns.clone(), guard.actor.clone()); + (ns, actor, guard.epoch, round, staged, guard.replica.clone()) + }; + let file = put(&replica, round, staged).await?; + { + let mut guard = slot.object.lock().await; + if guard.epoch == epoch { + slot.tiered(&mut guard, round, file); + } + } + if ownership.current_epoch(&ns, &actor).await? != Some(epoch) { + return Err(Error::Contended(format!( + "{ns}/{actor}: epoch {epoch} was superseded" + ))); + } + slot.ack(epoch, round); + Ok(()) + } + + /// Delete the sessions before each newly opened epoch's, and snapshot + /// every owned object whose log outgrew its database. Runs off the + /// commit path. + pub async fn tier(&self) { + let objects: Vec<_> = self.objects.borrow().values().cloned().collect(); + stream::iter(objects) + .for_each_concurrent(TIER_CONCURRENCY, |object| async move { + let (due, prune) = { + let mut guard = object.object.lock().await; + let prune = std::mem::take(&mut guard.prune); + ( + snapshot_due(&guard.files), + prune.then(|| guard.replica.clone()), + ) + }; + if let Some(replica) = prune { + prune_older(&replica).await; + } + if due { + self.compact(&object).await; + } + }) + .await; + } + + /// Upload what the bucket lacks for one object. + async fn tier_one(&self, object: &Slot) { + let guard = object.object.lock().await; + let Ok(through) = object.capture.borrow().last() else { + return; + }; + let staged = stage(&object.capture.borrow(), guard.tiered, through); + let Ok(Some(staged)) = staged else { + return; + }; + let replica = guard.replica.clone(); + drop(guard); + let Ok(file) = put(&replica, through, Some(staged)).await else { + return; + }; + let mut guard = object.object.lock().await; + object.tiered(&mut guard, through, file); + } + + /// Snapshot an epoch's session from its newest snapshot and the batches + /// past it, and delete what the new one covers, so a takeover's restore + /// reads one file, not one per upload. The snapshot replays the log, + /// never the live database, so it runs off the event loop. + async fn compact(&self, slot: &Slot) { + let (replica, epoch, plan, key) = { + let guard = slot.object.lock().await; + let key = format!("{}/{}", guard.ns, guard.actor); + (guard.replica.clone(), guard.epoch, plan(&guard.files), key) + }; + let snapshot = match merged(&replica, &plan).await { + Ok(snapshot) => snapshot, + Err(e) => return tracing::warn!("cluster: {key} compaction failed: {e}"), + }; + let from = plan.first().filter(|f| f.snapshot).map_or(0, |f| f.frame); + if snapshot.frame <= from { + return tracing::warn!("cluster: {key} log has a gap after {from}"); + } + let snapshot = match upload(&replica, snapshot).await { + Ok(snapshot) => snapshot, + Err(e) => return tracing::warn!("cluster: {key} snapshot upload failed: {e}"), + }; + let replaced: Vec = { + let mut guard = slot.object.lock().await; + if guard.epoch != epoch { + return; + } + slot.tiered(&mut guard, snapshot.frame, Some(snapshot)); + let (replaced, kept) = std::mem::take(&mut guard.files) + .into_iter() + .partition(|f| f.frame <= snapshot.frame && *f != snapshot); + guard.files = kept; + replaced + }; + let replica = &replica; + stream::iter(replaced) + .for_each_concurrent(TIER_CONCURRENCY, |f| async move { + let _ = std::fs::remove_file(replica.cache.join(f.name())); + let _ = replica.client.delete(&f).await; + }) + .await; + } + + /// Whether an object is open here, or opened by its claim. + pub fn holds(&self, ns: &str, actor: &str) -> bool { + let key = object_key(ns, actor); + self.objects.borrow().contains_key(&key) || self.opened.borrow().contains_key(&key) + } + + /// Stop serving an object workerd closed, keeping it for its eviction. + pub fn retire(&self, ns: &str, actor: &str) { + let key = object_key(ns, actor); + self.opened.borrow_mut().remove(&key); + if let Some(slot) = self.objects.borrow_mut().remove(&key) { + self.retiring.borrow_mut().insert(key, slot); + } + } + + /// Keep an object workerd evicted for being idle: upload what the bucket + /// lacks, and remember that this node's file holds the epoch. The record + /// still names this node, so other nodes route here, and the next + /// request wakes it without a restore. + pub async fn evict(&self, ns: &str, actor: &str, epoch: Epoch) { + let _permit = self.evictions.acquire().await; + let key = object_key(ns, actor); + let Some(slot) = self.retiring.borrow_mut().remove(&key) else { + return; + }; + self.tier_one(&slot).await; + let mut guard = slot.object.lock().await; + let last = slot.capture.borrow().last(); + guard.cursor = slot.capture.borrow().cursor(); + if guard.epoch == epoch && last.is_ok_and(|last| guard.tiered >= last) { + // Bounded: a forgotten entry costs its object a restore. + let mut map = self.dormant.borrow_mut(); + if map.len() >= DORMANT_MAX { + map.clear(); + } + map.insert(key, guard.clone()); + } + } + + /// The epoch this node evicted an object at, with the bucket holding it. + pub fn dormant(&self, ns: &str, actor: &str) -> Option { + let dormant = self.dormant.borrow(); + dormant.get(&object_key(ns, actor)).map(|o| o.epoch) + } + + /// Forget an object that left this node. + pub fn close(&self, ns: &str, actor: &str) { + let key = object_key(ns, actor); + self.objects.borrow_mut().remove(&key); + self.opened.borrow_mut().remove(&key); + } + + /// An owned object's database was deleted: its session continues with + /// an empty database, which the next commit makes durable. + pub fn database_reset(&self, ns: &str, actor: &str) { + if let Some(slot) = self.objects.borrow().get(&object_key(ns, actor)) { + slot.capture.borrow_mut().reset(); + } + } + + /// A write `SQLite` made to an owned object's WAL. + pub fn wal_write(&self, ns: &str, actor: &str, offset: u64, data: &[u8]) { + if let Some(slot) = self.objects.borrow().get(&object_key(ns, actor)) { + slot.capture.borrow_mut().write(offset, data); + } + } +} + +/// The capture's frames from `from` up to `to`, merged into one batch. +fn stage(capture: &Capture, from: Sequence, to: Sequence) -> Result>> { + if to <= from { + return Ok(None); + } + let batch = crate::log::merge(&capture.batch(from, to)?)?; + Ok(Some(crate::log::compress(&batch)?)) +} + +/// `plan`'s files replayed into this node's copy of a snapshot, on the +/// blocking pool. A missing copy is fetched from the bucket first. +async fn merged(replica: &Replica, plan: &[File]) -> Result { + let paths: Vec<_> = plan.iter().map(|f| replica.cache.join(f.name())).collect(); + for (f, path) in plan.iter().zip(&paths) { + if !path.exists() { + let tmp = tmp_path(path); + replica.client.get(f, &tmp).await?; + std::fs::rename(&tmp, path)?; + } + } + let files: Vec<_> = plan.iter().copied().zip(paths).collect(); + let cache = replica.cache.clone(); + blocking(move || crate::log::compact(&files, &cache)).await +} + +/// Writes a copy whole and renames it into place, so a reader never sees +/// part of one. +fn keep(path: &Path, bytes: &[u8]) -> std::io::Result<()> { + let tmp = tmp_path(path); + std::fs::write(&tmp, bytes)?; + std::fs::rename(&tmp, path) +} + +/// A temporary name beside `path`, unique within the process: a push and a +/// tiering pass can stage the same file at once. +fn tmp_path(path: &Path) -> PathBuf { + static NEXT: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0); + let n = NEXT.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let mut tmp = path.as_os_str().to_owned(); + tmp.push(format!(".{n}.tmp")); + tmp.into() +} + +pub async fn blocking( + f: impl FnOnce() -> Result + Send + 'static, +) -> Result { + tokio::task::spawn_blocking(f) + .await + .map_err(|e| Error::Replica(format!("blocking task: {e}")))? +} + +/// Where an epoch's session is: the bucket, and this node's copy of it. +#[derive(Clone)] +struct Replica { + client: Log, + /// `.log/e`. + cache: PathBuf, + session: u32, +} + +/// The copies' directory for `epoch` of the database at `path`, created +/// empty. Other epochs' copies are removed. +fn open_cache(path: &Path, epoch: Epoch) -> Result { + let root = cache_root(path); + remove_dir(&root)?; + let cache = root.join(format!("e{epoch}")); + std::fs::create_dir_all(&cache)?; + Ok(cache) +} + +fn cache_root(path: &Path) -> PathBuf { + let mut root = path.as_os_str().to_owned(); + root.push(".log"); + root.into() +} + +fn remove_dir(path: &Path) -> Result<()> { + match std::fs::remove_dir_all(path) { + Err(e) if e.kind() != std::io::ErrorKind::NotFound => Err(crate::Error::from(e)), + _ => Ok(()), + } +} + +/// Whether `files`, a session's, call for a snapshot: the log since the +/// last one reached its size, so bucket writes stay within about twice the +/// log whatever the database's size. +fn snapshot_due(files: &[File]) -> bool { + let (image, log) = plan(files).iter().fold((0, 0), |(image, log), f| { + if f.snapshot { + (f.size, log) + } else { + (image, log + f.size) + } + }); + log > 0 && log >= image +} + +/// Uploads this node's copy of `file`. +/// Delete the sessions before `replica`'s. A superseded owner still +/// uploading may leave files behind, which the next epoch deletes. +async fn prune_older(replica: &Replica) { + let files = match replica.client.older(replica.session).await { + Ok(files) => files, + Err(e) => return tracing::warn!("cluster: pruning old sessions: {e}"), + }; + stream::iter(files) + .for_each_concurrent(TIER_CONCURRENCY, |f| async move { + let _ = replica.client.delete(&f).await; + }) + .await; +} + +async fn upload(replica: &Replica, file: File) -> Result { + let path = replica.cache.join(file.name()); + replica.client.put(&file, &path).await?; + Ok(file) +} + +/// Keeps `staged`, if any, as this node's copy of the batch up to `to`, +/// then uploads it. +async fn put(replica: &Replica, to: Sequence, staged: Option>) -> Result> { + let Some(bytes) = staged else { + return Ok(None); + }; + let file = File { + snapshot: false, + session: replica.session, + frame: to, + size: bytes.len() as u64, + }; + let path = replica.cache.join(file.name()); + blocking(move || Ok(keep(&path, &bytes)?)).await?; + upload(replica, file).await.map(Some) +} + +fn remove_file(path: &Path) -> Result<()> { + match std::fs::remove_file(path) { + Err(e) if e.kind() != std::io::ErrorKind::NotFound => Err(crate::Error::from(e)), + _ => Ok(()), + } +} + +/// A local copy is never authoritative at takeover: every acknowledged +/// commit is in the bucket. +fn remove_database(path: &Path) -> Result<()> { + for suffix in ["", "-wal", "-shm"] { + let mut file = path.as_os_str().to_owned(); + file.push(suffix); + remove_file(Path::new(&file))?; + } + remove_dir(&cache_root(path)) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn file(snapshot: bool, frame: Sequence, size: u64) -> File { + File { + snapshot, + session: 1, + frame, + size, + } + } + + #[test] + fn a_snapshot_is_due_when_the_log_reaches_its_size() { + let mut files = vec![file(true, 0, 1000), file(false, 3, 960)]; + assert!(!snapshot_due(&files), "960 bytes of log, a 1000-byte image"); + files.push(file(false, 5, 40)); + assert!(snapshot_due(&files)); + files.push(file(true, 5, 1000)); + assert!(!snapshot_due(&files), "a snapshot covers the log"); + assert!(snapshot_due(&[file(false, 2, 1)]), "no snapshot yet"); + assert!(!snapshot_due(&[])); + } +} diff --git a/src/workerd/server/cluster/ownership.rs b/src/workerd/server/cluster/ownership.rs new file mode 100644 index 00000000000..ab0051084d9 --- /dev/null +++ b/src/workerd/server/cluster/ownership.rs @@ -0,0 +1,633 @@ +// Copyright (c) 2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +use std::cell::Cell; +use std::cell::RefCell; +use std::collections::HashMap; +use std::rc::Rc; + +use serde::Deserialize; +use serde::Serialize; + +use crate::Bucket; +use crate::Error; +use crate::Guard; +use crate::Registry; +use crate::Result; +use crate::registry::NodeRecord; + +/// One Durable Object's record in `do///own.json`, the only +/// authority for who owns it and whose log holds its latest commits. +#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize)] +struct OwnerRecord { + /// The owning instance (`NodeRecord::node`); empty for none. + node: String, + /// The owner's epoch: its session of the log. + epoch: u64, + /// The last epoch that opened. A claim writes the next `epoch` and + /// copies `from` from the record it replaced; the opening write sets it + /// to `epoch`. So an epoch is recovering while `from != epoch`, and + /// recovers `from`'s log (none when 0). + from: u64, +} + +impl OwnerRecord { + /// `node`'s claim on the next epoch. A claimant that died mid-recovery + /// leaves `from` unchanged. + fn successor(&self, node: &str) -> Self { + Self { + node: node.to_owned(), + epoch: self.epoch + 1, + from: self.from, + } + } + + fn owned(self) -> Owned { + Owned { + recover: (self.from != self.epoch).then_some(self.from), + epoch: self.epoch, + } + } +} + +/// This instance's claim on an object. +#[derive(Debug, PartialEq, Eq)] +pub struct Owned { + pub epoch: u64, + /// The epoch whose log a new owner recovers (0: none). `None` when this + /// instance already opened the epoch. + pub recover: Option, +} + +/// Who serves an object. +#[derive(Debug, PartialEq, Eq)] +pub enum Claim { + Owned(Owned), + /// A live peer owns it; send the request there. A `stale` route is + /// still used, and the caller refreshes it in the background. + Route { + node: String, + address: String, + stale: bool, + }, +} + +/// Claims, routes and releases Durable Objects through their owner records. +pub struct Ownership { + bucket: Bucket, + registry: Rc, + /// Whether this instance may claim objects: off once it drains. + pub may_claim: Cell, + /// Where each routed object's owner was, and when that was read. + routes: RefCell>, + /// The record each claim wrote, and the guard that it is still the + /// current one, so `open` changes it without reading it back. A claim + /// of an object with no record writes nothing: `open` creates the + /// record, open, in one conditional write, as celld claims a new cell. + claimed: RefCell>, +} + +struct Route { + node: String, + address: String, + read_ms: u64, +} + +/// How old a route gets before a use of it also rereads the owner record. +/// A route is used for as long as its node is in the registry listing, as +/// celld does: rereading on the request path put a bucket read on one +/// request in every interval, and dropping the route at a time limit does +/// nothing a dead owner's lease does not already do. A route to a live +/// former owner costs a hop, which that owner routes on. +const ROUTE_REFRESH_MS: u64 = 5_000; + +/// How many times a claim retries when another instance keeps winning. +const ATTEMPTS: usize = 8; + +fn draining(key: &str) -> Error { + Error::Replica(format!("{key}: this node is draining")) +} + +fn owner_key(namespace: &str, actor: &str) -> String { + format!("{}/own.json", crate::object_prefix(namespace, actor)) +} + +impl Ownership { + pub fn new(bucket: Bucket, registry: Rc) -> Self { + Self { + bucket, + registry, + may_claim: Cell::new(true), + routes: RefCell::new(HashMap::new()), + claimed: RefCell::new(HashMap::new()), + } + } + + /// Own the object, or learn which live peer does. A record with no owner, + /// or one naming an instance whose lease expired, is taken over with the + /// next epoch; the conditional write makes exactly one claimant win. + pub async fn acquire_or_route( + &self, + namespace: &str, + actor: &str, + now_ms: u64, + ) -> Result { + let key = owner_key(namespace, actor); + let me = self.registry.node(); + // A route whose node a stale listing no longer shows: its own record + // decides, not the object's. + let unlisted = self.routes.borrow().get(&key).map(|r| r.node.clone()); + if let Some(node) = unlisted + && !self.registry.listed_live(&node, now_ms) + { + self.registry.live(&node, now_ms).await?; + } + if let Some(route) = self.routes.borrow_mut().get_mut(&key) + && self.registry.listed_live(&route.node, now_ms) + { + // One refresh per interval: the first use past it takes it. + let stale = now_ms >= route.read_ms + ROUTE_REFRESH_MS; + if stale { + route.read_ms = now_ms; + } + return Ok(Claim::Route { + node: route.node.clone(), + address: route.address.clone(), + stale, + }); + } + for _ in 0..ATTEMPTS { + let (previous, guard) = match self.bucket.get(&key).await? { + None => (OwnerRecord::default(), Guard::Absent), + Some((body, version)) => { + let record: OwnerRecord = serde_json::from_slice(&body)?; + if record.node == me { + // A draining node lets its objects go, its own too. + if !self.may_claim.get() { + return Err(draining(&key)); + } + return Ok(Claim::Owned(record.owned())); + } + if !record.node.is_empty() + && let Some(peer) = self.registry.live(&record.node, now_ms).await? + { + // An earlier process at our address, restarted + // before its lease ran out: a route there loops back + // here. Its lease, not a route, ends this. + if peer.address == self.registry.advertise() { + return Err(Error::Replica(format!( + "{key}: owned by this address's previous instance" + ))); + } + return Ok(self.route(key, peer, now_ms)); + } + (record, Guard::Match(version)) + } + }; + if !self.may_claim.get() { + return Err(draining(&key)); + } + let record = previous.successor(me); + // A new object opens at its claim, where a loser still routes + // (`open_fresh`). + let fresh = matches!(guard, Guard::Absent); + let written = if fresh { + Some(Guard::Absent) + } else { + let body = serde_json::to_vec(&record)?; + let version = self.bucket.put_if(&key, body, guard).await?; + version.map(Guard::Match) + }; + if let Some(guard) = written { + self.claimed + .borrow_mut() + .insert(key, (record.clone(), guard)); + return Ok(Claim::Owned(record.owned())); + } + } + Err(Error::Contended(key)) + } + + fn route(&self, key: String, peer: NodeRecord, now_ms: u64) -> Claim { + let claim = Claim::Route { + node: peer.node.clone(), + address: peer.address.clone(), + stale: false, + }; + let route = Route { + node: peer.node, + address: peer.address, + read_ms: now_ms, + }; + self.routes.borrow_mut().insert(key, route); + claim + } + + /// Reread a stale route's owner record. The route goes if no live peer + /// owns the object any more; the next request then claims or routes. + pub async fn refresh_route(&self, namespace: &str, actor: &str, now_ms: u64) -> Result<()> { + let key = owner_key(namespace, actor); + let node = match self.bucket.get(&key).await? { + Some((body, _)) => serde_json::from_slice::(&body)?.node, + None => String::new(), + }; + let peer = if node.is_empty() || node == self.registry.node() { + None + } else { + self.registry.live(&node, now_ms).await? + }; + match peer { + Some(peer) => drop(self.route(key, peer, now_ms)), + None => drop(self.routes.borrow_mut().remove(&key)), + } + Ok(()) + } + + /// Open the epoch this instance claimed and recovered. Answers whether + /// the record still named this claim; if not, the epoch never opens. + pub async fn open(&self, namespace: &str, actor: &str, epoch: u64) -> Result { + let key = owner_key(namespace, actor); + let claimed = self.claimed.borrow_mut().remove(&key); + let (mut record, guard) = if let Some(claimed) = claimed { + claimed + } else { + let Some((body, version)) = self.bucket.get(&key).await? else { + return Ok(false); + }; + (serde_json::from_slice(&body)?, Guard::Match(version)) + }; + if record.node != self.registry.node() || record.epoch != epoch || record.from == epoch { + return Ok(false); + } + record.from = epoch; + let body = serde_json::to_vec(&record)?; + Ok(self.bucket.put_if(&key, body, guard).await?.is_some()) + } + + /// Move this instance's open epoch to the next, which recovers the one + /// it leaves like any other. `None` when the record no longer names this + /// open epoch. + pub async fn advance(&self, namespace: &str, actor: &str, epoch: u64) -> Result> { + let key = owner_key(namespace, actor); + let Some((body, version)) = self.bucket.get(&key).await? else { + return Ok(None); + }; + let previous: OwnerRecord = serde_json::from_slice(&body)?; + if previous.node != self.registry.node() + || previous.epoch != epoch + || previous.from != epoch + { + return Ok(None); + } + let record = previous.successor(&previous.node); + let body = serde_json::to_vec(&record)?; + let Some(version) = self + .bucket + .put_if(&key, body, Guard::Match(version)) + .await? + else { + return Ok(None); + }; + self.claimed + .borrow_mut() + .insert(key, (record.clone(), Guard::Match(version))); + Ok(Some(record.owned())) + } + + /// The epoch own.json names now: the re-read after a bucket upload that + /// makes the upload an acknowledgement (spec: `TierAck`). + pub async fn current_epoch(&self, namespace: &str, actor: &str) -> Result> { + Ok(match self.bucket.get(&owner_key(namespace, actor)).await? { + None => None, + Some((body, _)) => Some(serde_json::from_slice::(&body)?.epoch), + }) + } +} + +#[cfg(test)] +mod tests { + #![allow(clippy::unwrap_used)] + + use std::rc::Rc; + + use futures::executor::block_on; + + use crate::Bucket; + use crate::Claim; + use crate::Error; + use crate::Ownership; + use crate::Registry; + use crate::ownership::OwnerRecord; + use crate::ownership::owner_key; + + const TTL: u64 = 10_000; + + /// Matches any `Owned` claim at `epoch`. + fn owned(epoch: u64) -> EpochOf { + EpochOf(epoch) + } + + #[derive(Debug)] + struct EpochOf(u64); + + impl PartialEq for Claim { + fn eq(&self, other: &EpochOf) -> bool { + matches!(self, Self::Owned(o) if o.epoch == other.0) + } + } + + fn node(store: &Bucket, name: &str) -> (Rc, Ownership) { + let bucket = store.at("cluster"); + let registry = Rc::new(Registry::new( + bucket.clone(), + name, + &format!("{name}:1"), + TTL, + )); + (registry.clone(), Ownership::new(bucket, registry)) + } + + /// `own` claims and opens a new object `x` at epoch 1: a new object's + /// record appears when its epoch opens. + async fn open_x(own: &Ownership) { + let claim = own.acquire_or_route("ns", "x", 0).await; + assert_eq!(claim.unwrap(), owned(1)); + assert!(own.open("ns", "x", 1).await.unwrap()); + } + + #[test] + fn peers_are_the_live_registrations() { + block_on(async { + let store = Bucket::memory(""); + let (a, _) = node(&store, "a"); + let (b, _) = node(&store, "b"); + a.renew(0).await.unwrap(); + b.renew(5_000).await.unwrap(); + let names = |peers: Vec| { + peers.into_iter().map(|peer| peer.node).collect::>() + }; + assert_eq!(names(a.peers(1_000).await.unwrap()), ["a", "b"]); + // a's lease ran out at 10s; b's holds until 15s. + assert_eq!(names(a.peers(12_000).await.unwrap()), ["b"]); + b.deregister().await.unwrap(); + assert!(a.peers(12_000).await.unwrap().is_empty()); + }); + } + + #[test] + fn a_restart_never_routes_to_its_previous_instance() { + block_on(async { + let store = Bucket::memory(""); + let (a, a_own) = node(&store, "a"); + a.renew(0).await.unwrap(); + open_x(&a_own).await; + // The process restarts at a's address with a fresh node id while + // a's record is still live: routing there would reach itself. + let bucket = store.at("cluster"); + let a_ = Rc::new(Registry::new(bucket.clone(), "a_", "a:1", TTL)); + a_.renew(1_000).await.unwrap(); + let a_own_ = Ownership::new(bucket, a_); + let claim = a_own_.acquire_or_route("ns", "x", 1_000).await; + assert!(matches!(claim, Err(Error::Replica(_))), "{claim:?}"); + // Once a's lease ends, the restart takes it over. + let claim = a_own_.acquire_or_route("ns", "x", 11_000).await; + assert_eq!(claim.unwrap(), owned(2)); + }); + } + + #[test] + fn a_rewritten_node_record_fences_its_instance() { + block_on(async { + let store = Bucket::memory(""); + let (a, _) = node(&store, "a"); + let (impostor, _) = node(&store, "a"); + a.renew(0).await.unwrap(); + // A second process with the same identity cannot publish over it. + assert!(matches!(impostor.renew(0).await, Err(Error::Fenced(_)))); + a.renew(1_000).await.unwrap(); + }); + } + + #[test] + fn a_claim_owns_an_unowned_object_and_routes_to_a_live_owner() { + block_on(async { + let store = Bucket::memory(""); + let (a, a_own) = node(&store, "a"); + let (b, b_own) = node(&store, "b"); + a.renew(0).await.unwrap(); + b.renew(0).await.unwrap(); + open_x(&a_own).await; + // Asking again keeps it, at the same epoch. + assert_eq!( + a_own.acquire_or_route("ns", "x", 1).await.unwrap(), + owned(1) + ); + assert_eq!( + b_own.acquire_or_route("ns", "x", 1).await.unwrap(), + Claim::Route { + node: "a".into(), + address: "a:1".into(), + stale: false, + } + ); + }); + } + + #[test] + fn a_stale_listing_keeps_a_route_to_a_live_owner() { + block_on(async { + let store = Bucket::memory(""); + let (a, a_own) = node(&store, "a"); + let (b, b_own) = node(&store, "b"); + a.renew(0).await.unwrap(); + b.renew(0).await.unwrap(); + open_x(&a_own).await; + let route = b_own.acquire_or_route("ns", "x", 1).await; + assert!(matches!(route.unwrap(), Claim::Route { .. })); + // a renews, but b lists nothing more: what b knows of a expires. + a.renew(8_000).await.unwrap(); + // With the record gone a reread would claim x, so a route shows + // that only a's own record was read. + store.delete("cluster/do/ns/x/own.json").await.unwrap(); + assert_eq!( + b_own.acquire_or_route("ns", "x", 12_000).await.unwrap(), + Claim::Route { + node: "a".into(), + address: "a:1".into(), + stale: true, + } + ); + }); + } + + #[test] + fn a_dead_owners_object_is_taken_over_at_the_next_epoch() { + block_on(async { + let store = Bucket::memory(""); + let (a, a_own) = node(&store, "a"); + let (b, b_own) = node(&store, "b"); + a.renew(0).await.unwrap(); + b.renew(0).await.unwrap(); + open_x(&a_own).await; + // a stops renewing; b renews past a's expiry and takes x over. + b.renew(9_000).await.unwrap(); + assert_eq!( + b_own.acquire_or_route("ns", "x", 11_000).await.unwrap(), + owned(2) + ); + assert_eq!( + a_own.acquire_or_route("ns", "x", 11_000).await.unwrap(), + Claim::Route { + node: "b".into(), + address: "b:1".into(), + stale: false, + } + ); + }); + } + + #[test] + fn an_owner_advances_only_its_open_epoch() { + block_on(async { + let store = Bucket::memory(""); + let (a, a_own) = node(&store, "a"); + a.renew(0).await.unwrap(); + open_x(&a_own).await; + // The next epoch recovers the one it left, and does not advance + // until it opens. + let next = a_own.advance("ns", "x", 1).await.unwrap().unwrap(); + assert_eq!((next.epoch, next.recover), (2, Some(1))); + assert!(a_own.advance("ns", "x", 2).await.unwrap().is_none()); + assert!(a_own.open("ns", "x", 2).await.unwrap()); + // A record changed since is not advanced over. + vacate(&store).await; + assert!(a_own.advance("ns", "x", 2).await.unwrap().is_none()); + }); + } + + /// Clear the owner of `x`, as another writer might, under a cached read. + async fn vacate(store: &Bucket) { + let bucket = store.at("cluster"); + let key = owner_key("ns", "x"); + let (body, _) = bucket.get(&key).await.unwrap().unwrap(); + let mut record: OwnerRecord = serde_json::from_slice(&body).unwrap(); + record.node = String::new(); + bucket + .put(&key, serde_json::to_vec(&record).unwrap(), &[]) + .await + .unwrap(); + } + + #[test] + fn concurrent_claims_open_exactly_one_epoch() { + block_on(async { + let store = Bucket::memory(""); + let nodes: Vec<_> = (0..8).map(|i| node(&store, &format!("n{i}"))).collect(); + for (registry, _) in &nodes { + registry.renew(0).await.unwrap(); + } + let claims = futures::future::join_all( + nodes + .iter() + .map(|(_, own)| own.acquire_or_route("ns", "x", 0)), + ) + .await; + // A new object's claims all succeed; the conditional write that + // opens the epoch has exactly one winner. + assert!(claims.iter().all(|c| matches!(c, Ok(Claim::Owned(_))))); + let opens = + futures::future::join_all(nodes.iter().map(|(_, own)| own.open("ns", "x", 1))) + .await; + let owners = opens.iter().filter(|o| matches!(o, Ok(true))).count(); + assert_eq!(owners, 1); + }); + } + + #[test] + fn a_takeover_recovers_the_last_epoch_that_opened() { + block_on(async { + let store = Bucket::memory(""); + let (a, a_own) = node(&store, "a"); + let (b, b_own) = node(&store, "b"); + let (c, c_own) = node(&store, "c"); + for r in [&a, &b, &c] { + r.renew(0).await.unwrap(); + } + let Claim::Owned(first) = a_own.acquire_or_route("ns", "x", 0).await.unwrap() else { + panic!("a should own x") + }; + assert_eq!(first.recover, Some(0)); + assert!(a_own.open("ns", "x", 1).await.unwrap()); + // Only the claimant, at its epoch, opens. + assert!(!b_own.open("ns", "x", 1).await.unwrap()); + + // a's lease runs out; b claims epoch 2 and dies before opening. + b.renew(9_000).await.unwrap(); + c.renew(9_000).await.unwrap(); + let Claim::Owned(second) = b_own.acquire_or_route("ns", "x", 11_000).await.unwrap() + else { + panic!("b should own x") + }; + assert_eq!(second.recover, Some(1)); + + // c takes over from b's unopened epoch: it still recovers epoch 1. + c.renew(20_000).await.unwrap(); + let Claim::Owned(third) = c_own.acquire_or_route("ns", "x", 21_000).await.unwrap() + else { + panic!("c should own x") + }; + assert_eq!(third.epoch, 3); + assert_eq!(third.recover, Some(1)); + }); + } + + #[test] + fn a_listed_owners_route_is_used_and_refreshed_in_the_background() { + block_on(async { + let store = Bucket::memory(""); + let (a, a_own) = node(&store, "a"); + let (b, b_own) = node(&store, "b"); + a.renew(0).await.unwrap(); + b.renew(0).await.unwrap(); + b.peers(0).await.unwrap(); + open_x(&a_own).await; + let to_a = |stale| Claim::Route { + node: "a".into(), + address: "a:1".into(), + stale, + }; + let route = async |now| b_own.acquire_or_route("ns", "x", now).await.unwrap(); + assert_eq!(route(1_000).await, to_a(false)); + // x loses its owner; b still routes to a until it refreshes. + vacate(&store).await; + assert_eq!(route(5_000).await, to_a(false)); + // The first use past the interval is stale, and only the first. + assert_eq!(route(6_000).await, to_a(true)); + assert_eq!(route(6_001).await, to_a(false)); + // The refresh finds no owner, so b claims x. + b_own.refresh_route("ns", "x", 6_002).await.unwrap(); + assert_eq!(route(6_003).await, owned(2)); + }); + } + + #[test] + fn a_route_ends_with_its_owners_listed_lease() { + block_on(async { + let store = Bucket::memory(""); + let (a, a_own) = node(&store, "a"); + let (b, b_own) = node(&store, "b"); + a.renew(0).await.unwrap(); + b.renew(0).await.unwrap(); + b.peers(0).await.unwrap(); + open_x(&a_own).await; + b_own.acquire_or_route("ns", "x", 1).await.unwrap(); + // a dies; b's route goes when a's listed lease ends. + b.renew(9_000).await.unwrap(); + assert_eq!( + b_own.acquire_or_route("ns", "x", 11_000).await.unwrap(), + owned(2) + ); + }); + } +} diff --git a/src/workerd/server/cluster/registry.rs b/src/workerd/server/cluster/registry.rs new file mode 100644 index 00000000000..486dc7c72ed --- /dev/null +++ b/src/workerd/server/cluster/registry.rs @@ -0,0 +1,321 @@ +// Copyright (c) 2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +use std::cell::RefCell; +use std::collections::HashMap; + +use serde::Deserialize; +use serde::Serialize; + +use crate::Bucket; +use crate::Error; +use crate::Guard; +use crate::Result; +use crate::Version; + +/// One instance's record in `nodes/.json`. +#[derive(Clone, Debug, PartialEq, Eq, Serialize, Deserialize)] +pub struct NodeRecord { + /// The instance's identity: 16 random bytes, hex, new each start. + pub node: String, + /// Where peers reach it, `host:port`. + pub address: String, + /// The instance is live until this time and must renew before it. + pub expires_ms: u64, + /// The process that wrote it. + #[serde(default)] + pub instance: u64, +} + +/// This instance's membership: it publishes its own record and reads its +/// peers'. A peer whose lease expired is dead; nothing else is. +pub struct Registry { + bucket: Bucket, + node: String, + address: String, + ttl_ms: u64, + /// The version of our own record we last wrote, so a renewal replaces + /// exactly that and notices anyone else's write. + own: RefCell>, + /// Random per process: a renewal whose answer was lost may still have + /// landed, and this tells that write from another writer's. + instance: u64, + /// Live peers' records, as of the last listing. + listed: RefCell>, +} + +fn node_key(node: &str) -> String { + format!("nodes/{node}.json") +} + +impl Registry { + pub fn new(bucket: Bucket, node: &str, address: &str, ttl_ms: u64) -> Self { + Self { + bucket, + node: node.to_owned(), + address: address.to_owned(), + ttl_ms, + own: RefCell::new(None), + instance: std::hash::BuildHasher::hash_one( + &std::collections::hash_map::RandomState::new(), + std::time::SystemTime::now(), + ), + listed: RefCell::new(HashMap::new()), + } + } + + pub fn node(&self) -> &str { + &self.node + } + + /// Where this instance listens. + pub fn advertise(&self) -> &str { + &self.address + } + + /// Where peer `node` listens, as of the last listing. + pub fn address(&self, node: &str) -> Option { + self.listed.borrow().get(node).map(|r| r.address.clone()) + } + + /// Publish or renew this instance's record; answers the new expiry. The + /// first write requires the key to be absent (the node id is fresh per + /// process), and every later one requires our last version, so a record + /// someone else wrote fences this process instead of being overwritten. + pub async fn renew(&self, now_ms: u64) -> Result { + let record = NodeRecord { + node: self.node.clone(), + address: self.address.clone(), + expires_ms: now_ms + self.ttl_ms, + instance: self.instance, + }; + let key = node_key(&self.node); + let body = serde_json::to_vec(&record)?; + let mut guard = self + .own + .borrow() + .clone() + .map_or(Guard::Absent, Guard::Match); + loop { + if let Some(version) = self.bucket.put_if(&key, body.clone(), guard).await? { + self.own.replace(Some(version)); + return Ok(record.expires_ms); + } + // Not the version we know. If this process wrote it, earlier + // renewals landed and only their answers were lost (timed-out + // rounds on the fleet), and one may land between our read and + // write: retry. Anything else is another writer. + let Some((current, version)) = self.bucket.get(&key).await? else { + return Err(Error::Fenced(self.node.clone())); + }; + let current: NodeRecord = serde_json::from_slice(¤t)?; + if current.instance != self.instance { + return Err(Error::Fenced(self.node.clone())); + } + guard = Guard::Match(version); + } + } + + /// Remove this instance's record on a clean shutdown, so peers stop + /// routing to it before its lease would have run out. + pub async fn deregister(&self) -> Result<()> { + self.own.take(); + self.bucket.delete(&node_key(&self.node)).await + } + + /// Whether `node` held a lease past `now_ms` at the last listing. No + /// I/O: a route checks it on every use. + pub fn listed_live(&self, node: &str, now_ms: u64) -> bool { + self.listed + .borrow() + .get(node) + .is_some_and(|record| record.expires_ms > now_ms) + } + + /// A peer's record while its lease holds, from the last listing when it + /// was there: routing asks on every request, and a GET per request was + /// most of a routed request's time. + pub async fn live(&self, node: &str, now_ms: u64) -> Result> { + if let Some(record) = self.listed.borrow().get(node) + && record.expires_ms > now_ms + { + return Ok(Some(record.clone())); + } + self.fetch(node, now_ms).await + } + + async fn fetch(&self, node: &str, now_ms: u64) -> Result> { + let Some((body, _)) = self.bucket.get(&node_key(node)).await? else { + return Ok(None); + }; + let record: NodeRecord = serde_json::from_slice(&body)?; + if record.expires_ms + DEAD_RECORD_TTLS * self.ttl_ms < now_ms { + // A fenced or crashed instance leaves its record, every restart + // adds one, and every listing read them all: 40 after an + // afternoon of fleet tests. One this long expired belongs to a + // process that fenced long ago; were it still running, its next + // renewal's conditional write would fail and fence it. + self.bucket.delete(&node_key(node)).await?; + } + if record.expires_ms <= now_ms { + return Ok(None); + } + // Until the next listing: one read serves every request that finds + // the listing stale, and a stale listing is when the bucket is busy. + self.listed + .borrow_mut() + .insert(node.to_owned(), record.clone()); + Ok(Some(record)) + } + + /// Every live instance, this one included when its lease holds. + pub async fn peers(&self, now_ms: u64) -> Result> { + // The records are read concurrently: a lease round is bounded, and + // reading them one after another made it time out on R2's spikes. + let listed = self.bucket.list("nodes/").await?; + let nodes = listed.iter().filter_map(|entry| { + entry + .key + .strip_prefix("nodes/") + .and_then(|key| key.strip_suffix(".json")) + }); + let fetched = futures::future::join_all(nodes.map(|node| self.fetch(node, now_ms))).await; + let mut peers = Vec::new(); + for record in fetched { + peers.extend(record?); + } + *self.listed.borrow_mut() = peers.iter().map(|p| (p.node.clone(), p.clone())).collect(); + Ok(peers) + } +} + +/// How many lease lifetimes past its expiry a node record is deleted. +const DEAD_RECORD_TTLS: u64 = 10; + +#[cfg(test)] +mod tests { + #![allow(clippy::unwrap_used)] + + use super::*; + + fn block_on(f: F) -> F::Output { + tokio::runtime::Builder::new_current_thread() + .build() + .unwrap() + .block_on(f) + } + + #[test] + fn a_renewal_whose_answer_was_lost_is_not_a_fence() { + block_on(async { + let bucket = Bucket::memory(""); + let a = Registry::new(bucket.clone(), "a", "a:1", 10_000); + a.renew(0).await.unwrap(); + // The renewal at 1_000 lands, but its answer never arrives. + let landed = NodeRecord { + node: "a".into(), + address: "a:1".into(), + expires_ms: 11_000, + instance: a.instance, + }; + let body = serde_json::to_vec(&landed).unwrap(); + bucket.put(&node_key("a"), body, &[]).await.unwrap(); + assert_eq!(a.renew(2_000).await.unwrap(), 12_000); + // Someone else's write is still a fence. + let other = NodeRecord { + instance: a.instance + 1, + ..landed + }; + let body = serde_json::to_vec(&other).unwrap(); + bucket.put(&node_key("a"), body, &[]).await.unwrap(); + assert!(matches!(a.renew(3_000).await, Err(Error::Fenced(_)))); + }); + } + + /// A write's key and body. + type Write = (String, Vec); + + /// Lands `late`, a write of ours a timed-out round left in flight, just + /// after serving the next GET. + #[derive(Default)] + struct Late { + inner: crate::bucket::memory::Memory, + late: std::rc::Rc>>, + } + + #[async_trait::async_trait(?Send)] + impl crate::bucket::Store for Late { + async fn get( + &self, + key: &str, + to: Option<&std::path::Path>, + ) -> Result, Version)>> { + let got = self.inner.get(key, to).await; + if let Some((key, body)) = self.late.take() { + let body = crate::bucket::Payload::Bytes(body.into()); + self.inner.put(&key, body, None, &[]).await?; + } + got + } + async fn put( + &self, + key: &str, + body: crate::bucket::Payload, + guard: Option, + metadata: &[(&str, String)], + ) -> Result> { + self.inner.put(key, body, guard, metadata).await + } + async fn list(&self, prefix: &str) -> Result> { + self.inner.list(prefix).await + } + async fn delete(&self, key: &str) -> Result<()> { + self.inner.delete(key).await + } + } + + #[test] + fn two_late_renewals_of_ours_are_not_a_fence() { + block_on(async { + let store = Late::default(); + let late = store.late.clone(); + let bucket = Bucket::new(store, ""); + let a = Registry::new(bucket.clone(), "a", "a:1", 10_000); + a.renew(0).await.unwrap(); + let ours = |expires_ms| { + let record = NodeRecord { + node: "a".into(), + address: "a:1".into(), + expires_ms, + instance: a.instance, + }; + serde_json::to_vec(&record).unwrap() + }; + // Two timed-out rounds landed: one before this renewal's first + // read, one just after it. + bucket.put(&node_key("a"), ours(11_000), &[]).await.unwrap(); + late.replace(Some((node_key("a"), ours(12_000)))); + assert_eq!(a.renew(3_000).await.unwrap(), 13_000); + }); + } + + #[test] + fn a_listing_deletes_records_long_expired() { + block_on(async { + let bucket = Bucket::memory(""); + let a = Registry::new(bucket.clone(), "a", "a:1", 10_000); + let b = Registry::new(bucket.clone(), "b", "b:1", 10_000); + let c = Registry::new(bucket.clone(), "c", "c:1", 10_000); + a.renew(0).await.unwrap(); + b.renew(90_000).await.unwrap(); + c.renew(115_000).await.unwrap(); + // a expired at 10s, b at 100s: at 115s only a is ten lives dead. + let live = c.peers(115_000).await.unwrap(); + assert_eq!(live.iter().map(|p| &*p.node).collect::>(), ["c"]); + let keys: Vec<_> = bucket.list("nodes/").await.unwrap(); + let keys: Vec<_> = keys.iter().map(|e| &*e.key).collect(); + assert_eq!(keys, ["nodes/b.json", "nodes/c.json"]); + }); + } +} diff --git a/src/workerd/server/cluster/s3.rs b/src/workerd/server/cluster/s3.rs new file mode 100644 index 00000000000..f062979e6ff --- /dev/null +++ b/src/workerd/server/cluster/s3.rs @@ -0,0 +1,552 @@ +// Copyright (c) 2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +//! The four S3 calls the cluster makes, signed with `SigV4`. Requests use +//! path-style URLs, which S3, R2 and `MinIO` all accept. + +use std::cell::Cell; +use std::cell::RefCell; +use std::fmt::Write; +use std::path::Path; +use std::path::PathBuf; +use std::time::Duration; +use std::time::SystemTime; +use std::time::UNIX_EPOCH; + +use crate::Error; +use crate::Result; +use crate::bucket::Entry; +use crate::bucket::Guard; +use crate::bucket::Store; +use crate::bucket::Version; +use crate::hex; +use crate::hmac_sha256; +use crate::http::Http; +use crate::http::HttpResponse; +use crate::http::Payload; +use crate::sha256; + +pub struct S3 { + http: Http, + endpoint: String, + bucket: String, + region: String, + credentials: RefCell, + /// `aws_credentials_file`, reread when its modification time changes. + file: Option, + /// This node's wall clock less the bucket's, at the last response. + skew_ms: Cell>, +} + +#[derive(Clone, Default)] +struct Credentials { + key_id: String, + secret: String, + token: Option, + modified: Option, +} + +impl Credentials { + /// `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY` and `AWS_SESSION_TOKEN` + /// lines, as an env file writes them. + fn parse(text: &str, modified: Option) -> Option { + let mut out = Self { + modified, + ..Self::default() + }; + for line in text.lines() { + let line = line.trim().trim_start_matches("export "); + let Some((name, value)) = line.split_once('=') else { + continue; + }; + let value = value.trim().trim_matches(['\'', '"']).to_owned(); + match name.trim() { + "AWS_ACCESS_KEY_ID" => out.key_id = value, + "AWS_SECRET_ACCESS_KEY" => out.secret = value, + "AWS_SESSION_TOKEN" => out.token = Some(value), + _ => {} + } + } + (!out.key_id.is_empty() && !out.secret.is_empty()).then_some(out) + } +} + +impl S3 { + /// `url` is `s3:///`; answers the client and the prefix. + /// Options are `aws_endpoint`, `aws_region`, `aws_access_key_id`, + /// `aws_secret_access_key` and `aws_token`, or in place of the last three + /// `aws_credentials_file`, so that rotating short-lived credentials + /// needs no restart. Each defaults to its AWS environment variable + /// (`AWS_ENDPOINT_URL`, `AWS_SESSION_TOKEN`, else upper-cased). + pub fn new(url: &str, options: &[(String, String)]) -> Result<(Self, String)> { + let rest = url + .strip_prefix("s3://") + .ok_or_else(|| Error::Config(format!("{url}: expected s3:///")))?; + let (bucket, prefix) = rest.split_once('/').unwrap_or((rest, "")); + // An option missing from the config falls back to the environment + // variable the AWS tools read. + let option = |name: &str| { + let var = match name { + "aws_endpoint" => "AWS_ENDPOINT_URL".to_owned(), + "aws_token" => "AWS_SESSION_TOKEN".to_owned(), + _ => name.to_uppercase(), + }; + options + .iter() + .find(|(key, _)| key == name) + .map(|(_, value)| value.clone()) + .or_else(|| std::env::var(var).ok()) + }; + let required = |name: &str| option(name).ok_or_else(|| Error::Config(name.to_owned())); + let region = option("aws_region").unwrap_or_else(|| "us-east-1".into()); + let endpoint = option("aws_endpoint") + .unwrap_or_else(|| format!("https://s3.{region}.amazonaws.com")) + .trim_end_matches('/') + .to_owned(); + let file = option("aws_credentials_file").map(PathBuf::from); + let credentials = match &file { + Some(path) => read_credentials(path) + .ok_or_else(|| Error::Config(format!("{}: no credentials", path.display())))?, + None => Credentials { + key_id: required("aws_access_key_id")?, + secret: required("aws_secret_access_key")?, + token: option("aws_token"), + modified: None, + }, + }; + Ok(( + Self { + http: Http::new().map_err(|e| Error::Config(e.to_string()))?, + endpoint, + bucket: bucket.to_owned(), + region, + credentials: RefCell::new(credentials), + file, + skew_ms: Cell::new(None), + }, + prefix.to_owned(), + )) + } +} + +#[async_trait::async_trait(?Send)] +impl Store for S3 { + async fn get(&self, key: &str, to: Option<&Path>) -> Result, Version)>> { + let response = self.send("GET", key, &[], Vec::new(), none(), to).await?; + match response.status { + 200 => Ok(Some((response.body, Version(response.etag)))), + 404 => Ok(None), + _ => Err(failure("GET", key, &response)), + } + } + + /// `Ok(None)` when the guard failed and nothing was written. + async fn put( + &self, + key: &str, + body: Payload, + guard: Option, + metadata: &[(&str, String)], + ) -> Result> { + let mut headers: Vec<(String, String)> = metadata + .iter() + .map(|(name, value)| (format!("x-amz-meta-{name}"), value.clone())) + .collect(); + match guard { + None => {} + Some(Guard::Absent) => headers.push(("if-none-match".into(), "*".into())), + Some(Guard::Match(Version(etag))) => headers.push(("if-match".into(), etag)), + } + let response = self.send("PUT", key, &[], headers, body, None).await?; + match response.status { + 200 => Ok(Some(Version(response.etag))), + // 412: the guard failed. 409: a concurrent conditional write to + // the same key won the race, which is the same answer. + 409 | 412 => Ok(None), + _ => Err(failure("PUT", key, &response)), + } + } + + async fn delete(&self, key: &str) -> Result<()> { + let response = self + .send("DELETE", key, &[], Vec::new(), none(), None) + .await?; + match response.status { + 200 | 204 | 404 => Ok(()), + _ => Err(failure("DELETE", key, &response)), + } + } + + async fn list(&self, prefix: &str) -> Result> { + let mut entries = Vec::new(); + let mut token: Option = None; + loop { + let mut query = vec![("list-type", "2".to_owned()), ("prefix", prefix.to_owned())]; + if let Some(token) = token.take() { + query.push(("continuation-token", token)); + } + let response = self + .send("GET", "", &query, Vec::new(), none(), None) + .await?; + if response.status != 200 { + return Err(failure("LIST", prefix, &response)); + } + let xml = String::from_utf8_lossy(&response.body); + for contents in elements(&xml, "Contents") { + let key = elements(contents, "Key").next().map(unescape); + entries.extend(key.map(|key| Entry { key })); + } + match elements(&xml, "NextContinuationToken").next() { + Some(next) if elements(&xml, "IsTruncated").next() == Some("true") => { + token = Some(unescape(next)); + } + _ => return Ok(entries), + } + } + } + + /// How far this node's wall clock is outside the bucket's, from the last + /// response's `Date` header: 0 when the header fits the request's window. + fn clock_skew_ms(&self) -> Option { + self.skew_ms.get() + } +} + +impl S3 { + /// Sign and send one request. `key` is the object key, or empty for a + /// bucket-level call. + /// R2 answers 429 to more than about one write a second to one key, and + /// an owner record is written twice in a row when an object reclaims: + /// under a 10,000-object kill, four reclaims failed that way and dropped + /// their objects. A refused request did nothing, conditional or not, so + /// it is retried after a pause. + async fn send( + &self, + method: &str, + key: &str, + query: &[(&str, String)], + headers: Vec<(String, String)>, + body: Payload, + to: Option<&Path>, + ) -> Result { + let mut pause = Duration::from_millis(250); + loop { + let response = self + .send_once(method, key, query, headers.clone(), body.clone(), to) + .await?; + if !matches!(response.status, 429 | 503) || pause > Duration::from_secs(2) { + return Ok(response); + } + tokio::time::sleep(pause).await; + pause *= 2; + } + } + + async fn send_once( + &self, + method: &str, + key: &str, + query: &[(&str, String)], + mut headers: Vec<(String, String)>, + body: Payload, + to: Option<&Path>, + ) -> Result { + let path = format!("/{}/{}", encode(&self.bucket, false), encode(key, false)); + let mut query: Vec = query + .iter() + .map(|(name, value)| format!("{}={}", encode(name, true), encode(value, true))) + .collect(); + query.sort(); + let query = query.join("&"); + + let host = self + .endpoint + .split_once("://") + .map_or(self.endpoint.as_str(), |(_, host)| host); + self.refresh(); + let credentials = self.credentials.borrow().clone(); + let (date, time) = amz_date(SystemTime::now()); + // The payload is not hashed: TLS protects it, and hashing would mean + // a second pass over every LTX file. + let mut signed = vec![ + ("host".to_owned(), host.to_owned()), + ( + "x-amz-content-sha256".to_owned(), + "UNSIGNED-PAYLOAD".to_owned(), + ), + ("x-amz-date".to_owned(), time.clone()), + ]; + if let Some(token) = &credentials.token { + signed.push(("x-amz-security-token".to_owned(), token.clone())); + } + signed.sort(); + let names = signed + .iter() + .map(|(name, _)| name.as_str()) + .collect::>() + .join(";"); + let canonical = format!( + "{method}\n{path}\n{query}\n{}\n{names}\nUNSIGNED-PAYLOAD", + signed.iter().fold(String::new(), |out, (name, value)| out + + name + + ":" + + value + + "\n"), + ); + let scope = format!("{date}/{}/s3/aws4_request", self.region); + let to_sign = format!( + "AWS4-HMAC-SHA256\n{time}\n{scope}\n{}", + hex(&sha256(canonical.as_bytes())) + ); + let mut signing = format!("AWS4{}", credentials.secret).into_bytes(); + for part in [date.as_str(), &self.region, "s3", "aws4_request"] { + signing = hmac_sha256(&signing, part.as_bytes()); + } + let signature = hex(&hmac_sha256(&signing, to_sign.as_bytes())); + headers.extend(signed); + headers.push(( + "authorization".into(), + format!( + "AWS4-HMAC-SHA256 Credential={}/{scope}, SignedHeaders={names}, Signature={signature}", + credentials.key_id + ), + )); + + let url = if query.is_empty() { + format!("{}{path}", self.endpoint) + } else { + format!("{}{path}?{query}", self.endpoint) + }; + let headers = headers.into_iter().flat_map(<[String; 2]>::from).collect(); + // A hung request would stall whatever waits on it: a tiering pass + // waits for every object, so one could stop a node's tiering. A + // large body gets a second per MiB on top, at least. + let size = match &body { + Payload::Bytes(bytes) => bytes.len() as u64, + Payload::File(path) => std::fs::metadata(path).map_or(0, |m| m.len()), + }; + let head = REQUEST_TIMEOUT + Duration::from_secs(size >> 20); + let sent = SystemTime::now(); + let response = self + .http + .send( + method.into(), + url, + headers, + body, + to, + (head, REQUEST_TIMEOUT), + ) + .await + .map_err(|error| Error::Http(format!("{method} {key}: {error}")))?; + if let Some(date) = http_date_ms(&response.date) { + // The bucket stamped the header, in whole seconds, between the + // send and the answer: this clock is off only by what lies + // outside that window. Measured from the send alone, a write + // R2 held 8 s at its 5-minute mark read as a clock 8 s slow. + let (sent, received) = (unix_ms(sent), unix_ms(SystemTime::now())); + let skew = if received < date { + received - date + } else if sent > date + 1000 { + sent - date - 1000 + } else { + 0 + }; + self.skew_ms.set(Some(skew)); + } + Ok(response) + } + + /// Rereads the credentials file if it changed. A file caught mid-write + /// or emptied keeps the credentials in use. + fn refresh(&self) { + let Some(path) = &self.file else { + return; + }; + let modified = std::fs::metadata(path).and_then(|m| m.modified()).ok(); + if modified.is_none() || modified == self.credentials.borrow().modified { + return; + } + if let Some(credentials) = read_credentials(path) { + *self.credentials.borrow_mut() = credentials; + } + } +} + +fn read_credentials(path: &std::path::Path) -> Option { + let modified = std::fs::metadata(path).and_then(|m| m.modified()).ok(); + Credentials::parse(&std::fs::read_to_string(path).ok()?, modified) +} + +fn none() -> Payload { + Payload::Bytes(bytes::Bytes::new()) +} + +/// How long one bucket request may take. R2 answers in 0.1-1 s from the +/// lab, with spikes of a few seconds. +const REQUEST_TIMEOUT: Duration = Duration::from_secs(20); + +fn failure(call: &str, key: &str, response: &HttpResponse) -> Error { + let body = String::from_utf8_lossy(&response.body); + let code = elements(&body, "Code").next().unwrap_or(""); + Error::Http(format!("{call} {key}: {} {code}", response.status)) +} + +/// `SigV4` URI encoding: everything but unreserved characters, and `/` too +/// unless the value is a path. +fn encode(value: &str, slash: bool) -> String { + value.bytes().fold(String::new(), |mut out, byte| { + if byte.is_ascii_alphanumeric() || b"-_.~".contains(&byte) || (byte == b'/' && !slash) { + out.push(char::from(byte)); + } else { + let _ = write!(out, "%{byte:02X}"); + } + out + }) +} + +/// The text of each `…` element, in order. S3's listing XML has +/// no attributes on the elements read here and no nesting of equal names. +fn elements<'a>(xml: &'a str, name: &str) -> impl Iterator { + let open = format!("<{name}>"); + let close = format!(""); + let mut rest = xml; + std::iter::from_fn(move || { + let start = rest.find(&open)? + open.len(); + let end = start + rest[start..].find(&close)?; + let text = &rest[start..end]; + rest = &rest[end + close.len()..]; + Some(text) + }) +} + +fn unescape(text: &str) -> String { + text.replace("<", "<") + .replace(">", ">") + .replace(""", "\"") + .replace("'", "'") + .replace("&", "&") +} + +/// Days since 1970-01-01 to a civil date (Howard Hinnant's algorithm). +fn civil(days: i64) -> (i64, u32, u32) { + let z = days + 719_468; + let era = z.div_euclid(146_097); + let doe = z.rem_euclid(146_097); + let yoe = (doe - doe / 1460 + doe / 36_524 - doe / 146_096) / 365; + let doy = doe - (365 * yoe + yoe / 4 - yoe / 100); + let mp = (5 * doy + 2) / 153; + let day = u32::try_from(doy - (153 * mp + 2) / 5 + 1).unwrap_or(1); + let month = u32::try_from(if mp < 10 { mp + 3 } else { mp - 9 }).unwrap_or(1); + (yoe + era * 400 + i64::from(month <= 2), month, day) +} + +fn unix_ms(t: SystemTime) -> i64 { + i64::try_from(t.duration_since(UNIX_EPOCH).unwrap_or_default().as_millis()).unwrap_or(0) +} + +/// An RFC 7231 date, `Mon, 12 Oct 2026 23:10:55 GMT`, in ms since the epoch. +fn http_date_ms(date: &str) -> Option { + httpdate::parse_http_date(date).ok().map(unix_ms) +} + +/// `(YYYYMMDD, YYYYMMDDTHHMMSSZ)`. +fn amz_date(now: SystemTime) -> (String, String) { + let secs = + i64::try_from(now.duration_since(UNIX_EPOCH).unwrap_or_default().as_secs()).unwrap_or(0); + let (year, month, day) = civil(secs.div_euclid(86_400)); + let s = secs.rem_euclid(86_400); + let date = format!("{year:04}{month:02}{day:02}"); + let time = format!("{date}T{:02}{:02}{:02}Z", s / 3600, s % 3600 / 60, s % 60); + (date, time) +} + +/// Unix milliseconds as Go's `time.RFC3339Nano` prints them: trailing zeros +/// of the fraction dropped, and no fraction at all when it is zero. +pub fn format_rfc3339_nano(unix_millis: i64) -> String { + let millis = unix_millis.max(0); + let secs = millis / 1000; + let (year, month, day) = civil(secs / 86_400); + let s = secs % 86_400; + let mut out = format!( + "{year:04}-{month:02}-{day:02}T{:02}:{:02}:{:02}", + s / 3600, + s % 3600 / 60, + s % 60 + ); + let fraction = millis % 1000; + if fraction != 0 { + let digits = format!("{fraction:03}"); + out = out + "." + digits.trim_end_matches('0'); + } + out + "Z" +} + +#[cfg(test)] +mod tests { + #![allow(clippy::unwrap_used)] + + use super::*; + + #[test] + fn dates_round_trip() { + let t = UNIX_EPOCH + Duration::from_millis(1_790_348_414_500); + assert_eq!( + format_rfc3339_nano(1_790_348_414_500), + "2026-09-25T15:00:14.5Z" + ); + assert_eq!( + format_rfc3339_nano(1_790_348_414_000), + "2026-09-25T15:00:14Z" + ); + assert_eq!(amz_date(t).1, "20260925T150014Z"); + assert_eq!(civil(11_016), (2000, 2, 29)); + } + + #[test] + fn http_dates() { + assert_eq!( + http_date_ms("Mon, 12 Oct 2026 23:10:55 GMT"), + Some(1_791_846_655_000) + ); + assert_eq!( + http_date_ms("Tue, 29 Feb 2000 00:00:00 GMT"), + Some(951_782_400_000) + ); + assert_eq!(http_date_ms("nonsense"), None); + } + + #[test] + fn credentials_from_an_env_file() { + let c = Credentials::parse( + "# r2\nexport AWS_ACCESS_KEY_ID='k'\nAWS_SECRET_ACCESS_KEY=\"s\"\n\ + AWS_SESSION_TOKEN=t=\nS3_ENDPOINT=https://x\n", + None, + ) + .unwrap(); + assert_eq!( + (&*c.key_id, &*c.secret, c.token.as_deref()), + ("k", "s", Some("t=")) + ); + assert!(Credentials::parse("AWS_ACCESS_KEY_ID=k\n", None).is_none()); + } + + #[test] + fn listing_xml() { + let xml = "true\ + a&b3\ + t"; + let keys: Vec<_> = elements(xml, "Contents") + .map(|c| unescape(elements(c, "Key").next().unwrap())) + .collect(); + assert_eq!(keys, ["a&b"]); + assert_eq!(elements(xml, "NextContinuationToken").next(), Some("t")); + } + + #[test] + fn sigv4_encoding() { + assert_eq!(encode("ns/a b+c", false), "ns/a%20b%2Bc"); + assert_eq!(encode("ns/a", true), "ns%2Fa"); + } +} diff --git a/src/workerd/server/factory/worker-factory-actor.c++ b/src/workerd/server/factory/worker-factory-actor.c++ index 442989e6112..b60393e82e1 100644 --- a/src/workerd/server/factory/worker-factory-actor.c++ +++ b/src/workerd/server/factory/worker-factory-actor.c++ @@ -15,26 +15,176 @@ namespace workerd::server { // ======================================================================================= // ActorStorage +namespace { + +// The directory a clustered namespace's SQLite opens its files through. Opening fails once this +// instance's cluster lease has lapsed, and writes to a WAL are reported to the cluster, which cuts +// commits from them. It has no fd, so SQLite uses the KJ VFS, on every platform. +class ClusterDirectory final: public kj::Directory { + public: + ClusterDirectory(kj::Own inner, const ActorNamespaceHandle& actors) + : inner(kj::mv(inner)), + actors(actors) {} + + Metadata stat() const override { + return inner->stat(); + } + void sync() const override { + inner->sync(); + } + void datasync() const override { + inner->datasync(); + } + kj::Own cloneFsNode() const override { + return kj::heap(inner->clone(), actors); + } + kj::Array listNames() const override { + return inner->listNames(); + } + kj::Array listEntries() const override { + return inner->listEntries(); + } + bool exists(kj::PathPtr path) const override { + return inner->exists(path); + } + kj::Maybe tryLstat(kj::PathPtr path) const override { + return inner->tryLstat(path); + } + kj::Maybe> tryOpenFile(kj::PathPtr path) const override { + actors.check_open(); + return inner->tryOpenFile(path); + } + kj::Maybe> tryOpenSubdir(kj::PathPtr path) const override { + return inner->tryOpenSubdir(path); + } + kj::Maybe tryReadlink(kj::PathPtr path) const override { + return inner->tryReadlink(path); + } + kj::Maybe> tryOpenFile( + kj::PathPtr path, kj::WriteMode mode) const override { + actors.check_open(); + auto name = path.toString(); + return inner->tryOpenFile(path, mode) + .map([&](kj::Own&& file) -> kj::Own { + if (!name.endsWith(".sqlite-wal")) return kj::mv(file); + return kj::heap(kj::mv(file), kj::mv(name), actors); + }); + } + kj::Own> replaceFile(kj::PathPtr path, kj::WriteMode mode) const override { + return inner->replaceFile(path, mode); + } + kj::Own createTemporary() const override { + actors.check_open(); + return inner->createTemporary(); + } + kj::Maybe> tryAppendFile( + kj::PathPtr path, kj::WriteMode mode) const override { + return inner->tryAppendFile(path, mode); + } + kj::Maybe> tryOpenSubdir( + kj::PathPtr path, kj::WriteMode mode) const override { + return inner->tryOpenSubdir(path, mode); + } + kj::Own> replaceSubdir( + kj::PathPtr path, kj::WriteMode mode) const override { + return inner->replaceSubdir(path, mode); + } + bool trySymlink(kj::PathPtr linkpath, kj::StringPtr content, kj::WriteMode mode) const override { + return inner->trySymlink(linkpath, content, mode); + } + bool tryTransfer(kj::PathPtr toPath, + kj::WriteMode toMode, + const kj::Directory& fromDirectory, + kj::PathPtr fromPath, + kj::TransferMode mode) const override { + return inner->tryTransfer(toPath, toMode, fromDirectory, fromPath, mode); + } + bool tryRemove(kj::PathPtr path) const override { + return inner->tryRemove(path); + } + + private: + // A WAL file: each write, once it succeeds, goes to the cluster. + class WalFile final: public kj::File { + public: + WalFile(kj::Own inner, kj::String name, const ActorNamespaceHandle& actors) + : inner(kj::mv(inner)), + name(kj::mv(name)), + actors(actors) {} + + Metadata stat() const override { + return inner->stat(); + } + void sync() const override { + inner->sync(); + } + void datasync() const override { + inner->datasync(); + } + kj::Own cloneFsNode() const override { + return kj::heap(inner->clone(), kj::str(name), actors); + } + size_t read(uint64_t offset, kj::ArrayPtr buffer) const override { + return inner->read(offset, buffer); + } + kj::Array mmap(uint64_t offset, uint64_t size) const override { + return inner->mmap(offset, size); + } + kj::Array mmapPrivate(uint64_t offset, uint64_t size) const override { + return inner->mmapPrivate(offset, size); + } + void write(uint64_t offset, kj::ArrayPtr data) const override { + inner->write(offset, data); + actors.wal_write( + toRust(name), offset, ::rust::Slice(data.begin(), data.size())); + } + void zero(uint64_t offset, uint64_t size) const override { + inner->zero(offset, size); + } + void truncate(uint64_t size) const override { + inner->truncate(size); + } + kj::Own mmapWritable( + uint64_t offset, uint64_t size) const override { + return inner->mmapWritable(offset, size); + } + + private: + kj::Own inner; + kj::String name; + const ActorNamespaceHandle& actors; + }; + + kj::Own inner; + const ActorNamespaceHandle& actors; +}; + +} // namespace + ActorStorage::Impl::Impl(const WorkerFactory& factory, kj::Maybe> directoryParam, kj::String uniqueKey, - ::rust::Box actors) - : directory(kj::mv(directoryParam)), + ::rust::Box actorsParam) + : actors(kj::mv(actorsParam)), + directory(kj::mv(directoryParam)), vfsDirectory(directory.map([](kj::Own& d) { return d->clone(); }) .orDefault([&]() { return kj::newInMemoryDirectory(kj::systemPreciseCalendarClock()); })), - vfs(*vfsDirectory), + sqliteDirectory(actors->replicated() + ? kj::heap(vfsDirectory->clone(), *actors) + : vfsDirectory->clone()), + vfs(*sqliteDirectory), uniqueKey(kj::mv(uniqueKey)), alarmScheduler(kj::heap(kj::systemPreciseCalendarClock(), factory.getImpl().timer, vfs, kj::Path({"metadata.sqlite"}), - [uniqueKey = this->uniqueKey.asPtr(), actors = kj::mv(actors)]( - const ActorKey& actor) mutable -> kj::Own { + [this](const ActorKey& actor) -> kj::Own { // The id carries the persisted name so that `ctx.id.name` is set when the alarm runs. - auto id = ActorIdFactoryImpl(uniqueKey).idFromStringNamed( - kj::str(actor.actorId), actor.name.map([](kj::StringPtr n) { return kj::str(n); })); + auto id = ActorIdFactoryImpl(this->uniqueKey) + .idFromStringNamed(kj::str(actor.actorId), + actor.name.map([](kj::StringPtr n) { return kj::str(n); })); return actors->actor_for_alarm(kj::heap(kj::mv(id))); })) {} @@ -234,6 +384,9 @@ class ActorSqliteHooks final: public ActorSqlite::Hooks { kj::Promise scheduleRun( kj::Maybe newAlarmTime, kj::Promise priorTask) override { KJ_IF_SOME(scheduledTime, newAlarmTime) { + // TODO(cluster): an alarm must follow its object between instances. + JSG_REQUIRE( + !storage->actors->replicated(), Error, "Alarms are not yet supported in cluster mode."); storage->alarmScheduler->setAlarm(actor, scheduledTime); } else { storage->alarmScheduler->deleteAlarm(actor); @@ -261,10 +414,11 @@ class FacetAlarmHooks final: public ActorSqlite::Hooks { Worker::Actor::MakeActorCacheFunc makeActorCacheFunc(kj::Own storage, const ActorStorageSpec& spec, kj::String rootKey, - kj::Maybe actorName) { + kj::Maybe actorName, + kj::Maybe<::rust::Box> committer) { return [storage = kj::mv(storage), durable = spec.durable, facetId = spec.facet_id, - rootKey = kj::mv(rootKey), - actorName = kj::mv(actorName)](const ActorCache::SharedLru& sharedLru, + rootKey = kj::mv(rootKey), actorName = kj::mv(actorName), + committer = kj::mv(committer)](const ActorCache::SharedLru& sharedLru, OutputGate& outputGate, ActorCache::Hooks& hooks, SqliteObserver& sqliteObserver) mutable -> kj::Maybe> { if (!durable) return kj::none; @@ -290,12 +444,20 @@ Worker::Actor::MakeActorCacheFunc makeActorCacheFunc(kj::Own db->run("PRAGMA journal_mode=WAL;"); db->afterReset([storage = kj::addRef(*storage), rootKey = kj::str(rootKey), selfId]( SqliteDatabase& db) mutable { + storage->actors->database_reset(toRust(rootKey)); db.run("PRAGMA journal_mode=WAL;"); storage->deleteDescendantStorage(*KJ_ASSERT_NONNULL(storage->directory), rootKey, selfId); }); - return kj::heap(kj::mv(db), outputGate, - [](SpanParent) -> kj::Promise { return kj::READY_NOW; }, *sqliteHooks) - .attach(kj::mv(sqliteHooks)); + kj::Function(SpanParent)> commit = [](SpanParent) -> kj::Promise { + return kj::READY_NOW; + }; + KJ_IF_SOME(c, committer) { + commit = [c = kj::mv(c)]( + SpanParent) -> kj::Promise { return result_task_run(c->commit()); }; + committer = kj::none; + } + auto sqlite = kj::heap(kj::mv(db), outputGate, kj::mv(commit), *sqliteHooks); + return sqlite.attach(kj::mv(sqliteHooks)); }; } @@ -396,8 +558,10 @@ kj::Own worker_new_actor(const CompiledWorker& worker, KJ_IF_SOME(doId, actorId.tryGet>()) { actorName = doId->getName().map([](kj::StringPtr n) { return kj::str(n); }); } - auto makeActorCache = makeActorCacheFunc( - kj::addRef(storage.getImpl()), spec, kj::str(spec.root_key), kj::mv(actorName)); + kj::Maybe<::rust::Box> committer; + if (spec.replicated) committer = hooks->committer(); + auto makeActorCache = makeActorCacheFunc(kj::addRef(storage.getImpl()), spec, + kj::str(spec.root_key), kj::mv(actorName), kj::mv(committer)); auto makeStorage = [enableSql = spec.enable_sql](jsg::Lock& js, const Worker::Api& api, ActorCacheInterface& actorCache) -> jsg::Ref { @@ -480,6 +644,14 @@ void actor_reset_storage(const ActorHandle& actor) { } } +void actor_close_storage(const ActorHandle& actor) { + KJ_IF_SOME(cache, actor.getActor().getPersistent()) { + KJ_IF_SOME(db, cache.getSqliteDatabase()) { + db.close(); + } + } +} + kj::Maybe> actor_hibernation_manager(const ActorHandle& actor) { return actor.getActor().getHibernationManager().map( [](HibernationManager& m) { return kj::Rc(m.addRef()); }); diff --git a/src/workerd/server/factory/worker-factory-impl.h b/src/workerd/server/factory/worker-factory-impl.h index 00b4a28ed01..c94183b650f 100644 --- a/src/workerd/server/factory/worker-factory-impl.h +++ b/src/workerd/server/factory/worker-factory-impl.h @@ -215,8 +215,13 @@ class ActorStorage::Impl final: public kj::Refcounted { kj::String uniqueKey, ::rust::Box actors); + // Declared first: the VFS's hooks and the alarm scheduler call into it. + ::rust::Box actors; kj::Maybe> directory; kj::Own vfsDirectory; // the in-memory directory when there is none + // What SQLite opens files through: `vfsDirectory`, or in a cluster a wrapper that fences opens + // on the lease and reports WAL writes, which the cluster ignores for the scheduler's database. + kj::Own sqliteDirectory; SqliteDatabase::Vfs vfs; kj::String uniqueKey; kj::Own alarmScheduler; @@ -239,14 +244,24 @@ class ActorStorage::Impl final: public kj::Refcounted { class RpcClient::Impl final { public: - Impl(const WorkerFactory& factory, kj::Own connection) + // A connection and its RPC system, shared by the capabilities obtained over it. + struct Session: public kj::Refcounted { + explicit Session(kj::Own connection) + : connection(kj::mv(connection)), + rpcSystem(*this->connection) {} + kj::Own connection; + capnp::TwoPartyClient rpcSystem; + }; + + Impl(const WorkerFactory& factory, kj::Own session, rpc::WorkerdBootstrap::Client target) : factory(factory), - connection(kj::mv(connection)), - rpcSystem(*this->connection) {} + session(kj::mv(session)), + target(kj::mv(target)) {} const WorkerFactory& factory; - kj::Own connection; - capnp::TwoPartyClient rpcSystem; + kj::Own session; + // Where events go: the session's bootstrap, or an actor obtained over it. + rpc::WorkerdBootstrap::Client target; }; // The `WorkerdBootstrap` a peer sees over capnp: every event it starts goes to `service`. diff --git a/src/workerd/server/factory/worker-factory-rpc.c++ b/src/workerd/server/factory/worker-factory-rpc.c++ index feeeec6b7c1..5a989c0cbf7 100644 --- a/src/workerd/server/factory/worker-factory-rpc.c++ +++ b/src/workerd/server/factory/worker-factory-rpc.c++ @@ -178,15 +178,28 @@ RpcClient::Impl& RpcClient::getImpl() const { } kj::Own new_rpc_client(const WorkerFactory& factory, kj::Own stream) { - return kj::heap(kj::heap(factory, kj::mv(stream))); + auto session = kj::refcounted(kj::mv(stream)); + auto target = session->rpcSystem.bootstrap().castAs(); + return kj::heap(kj::heap(factory, kj::mv(session), kj::mv(target))); +} + +kj::Own rpc_client_debug_actor( + const RpcClient& session, ::rust::Str service, ::rust::Str className, ::rust::Str actorId) { + auto& impl = session.getImpl(); + auto peer = impl.session->rpcSystem.bootstrap().castAs(); + auto request = peer.getActorRequest(); + request.setService(kj::str(service)); + request.setEntrypoint(kj::str(className)); + request.setActorId(kj::str(actorId)); + return kj::heap(kj::heap( + impl.factory, kj::addRef(*impl.session), request.send().getActor())); } kj::Promise rpc_client_custom_event(const RpcClient& client, kj::Own event, kj::Maybe<::rust::Str> cfBlobJson) { auto& impl = client.getImpl(); - auto bootstrap = impl.rpcSystem.bootstrap().castAs(); - auto request = bootstrap.startEventRequest(capnp::MessageSize{4, 0}); + auto request = impl.target.startEventRequest(capnp::MessageSize{4, 0}); KJ_IF_SOME(cf, cfBlobJson) request.setCfBlobJson(kj::str(cf)); auto dispatcher = request.send().getDispatcher(); // Workerd-to-workerd RPC does not support `restore()`, so Frankenvalues need no handler. @@ -196,8 +209,20 @@ kj::Promise rpc_client_custom_event(const RpcCl co_return rust::worker::CustomEventResult{.outcome = rust::worker::toRustOutcome(result.outcome)}; } +kj::Own rpc_client_start_request( + const RpcClient& client, kj::Own metadata) { + auto& impl = client.getImpl(); + auto request = impl.target.startEventRequest(capnp::MessageSize{4, 0}); + KJ_IF_SOME(cf, metadata->cfBlobJson) request.setCfBlobJson(cf); + request.setFromPersistentStub(metadata->fromPersistentStub == Persistent::YES); + auto dispatcher = request.send().getDispatcher(); + auto& factory = impl.factory.getImpl(); + return kj::heap(factory.httpOverCapnpFactory, factory.byteStreamFactory, + getUnsupportedFrankenvalueHandler(), kj::mv(dispatcher)); +} + kj::Promise rpc_client_on_disconnect(const RpcClient& client) { - return client.getImpl().rpcSystem.onDisconnect(); + return client.getImpl().session->rpcSystem.onDisconnect(); } // ======================================================================================= diff --git a/src/workerd/server/factory/worker-factory.c++ b/src/workerd/server/factory/worker-factory.c++ index aab6ec55ff9..2c0a790105e 100644 --- a/src/workerd/server/factory/worker-factory.c++ +++ b/src/workerd/server/factory/worker-factory.c++ @@ -85,7 +85,9 @@ WorkerFactory::Impl::Impl(WorkerFactory& factory, timer, entropySource, headerTableBuilder, httpOverCapnpFactory, byteStreamFactory), headerTable(buildHeaderTable(headerTableBuilder, this->config)), memoryCacheProvider(kj::heap()), - channelTokenHandler(factory), + channelTokenHandler(factory, + this->config.hasCluster() ? kj::Maybe(this->config.getCluster().getKey()) + : kj::none), tasks(*this) { // The config's logging settings go on top of the caller's. auto& logging = this->options->loggingOptions; diff --git a/src/workerd/server/factory/worker-factory.h b/src/workerd/server/factory/worker-factory.h index e8b9574e867..7f7cb08396d 100644 --- a/src/workerd/server/factory/worker-factory.h +++ b/src/workerd/server/factory/worker-factory.h @@ -54,6 +54,8 @@ struct SpawnedTask; struct ActorClassChannel; struct ActorNamespaceHandle; struct ActorHooks; +struct Committer; +struct ResultTask; struct AbortReason; struct KeepAlive; struct WorkerInterfaceList; @@ -384,11 +386,17 @@ class RpcClient final { mutable kj::Own impl; }; +void actor_close_storage(const ActorHandle& actor); + kj::Own new_rpc_client(const WorkerFactory& factory, kj::Own stream); +kj::Own rpc_client_debug_actor( + const RpcClient& session, ::rust::Str service, ::rust::Str className, ::rust::Str actorId); kj::Promise rpc_client_custom_event(const RpcClient& client, kj::Own event, kj::Maybe<::rust::Str> cfBlobJson); kj::Promise rpc_client_on_disconnect(const RpcClient& client); +kj::Own rpc_client_start_request( + const RpcClient& client, kj::Own metadata); uint16_t factory_start_inspector(const WorkerFactory& factory, ::rust::Str address); diff --git a/src/workerd/server/server-test.c++ b/src/workerd/server/server-test.c++ index d143fd5bae6..c4a45f41b9f 100644 --- a/src/workerd/server/server-test.c++ +++ b/src/workerd/server/server-test.c++ @@ -1075,6 +1075,35 @@ KJ_TEST("Server: wrapped bindings work under the new module registry") { conn.httpGet200("/", "wrapped: object"); } +kj::String clusterConfig(kj::StringPtr key) { + return kj::str(R"(( + services = [( + name = "hello", + worker = ( + compatibilityDate = "2024-01-01", + modules = [(name = "main.js", esModule = "export default {}")] + ) + )], + cluster = (bucket = "s3://bucket", address = "127.0.0.1:0", key = ")", + key, R"("), + ))"); +} + +KJ_TEST("Server: cluster mode requires --experimental") { + TestServer test(clusterConfig("0123456789abcdef0123456789abcdef")); + test.expectErrors(R"( + Cluster mode is experimental; it requires --experimental. + )"_blockquote); +} + +KJ_TEST("Server: a short cluster key is refused") { + TestServer test(clusterConfig("0123456789abcdef")); + test.server.allowExperimental(); + test.expectErrors(R"( + cluster.key must be at least 32 bytes. + )"_blockquote); +} + KJ_TEST("Server: duplicate module names are reported as config errors (legacy registry)") { TestServer test(singleWorker(R"(( compatibilityDate = "2024-10-01", diff --git a/src/workerd/server/server/BUILD.bazel b/src/workerd/server/server/BUILD.bazel index 7de9a202fb2..6eaf5929d79 100644 --- a/src/workerd/server/server/BUILD.bazel +++ b/src/workerd/server/server/BUILD.bazel @@ -35,6 +35,8 @@ wd_rust_crate( "//src/workerd/io:bench_capnp_rust", "//src/workerd/server:compiled-bindings_capnp_rust", "//src/workerd/server:workerd_capnp_rust", + "//src/workerd/server/cluster", + "//src/workerd/server/cluster/logic:cluster-logic", "@crates_vendor//:base64", "@crates_vendor//:capnp", "@crates_vendor//:data-encoding", @@ -46,6 +48,7 @@ wd_rust_crate( "@crates_vendor//:ipnet", "@crates_vendor//:percent-encoding", "@crates_vendor//:rand", + "@crates_vendor//:ring", "@crates_vendor//:rustls", "@crates_vendor//:serde", "@crates_vendor//:serde_json", diff --git a/src/workerd/server/server/actor.rs b/src/workerd/server/server/actor.rs index afeb9774aa1..290998232f3 100644 --- a/src/workerd/server/server/actor.rs +++ b/src/workerd/server/server/actor.rs @@ -21,6 +21,7 @@ use std::cell::Cell; use std::cell::OnceCell; use std::cell::RefCell; +use std::collections::HashMap; use std::rc::Rc; use std::rc::Weak; use std::time::Duration; @@ -55,6 +56,11 @@ use crate::channels::SubrequestChannel; use crate::channels::TokenUsage; use crate::channels::WorkerInterface; use crate::channels::attach; +use crate::cluster::ClusterNode; +use crate::cluster::Committer; +use crate::cluster::Owned; +use crate::cluster::Owner; +use crate::cluster::RpcConnection; use crate::config::Factory; use crate::tasks::TaskHandle; @@ -76,6 +82,8 @@ const EVICT_TIMEOUT_MESSAGE: &str = pub struct ActorNamespace { this: Weak, factory: Rc, + /// The worker service the class belongs to, which a peer names to reach an actor. + service: String, class_name: String, config: ActorConfig, actor_class: Rc, @@ -91,6 +99,27 @@ pub struct ActorNamespace { actors: RefCell>>, /// The cleanup loop, started by the first request; dropped with the namespace. cleanup_task: RefCell>, + /// Set in cluster mode for a namespace on disk: its actors are owned by one instance of + /// the cluster at a time, and replicated. + cluster: OnceCell>, + /// The namespace's directory, once linked on disk. + directory: OnceCell, + /// Actors being resolved in the cluster, each woken when its first resolution ends. + resolving: RefCell>>, +} + +/// The first resolution of an actor: its end, however it ends, wakes the requests that waited. +struct Resolving<'a> { + pending: &'a RefCell>>, + key: String, +} + +impl Drop for Resolving<'_> { + fn drop(&mut self) { + if let Some(done) = self.pending.borrow_mut().remove(&self.key) { + done.notify_waiters(); + } + } } impl ActorNamespace { @@ -101,6 +130,7 @@ impl ActorNamespace { /// Workflow's props. `persistent_self_tokens` is that class's worker's flag. pub fn new( factory: Rc, + service: String, class_name: String, config: ActorConfig, actor_class: Rc, @@ -109,6 +139,7 @@ impl ActorNamespace { Rc::new_cyclic(|this| Self { this: Weak::clone(this), factory, + service, class_name, config, actor_class, @@ -116,9 +147,89 @@ impl ActorNamespace { storage: OnceCell::new(), actors: RefCell::new(LinkedHashMap::new()), cleanup_task: RefCell::new(None), + cluster: OnceCell::new(), + directory: OnceCell::new(), + resolving: RefCell::new(HashMap::new()), }) } + /// Puts a durable namespace on disk under the cluster's ownership; before `link`. + pub fn set_cluster(&self, cluster: Rc) { + let _ = self.cluster.set(cluster); + } + + /// The cluster, for a namespace linked on disk; in-memory namespaces stay local. + pub(crate) fn cluster(&self) -> Option<&Rc> { + self.directory.get().and_then(|_| self.cluster.get()) + } + + /// The database of root actor `key`. + fn sqlite_path(&self, key: &str) -> Result { + let directory = self + .directory + .get() + .ok_or_else(|| kj::failed!("the namespace has no directory"))?; + Ok(format!("{directory}/{key}.sqlite")) + } + + /// The owner of the actor with `id`: a container here, holding its ownership, or a session + /// with the instance that owns it. + async fn resolve(&self, id: KjOwn) -> Result { + let key = ffi::actor_id_key(&id); + if let Some(container) = self.owned_container(&key) { + return Ok(Resolved::Here(Client(container))); + } + let cluster = self + .cluster() + .ok_or_else(|| kj::failed!("not in a cluster"))?; + // One cold resolution per actor: the rest wait for it, then find its container or its + // cached route, rather than each reading and claiming the owner record in the bucket. + let pending = self.resolving.borrow().get(&key).cloned(); + let _first = if let Some(done) = pending { + done.notified().await; + if let Some(container) = self.owned_container(&key) { + return Ok(Resolved::Here(Client(container))); + } + None + } else { + let done = Rc::new(Notify::new()); + self.resolving.borrow_mut().insert(key.clone(), done); + Some(Resolving { + pending: &self.resolving, + key: key.clone(), + }) + }; + let unique_key = self.unique_key().unwrap_or_default().to_owned(); + match cluster + .route(&unique_key, &self.service, &self.class_name, &key) + .await? + { + Owner::Remote(connection) => Ok(Resolved::Remote(connection)), + Owner::Here(epoch) => { + // A concurrent request may have taken it meanwhile. + if let Some(container) = self.owned_container(&key) { + return Ok(Resolved::Here(Client(container))); + } + let container = ActorContainer::new_root(key.clone(), self, id); + let owned = Owned::new(Rc::clone(cluster), &unique_key, &key, epoch); + *container.owned.borrow_mut() = Some(owned); + let replaced = self.actors.borrow_mut().insert(key, Rc::clone(&container)); + drop(replaced); + Ok(Resolved::Here(Client(container))) + } + } + } + + fn owned_container(&self, key: &str) -> Option> { + let actors = self.actors.borrow(); + let container = actors.get(key)?; + container + .owned + .borrow() + .is_some() + .then(|| Rc::clone(container)) + } + fn storage(&self) -> Result<&ffi::ActorStorage> { self.storage.get().map(|storage| &**storage).ok_or_else(|| { kj::failed!( @@ -198,6 +309,14 @@ impl ActorNamespace { /// The channel of the actor with `id`, started on first use. `persistent` is recorded on the /// channel: whether stubs to it may be stored. pub fn channel(&self, id: KjOwn, persistent: Persistent) -> Rc { + if self.cluster().is_some() { + return Rc::new(RoutedChannel { + namespace: Weak::clone(&self.this), + id, + persistent, + bound: Rc::default(), + }); + } Rc::new(ActorChannel { container: Client(self.container_for(id)), persistent, @@ -277,6 +396,13 @@ impl ActorNamespace { } ActorConfig::Ephemeral { .. } => ("", ""), }; + if !path.is_empty() { + let directory = format!("{path}/{unique_key}"); + if self.cluster.get().is_some() { + claim_directory(&directory)?; + } + let _ = self.directory.set(directory); + } let storage = ffi::factory_new_actor_storage( self.factory.raw(), path, @@ -313,6 +439,32 @@ impl ActorNamespace { } } +/// Marks a namespace's `directory` as the cluster's. The cluster replaces an +/// object's database with the bucket's copy, so a directory holding Durable +/// Objects from before cluster mode is refused rather than deleted. +fn claim_directory(directory: &str) -> Result<()> { + let directory = std::path::Path::new(directory); + let marker = directory.join(".cluster"); + if marker.exists() { + return Ok(()); + } + let databases = std::fs::read_dir(directory).into_iter().flatten().flatten(); + let held = databases.map(|e| e.file_name()).any(|name| { + let name = name.to_string_lossy(); + name.ends_with(".sqlite") && name != "metadata.sqlite" + }); + if held { + return Err(kj::failed!( + "{} holds Durable Objects from before cluster mode, which the cluster would \ + replace: move them away", + directory.display() + )); + } + std::fs::create_dir_all(directory) + .and_then(|()| std::fs::write(&marker, "")) + .map_err(|e| kj::failed!("{}: {e}", marker.display())) +} + impl ActorNamespaceHandle { pub(crate) fn actor_for_alarm( &self, @@ -326,6 +478,118 @@ impl ActorNamespaceHandle { .container_for(id) .start_request(ffi::new_request_metadata(KjMaybe::None, KjMaybe::None)) } + + fn namespace(&self) -> Option> { + self.0.upgrade() + } + + pub(crate) fn replicated(&self) -> bool { + self.namespace().is_some_and(|ns| ns.cluster().is_some()) + } + + pub(crate) fn check_open(&self) -> Result<()> { + match self.namespace().as_ref().and_then(|ns| ns.cluster()) { + Some(node) if !node.cluster.lease_holds() => Err(kj::failed!( + "this instance's cluster lease in the bucket has lapsed" + )), + _ => Ok(()), + } + } + + pub(crate) fn database_reset(&self, root_key: &str) { + if let Some(ns) = self.namespace() + && let (Some(node), Some(unique_key)) = (ns.cluster(), ns.unique_key()) + { + node.cluster.database_reset(unique_key, root_key); + } + } + + pub(crate) fn wal_write(&self, file: &str, offset: u64, data: &[u8]) { + if let Some(ns) = self.namespace() + && let (Some(node), Some(unique_key)) = (ns.cluster(), ns.unique_key()) + { + node.cluster.wal_write(unique_key, file, offset, data); + } + } +} + +/// Where a request to a clustered actor goes. +enum Resolved { + Here(Client), + Remote(Rc), +} + +/// A stub's channel to an actor in a clustered namespace. Its first request finds the owner; +/// the stub stays with that instance, as a local stub stays with its container. +struct RoutedChannel { + namespace: Weak, + id: KjOwn, + persistent: Persistent, + bound: Rc>>>, +} + +impl Channel for RoutedChannel { + fn start_request( + &self, + mut metadata: KjOwn, + ) -> Result> { + ffi::request_metadata_set_from_persistent_stub(metadata.as_mut(), self.persistent); + let namespace = self + .namespace + .upgrade() + .ok_or_else(|| kj::disconnected!("the Durable Object namespace is gone"))?; + let id = ffi::actor_id_clone(&self.id); + let bound = Rc::clone(&self.bound); + Ok(PromisedInterface::new(async move { + let resolved = bound.borrow().clone(); + let resolved = if let Some(resolved) = resolved { + resolved + } else { + let resolved = Rc::new(namespace.resolve(id).await?); + Rc::clone(bound.borrow_mut().get_or_insert(resolved)) + }; + match &*resolved { + Resolved::Here(container) => container.start_request(metadata), + Resolved::Remote(connection) => Ok(attach( + connection.start_request(metadata), + Rc::clone(connection), + )), + } + }) + .into_kj()) + } + + fn token(&self, usage: TokenUsage) -> Result> { + let namespace = self + .namespace + .upgrade() + .ok_or_else(|| kj::disconnected!("the Durable Object namespace is gone"))?; + let unique_key = namespace + .unique_key() + .ok_or_else(|| kj::failed!("only durable actors have channel tokens"))?; + Ok(ffi::factory_encode_actor_token( + namespace.factory.raw(), + unique_key, + &self.id, + self.persistent, + usage, + )?) + } + + fn evict_for_test(&self, hibernate: bool) -> LocalBoxFuture<'_, Result<()>> { + Box::pin(async move { + let namespace = self + .namespace + .upgrade() + .ok_or_else(|| kj::disconnected!("the Durable Object namespace is gone"))?; + match namespace.resolve(ffi::actor_id_clone(&self.id)).await? { + Resolved::Here(container) => container.evict_for_test(hibernate).await, + Resolved::Remote(_) => Err(kj::failed!( + "jsg.Error: Cannot evict a Durable Object another instance owns." + )), + } + }) + } } // ===================================================================================== @@ -377,6 +641,8 @@ pub struct ActorContainer { on_broken_task: RefCell>, /// The idle shutdown, armed when the last request ends and cancelled by the next. shutdown_task: RefCell>, + /// A clustered root's ownership, for as long as the container is in its namespace's map. + owned: RefCell>, } impl ActorContainer { @@ -439,6 +705,7 @@ impl ActorContainer { idle: Notify::new(), on_broken_task: RefCell::new(None), shutdown_task: RefCell::new(None), + owned: RefCell::new(None), } } } @@ -549,6 +816,7 @@ impl ActorContainer { let class = self.class()?; class.when_ready().await?; self.require_not_broken()?; + self.prepare_storage().await?; // A concurrent request may have started the actor meanwhile. if !self.is_running() { self.start(&class)?; @@ -560,6 +828,33 @@ impl ActorContainer { .ok_or_else(|| kj::failed!("the actor did not start")) } + /// In cluster mode, readies the actor's database before the actor opens it: it is taken over + /// (recovered from earlier epochs). An actor whose takeover fails leaves its namespace, so the + /// next request claims anew. + async fn prepare_storage(&self) -> Result<()> { + let ns = self.namespace()?; + if ns.cluster().is_none() { + return Ok(()); + } + let root_path = ns.sqlite_path(&self.root_key)?; + if self.is_root() { + let Some(restore) = self.owned.borrow().as_ref().map(|o| o.restore(root_path)) else { + return Err(kj::failed!("a clustered actor without ownership")); + }; + if let Err(e) = restore.await { + self.abort(Some(e.clone())); + ns.actors.borrow_mut().remove(&self.key); + return Err(e); + } + } else { + // TODO(cluster): facets, each a log of its own in the root's epoch. + return Err(kj::failed!( + "jsg.Error: Facets are not supported in cluster mode." + )); + } + Ok(()) + } + /// Constructs the actor and starts watching it for breakage. fn start(&self, class: &Rc) -> Result<()> { let ns = self.namespace()?; @@ -570,6 +865,7 @@ impl ActorContainer { enable_sql: ns.enable_sql(), root_key: self.root_key.clone(), facet_id: self.facet_id()?.into(), + replicated: ns.cluster().is_some(), }; let actor = Rc::new(class.new_actor(NewActor { id, @@ -619,6 +915,13 @@ impl Client { actor: &ffi::ActorHandle, ) -> Result> { let ns = self.namespace()?; + // A node stalled past its lease may have been taken over: serving + // even a read from its database could return stale data. + if ns.cluster().is_some_and(|node| !node.cluster.lease_holds()) { + return Err(kj::failed!( + "this instance's cluster lease in the bucket has lapsed" + )); + } let inner = { let state = self.class_and_id.borrow(); let ClassState::Ready { class, id } = &*state else { @@ -734,9 +1037,24 @@ impl ActorContainer { } } drop(manager); + self.release(actor.as_deref()); drop(actor); } + /// Closes a clustered actor's database and gives up its ownership, so nothing writes behind + /// the next owner. + fn release(&self, actor: Option<&KjOwn>) { + if let Some(actor) = actor + && self + .namespace + .upgrade() + .is_some_and(|ns| ns.cluster().is_some()) + { + ffi::actor_close_storage(actor); + } + drop(self.owned.borrow_mut().take()); + } + /// Aborts the actor and its facets; every request on them fails with `reason`. Without one /// the actors are shut down, failing nothing in flight, and later requests fail for an /// unknown reason. The caller removes the container from any map that could route traffic to @@ -759,6 +1077,7 @@ impl ActorContainer { *self.broken_reason.borrow_mut() = Some( reason.unwrap_or_else(|| kj::failed!("jsg.Error: Actor aborted for unknown reason.")), ); + self.release(actor.as_deref()); drop(actor); } @@ -918,9 +1237,11 @@ impl Drop for ActorContainer { for facet in self.facets.get_mut().values() { facet.abort(None); } - if let Some(actor) = self.actor.get_mut().take() { - ffi::actor_abort(&actor, &AbortReason(None)); + let actor = self.actor.get_mut().take(); + if let Some(actor) = &actor { + ffi::actor_abort(actor, &AbortReason(None)); } + self.release(actor.as_deref()); } } @@ -1027,6 +1348,13 @@ impl ActorContainer { } fn clone_facet(&self, src: &str, dst: &str) -> Result<()> { + let ns = self.namespace()?; + if ns.cluster().is_some() { + // TODO(cluster): clone in the bucket's terms. + return Err(kj::failed!( + "jsg.Error: Cloning facets is not supported in cluster mode." + )); + } // Replacing a facet implies aborting it. self.abort_facet(dst, kj::failed!("jsg.Error: Facet was cloned-over.")); if src == dst { @@ -1034,7 +1362,6 @@ impl ActorContainer { // abort matches `delete(dst)`, and the storage stays as it is. return Ok(()); } - let ns = self.namespace()?; ffi::actor_storage_clone_facet( ns.storage()?, &self.root_key, @@ -1047,6 +1374,23 @@ impl ActorContainer { } impl ActorHooks { + pub(crate) fn committer(&self) -> Result> { + // Only a clustered actor's storage asks; see `ActorStorageSpec::replicated`. + let container = self.container()?; + let ns = container + .namespace + .upgrade() + .ok_or_else(|| kj::disconnected!("shut down"))?; + let node = ns + .cluster() + .ok_or_else(|| kj::failed!("not in a cluster"))?; + Ok(Box::new(Committer { + cluster: Rc::clone(&node.cluster), + namespace: ns.unique_key().unwrap_or_default().to_owned(), + object: container.root_key.clone(), + })) + } + fn container(&self) -> Result> { self.0 .upgrade() diff --git a/src/workerd/server/server/bridge.rs b/src/workerd/server/server/bridge.rs index bf4eeddfcb4..4feaa6b4bca 100644 --- a/src/workerd/server/server/bridge.rs +++ b/src/workerd/server/server/bridge.rs @@ -25,6 +25,9 @@ pub use crate::channels::ServerHandle; pub use crate::channels::SubrequestChannel; pub use crate::channels::WorkerInterfaceList; pub use crate::channels::WorkerStub; +pub use crate::cluster::Committer; +pub use crate::cluster::ResultTask; +pub use crate::cluster::result_task_run; pub use crate::entry::PendingCommand; pub use crate::entry::run_pending_command; pub use crate::in_process::InProcessServer; @@ -164,6 +167,8 @@ pub mod ffi { root_key: String, /// A facet's number within its root actor's storage; none for a root actor. facet_id: KjMaybe, + /// The cluster replicates the actor's commits: each waits for `Committer::commit`. + replicated: bool, } /// What a channel token is for (`IoChannelFactory::ChannelTokenUsage`): restoring a stub over @@ -423,6 +428,8 @@ pub mod ffi { ) -> Result; /// Resets the actor's SQLite database while its connection is open (`deleteAllActors`). fn actor_reset_storage(actor: &ActorHandle); + /// Closes the actor's database, if it has one; later use of it throws. + fn actor_close_storage(actor: &ActorHandle); /// The actor's hibernation manager, to hand to the actor that replaces it after eviction. fn actor_hibernation_manager(actor: &ActorHandle) -> KjMaybe>; type HibernationManager; @@ -559,6 +566,15 @@ pub mod ffi { factory: &WorkerFactory, stream: KjOwn, ) -> KjOwn; + /// An actor on a cluster peer, by `WorkerdDebugPort.getActor` over `session`, a peer + /// session from `new_rpc_client`. Pipelined: no round trip until its first event. It + /// keeps the session alive. + fn rpc_client_debug_actor( + session: &RpcClient, + service: &str, + class_name: &str, + actor_id: &str, + ) -> KjOwn; /// Sends a custom event (a trace, a JS RPC session, a tail stream, a UDP flow) to the /// peer's `WorkerdBootstrap`. async fn rpc_client_custom_event( @@ -567,6 +583,12 @@ pub mod ffi { cf_blob_json: KjMaybe<&str>, ) -> Result; async fn rpc_client_on_disconnect(client: &RpcClient); + /// Starts an event on the peer's `WorkerdBootstrap`: HTTP, WebSockets and JS RPC all + /// cross as capnp. The client must outlive the returned interface. + fn rpc_client_start_request( + client: &RpcClient, + metadata: KjOwn, + ) -> KjOwn; /// Starts the inspector on `address` on its own thread; every isolate the factory /// creates afterwards registers with it. Returns the bound port. @@ -820,6 +842,15 @@ pub mod ffi { self: &ActorNamespaceHandle, id: KjOwn, ) -> Result>; + /// Whether the cluster replicates the namespace's actors. + fn replicated(self: &ActorNamespaceHandle) -> bool; + /// Called as SQLite opens one of the namespace's files: fails once this instance's + /// cluster lease has lapsed. + fn check_open(self: &ActorNamespaceHandle) -> Result<()>; + /// A write SQLite made to one of the namespace's WAL files. + fn wal_write(self: &ActorNamespaceHandle, file: &str, offset: u64, data: &[u8]); + /// The database of the root actor `root_key` was deleted. + fn database_reset(self: &ActorNamespaceHandle, root_key: &str); /// An actor's way back to the server: the requests it raises for itself, its facets /// (named child actors sharing its storage), and its transitions between idle and active. @@ -844,6 +875,15 @@ pub mod ffi { fn active(self: &ActorHooks); /// The actor's last request ended. fn inactive(self: &ActorHooks); + /// What makes the actor's commits durable, when the cluster replicates it. + fn committer(self: &ActorHooks) -> Result>; + + type Committer; + /// Makes the commits captured so far durable. + fn commit(self: &Committer) -> Box; + /// A Rust future C++ awaits with `result_task_run`. + type ResultTask; + async fn result_task_run(task: Box) -> Result<()>; /// The reason of an `actor_abort`. `raise` returns it as the `Err`, which C++ catches as /// the equivalent `kj::Exception`; `Ok` when there is none. diff --git a/src/workerd/server/server/cluster-test.rs b/src/workerd/server/server/cluster-test.rs new file mode 100644 index 00000000000..4e3c3897af2 --- /dev/null +++ b/src/workerd/server/server/cluster-test.rs @@ -0,0 +1,71 @@ +// Copyright (c) 2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +use std::cell::Cell; + +use futures::future::join_all; + +use super::*; + +fn runtime() -> tokio::runtime::Runtime { + tokio::runtime::Builder::new_current_thread() + .enable_all() + .build() + .unwrap() +} + +/// A dial that counts itself and, while `fail` is set, fails. +fn dial(dials: &Rc>, fail: bool) -> Dial { + let dials = Rc::clone(dials); + async move { + dials.set(dials.get() + 1); + tokio::task::yield_now().await; + if fail { + Err("refused".to_owned()) + } else { + Ok(dials.get()) + } + } + .boxed_local() +} + +#[test] +fn concurrent_callers_share_one_dial() { + let (sessions, dials) = (Sessions::default(), Rc::new(Cell::new(0))); + let gets = (0..3).map(|_| sessions.get("peer", || dial(&dials, false))); + let got = runtime().block_on(join_all(gets)); + assert_eq!(got, vec![Ok(1), Ok(1), Ok(1)]); + assert_eq!(dials.get(), 1); +} + +#[test] +fn a_failed_dial_is_forgotten() { + let (sessions, dials) = (Sessions::default(), Rc::new(Cell::new(0))); + let rt = runtime(); + let failed = rt.block_on(sessions.get("peer", || dial(&dials, true))); + assert_eq!(failed, Err("refused".to_owned())); + let retried = rt.block_on(sessions.get("peer", || dial(&dials, false))); + assert_eq!(retried, Ok(2)); +} + +#[test] +fn a_disconnected_session_is_forgotten() { + let (sessions, dials) = (Sessions::default(), Rc::new(Cell::new(0))); + let rt = runtime(); + assert_eq!( + rt.block_on(sessions.get("peer", || dial(&dials, false))), + Ok(1) + ); + // Another session's disconnect leaves this one. + sessions.forget("peer", |&s| s == 7); + assert_eq!( + rt.block_on(sessions.get("peer", || dial(&dials, false))), + Ok(1) + ); + sessions.forget("peer", |&s| s == 1); + assert_eq!( + rt.block_on(sessions.get("peer", || dial(&dials, false))), + Ok(2) + ); +} diff --git a/src/workerd/server/server/cluster.rs b/src/workerd/server/server/cluster.rs new file mode 100644 index 00000000000..88e40055a28 --- /dev/null +++ b/src/workerd/server/server/cluster.rs @@ -0,0 +1,542 @@ +// Copyright (c) 2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +//! Cluster mode (`Config.cluster`): this instance's membership in a cluster whose shared state +//! is a bucket (the `cluster` crate), and the port peers reach it on. +//! +//! The port speaks HTTP/1.1 through kj-hyper. `CONNECT`, its MAC in `cf-cluster-auth`, opens the +//! one capnp session an instance keeps to each peer: the tunnel carries `WorkerdDebugPort`, +//! whose `getActor` gives a `WorkerdBootstrap` for one Durable Object, so HTTP, WebSockets and +//! JS RPC to the object cross unchanged. A request for an object this instance +//! does not own routes on through its own channel to the owner. + +use std::cell::RefCell; +use std::collections::HashMap; +use std::future::Future; +use std::pin::Pin; +use std::rc::Rc; + +use cluster::Cluster; +use cluster::Route; +use futures::FutureExt; +use futures::TryFutureExt; +use futures::future::LocalBoxFuture; +use futures::future::Shared; +use kj::http::Headers; +use kj::http::HeadersRef; +use kj::http::Method; +use kj::http::ServiceResponse; +use kj_hyper::client::Dialed; +use kj_hyper::server::Connect; +use kj_hyper::server::Handler; +use kj_hyper::server::ServerSettings; +use kj_hyper::server::Shutdown; +use kj_rs::KjOwn; +use tokio::io::AsyncReadExt; +use tokio::io::AsyncWriteExt; +use tokio::net::TcpListener; +use tokio::net::TcpStream; +use workerd_capnp::cluster_config; + +use crate::Result; +use crate::bridge::ffi; +use crate::config::Factory; +use crate::services::header_table; +use crate::services::send_error; + +/// A session's CONNECT carries `..`: see `connect_mac`. +const AUTH_HEADER: &str = "cf-cluster-auth"; +/// How far a CONNECT's timestamp may be from this instance's clock. +const AUTH_WINDOW_MS: u64 = 30_000; +/// How long dialing a peer and its answer to the CONNECT may take. +const TUNNEL_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(5); + +/// This instance's cluster: the crate's state and the factory sessions to owners start from. +pub struct ClusterNode { + pub cluster: Rc, + factory: Rc, + /// One session per peer address, dropped when it disconnects. + sessions: Rc>>, + /// The address peers dial, which their CONNECTs are bound to. + advertise: String, + /// The nonces of CONNECTs accepted within twice the window, by timestamp. + nonces: RefCell>, +} + +pub struct RpcConnection(KjOwn); + +type Dial = LocalBoxFuture<'static, std::result::Result>; + +/// One value per key, a peer's session: the callers that arrive while it is +/// being made share the attempt, and a failed attempt is forgotten, so the +/// next caller makes another. +pub struct Sessions { + map: RefCell>>>, +} + +impl Default for Sessions { + fn default() -> Self { + Self { + map: RefCell::default(), + } + } +} + +impl Sessions { + /// `key`'s value: the one made, the one being made, or one `make` makes. + pub async fn get( + &self, + key: &str, + make: impl FnOnce() -> Dial, + ) -> std::result::Result { + let attempt = self.map.borrow().get(key).cloned(); + let attempt = attempt.unwrap_or_else(|| { + let attempt = make().shared(); + let mut map = self.map.borrow_mut(); + map.insert(key.to_owned(), attempt.clone()); + attempt + }); + let result = attempt.clone().await; + if result.is_err() { + let mut map = self.map.borrow_mut(); + if map.get(key).is_some_and(|a| a.ptr_eq(&attempt)) { + map.remove(key); + } + } + result + } + + /// Forgets `key`'s value if it is the one `is` matches. + pub fn forget(&self, key: &str, is: impl Fn(&V) -> bool) { + let mut map = self.map.borrow_mut(); + let made = map.get(key).and_then(|a| a.peek()); + if made.is_some_and(|r| r.as_ref().is_ok_and(&is)) { + map.remove(key); + } + } +} + +/// Binds the cluster port, before the services start. +/// Returns the listener and the address to advertise. +pub async fn bind( + conf: cluster_config::Reader<'_>, + address: Option<&str>, + advertise: Option<&str>, +) -> Result<(TcpListener, String)> { + let address = address.map_or_else(|| text(conf.get_address()), |a| Ok(a.to_owned()))?; + refuse_public(conf, &address)?; + let listener = TcpListener::bind(&address) + .await + .map_err(|e| kj::failed!("cluster address {address}: {e}"))?; + let bound = listener.local_addr().map_err(|e| kj::failed!("{e}"))?; + let advertise = match advertise { + Some(a) => a.to_owned(), + None if conf.has_advertise() => text(conf.get_advertise())?, + None if bound.ip().is_unspecified() => { + return Err(kj::failed!( + "cluster address {address} is a wildcard, which peers cannot reach: set \ + cluster.advertise or --cluster-advertise" + )); + } + None => bound.to_string(), + }; + refuse_public(conf, &advertise)?; + Ok((listener, advertise)) +} + +/// Fails for an `address` that is a public IP, unless the config allows it. +fn refuse_public(conf: cluster_config::Reader<'_>, address: &str) -> Result<()> { + let Ok(ip) = address.parse::().map(|a| a.ip()) else { + return Ok(()); + }; + if is_public(ip) && !conf.get_unsafe_public_address() { + return Err(kj::failed!( + "cluster address {ip} is public, and traffic between instances is not encrypted: \ + use a private network (a VPC, WireGuard, Tailscale), or set \ + cluster.unsafePublicAddress" + )); + } + Ok(()) +} + +/// Whether `ip` is on a network others route: not loopback, private, link-local, shared +/// (100.64.0.0/10, which Tailscale uses), or a wildcard, which this cannot judge. +fn is_public(ip: std::net::IpAddr) -> bool { + match ip { + std::net::IpAddr::V4(v4) => { + let [a, b, ..] = v4.octets(); + let shared = a == 100 && (64..128).contains(&b); + let local = v4.is_loopback() || v4.is_private() || v4.is_link_local(); + !(local || shared || v4.is_unspecified()) + } + std::net::IpAddr::V6(v6) => { + let local = v6.is_loopback() || v6.is_unique_local() || v6.is_unicast_link_local(); + !(local || v6.is_unspecified()) + } + } +} + +/// The MAC a CONNECT to `target` carries, at `ms` with `nonce`: a header seen on the wire is +/// refused at any other target, once used, or once stale. +fn connect_mac(cluster: &Cluster, target: &str, ms: u64, nonce: &str) -> Vec { + cluster.mac(format!("workerd cluster connect v1\n{target}\n{ms}\n{nonce}").as_bytes()) +} + +impl ClusterNode { + /// Joins the cluster: the crate's state for `conf`, this instance at `address`. + pub fn new( + factory: Rc, + conf: cluster_config::Reader<'_>, + address: String, + ) -> Result> { + let options = conf.get_bucket_options().map_err(|e| capnp(&e))?; + let setup = cluster::Setup { + url: text(conf.get_bucket())?, + options: options.iter().map(text).collect::>()?, + node: cluster::hex(&rand::random::<[u8; 16]>()), + address: address.clone(), + ttl_ms: cluster_logic::lease::LEASE_MS, + key: text(conf.get_key())?.into_bytes(), + }; + let cluster = Cluster::new(setup).map_err(|e| kj::failed!("cluster: {e}"))?; + Ok(Rc::new(Self { + cluster, + factory, + sessions: Rc::default(), + advertise: address, + nonces: RefCell::default(), + })) + } + + /// Where a request for Durable Object `actor` of `namespace` should run. + /// If this instance owns the object, or claims it now, the result is + /// `Owner::Here` with the ownership epoch. Otherwise it is + /// `Owner::Remote`: a stub, over the session to the instance that owns + /// the object, for the object as class `class` of worker `service`. + pub async fn route( + &self, + namespace: &str, + service: &str, + class: &str, + actor: &str, + ) -> Result { + let now = cluster::now_ms(); + let route = self + .cluster + .acquire_or_route(namespace, actor, now) + .await + .map_err(|e| kj::disconnected!("{namespace}/{actor}: {e}"))?; + let (node, address, stale) = match route { + Route::Owned(epoch) => return Ok(Owner::Here(epoch)), + Route::Remote { + node, + address, + stale, + } => (node, address, stale), + }; + if stale { + let cluster = Rc::clone(&self.cluster); + let (namespace, actor) = (namespace.to_owned(), actor.to_owned()); + self.factory.spawn_detached(async move { + let _ = cluster.refresh_route(&namespace, &actor, now).await; + }); + } + let session = self + .session(&address) + .await + .map_err(|e| kj::disconnected!("{namespace}/{actor} on {node}: {e}"))?; + // A channel per stub, so a stub stays with the instance it reached. + Ok(Owner::Remote(Rc::new(RpcConnection( + ffi::rpc_client_debug_actor(&session.0, service, class, actor), + )))) + } + + /// The session to the peer at `address`: the open one, the one being + /// dialed, or a new one. + async fn session(&self, address: &str) -> std::io::Result> { + let dial = || self.dial(address); + let session = self.sessions.get(address, dial).await; + session.map_err(std::io::Error::other) + } + + fn dial(&self, address: &str) -> Dial> { + let cluster = Rc::clone(&self.cluster); + let factory = Rc::clone(&self.factory); + let sessions = Rc::clone(&self.sessions); + let address = address.to_owned(); + async move { + // A stalled peer accepts the connection and never answers. + let tunnel = tokio::time::timeout(TUNNEL_TIMEOUT, tunnel(&cluster, &address)); + let stream = tunnel + .await + .map_err(|_| "no answer to CONNECT".to_owned())? + .map_err(|e| e.to_string())?; + let session = Rc::new(RpcConnection(ffi::new_rpc_client( + factory.raw(), + Dialed::from(stream).into_kj(), + ))); + let watched = Rc::clone(&session); + factory.spawn_detached(async move { + let _ = ffi::rpc_client_on_disconnect(&watched.0).await; + sessions.forget(&address, |s| Rc::ptr_eq(s, &watched)); + }); + Ok(session) + } + .boxed_local() + } +} + +async fn tunnel(node: &Cluster, address: &str) -> std::io::Result { + let mut stream = TcpStream::connect(address).await?; + stream.set_nodelay(true)?; + let ms = cluster::now_ms(); + let nonce = cluster::hex(&rand::random::<[u8; 16]>()); + let mac = cluster::hex(&connect_mac(node, address, ms, &nonce)); + let auth = format!("{AUTH_HEADER}: {ms}.{nonce}.{mac}"); + let request = format!("CONNECT cluster:1 HTTP/1.1\r\nhost: cluster:1\r\n{auth}\r\n\r\n"); + stream.write_all(request.as_bytes()).await?; + // The response head, read a byte at a time so that nothing past it is consumed. + let mut head = Vec::new(); + while !head.ends_with(b"\r\n\r\n") { + if head.len() > 4096 { + return Err(std::io::Error::other("response head too long")); + } + head.push(stream.read_u8().await?); + } + if !head.starts_with(b"HTTP/1.1 200") { + let status = String::from_utf8_lossy(head.split(|&b| b == b'\r').next().unwrap_or(&[])); + return Err(std::io::Error::other(format!("refused: {status}"))); + } + Ok(stream) +} + +/// Where a Durable Object runs, for a request about to start on it. +pub enum Owner { + Here(u64), + Remote(Rc), +} + +impl RpcConnection { + /// Starts a request on the remote owner; the caller keeps the session alive with it. + pub fn start_request( + &self, + metadata: KjOwn, + ) -> KjOwn { + ffi::rpc_client_start_request(&self.0, metadata) + } +} + +// ===================================================================================== +// The port + +/// Serves the cluster port until the server is dropped. +pub async fn serve(listener: TcpListener, node: Rc) { + let settings = Rc::new(ServerSettings::default()); + let shutdown = Rc::new(Shutdown::new()); + loop { + let Ok((stream, _)) = listener.accept().await else { + continue; + }; + let _ = stream.set_nodelay(true); + let hangup = kj_rs_io::when_write_disconnected(&stream) + .err_into() + .boxed(); + let handler: Rc = Rc::new(PeerConnection { + node: Rc::clone(&node), + }); + let (settings, shutdown) = (Rc::clone(&settings), Rc::clone(&shutdown)); + let factory = Rc::clone(&node.factory); + node.factory.spawn_detached(async move { + let table = header_table(&factory); + if let Err(e) = kj_hyper::server::serve_connection( + stream, hangup, table, settings, handler, &shutdown, + ) + .await + { + tracing::warn!("cluster connection failed: {}", e.description()); + } + }); + } +} + +struct PeerConnection { + node: Rc, +} + +impl PeerConnection { + /// Checks a CONNECT's `AUTH_HEADER`: its MAC, its clock, and that its nonce is new. + fn verify(&self, header: Option<&[u8]>) -> Result<()> { + let bad = || kj::failed!("cluster CONNECT without a valid {AUTH_HEADER}"); + let header = header + .and_then(|h| std::str::from_utf8(h).ok()) + .ok_or_else(bad)?; + let [ms, nonce, mac] = header.split('.').collect::>()[..] else { + return Err(bad()); + }; + let ms: u64 = ms.parse().map_err(|_| bad())?; + let mac = cluster::unhex(mac).ok_or_else(bad)?; + let node = &self.node; + if !constant_time_eq( + &connect_mac(&node.cluster, &node.advertise, ms, nonce), + &mac, + ) { + return Err(bad()); + } + let now = cluster::now_ms(); + if now.abs_diff(ms) > AUTH_WINDOW_MS { + return Err(kj::failed!( + "cluster CONNECT {}ms off this clock", + now.abs_diff(ms) + )); + } + let mut nonces = node.nonces.borrow_mut(); + nonces.retain(|_, t| now.abs_diff(*t) <= 2 * AUTH_WINDOW_MS); + if nonces.insert(nonce.to_owned(), ms).is_some() { + return Err(kj::failed!("cluster CONNECT replayed")); + } + Ok(()) + } +} + +#[async_trait::async_trait(?Send)] +impl Handler for PeerConnection { + async fn request<'a>( + &'a self, + _method: Method, + _url: &'a [u8], + _headers: HeadersRef<'a>, + _body: Pin<&'a mut kj::io::AsyncInputStream>, + response: ServiceResponse<'a>, + ) -> Result<()> { + let table = header_table(&self.node.factory); + send_error(response, 404, "Not Found", &Headers::new(table)).await + } + + async fn connect<'a>( + &'a self, + _host: &'a [u8], + headers: HeadersRef<'a>, + connect: Connect, + ) -> Result<()> { + let table = header_table(&self.node.factory); + let empty = Headers::new(table); + if let Err(e) = self.verify(headers.get_by_name(AUTH_HEADER)) { + tracing::warn!("cluster CONNECT refused: {}", e.description()); + connect.reject(403, "Forbidden", HeadersRef::from(&empty), Some(0))?; + return Ok(()); + } + let tunnel = connect.accept(200, "OK", HeadersRef::from(&empty))?; + // A peer holds the cluster key, so it gets the debug port's interface. + Ok(ffi::factory_accept_debug_port(self.node.factory.raw(), tunnel.into_kj()).await?) + } +} + +fn constant_time_eq(a: &[u8], b: &[u8]) -> bool { + a.len() == b.len() && a.iter().zip(b).fold(0, |acc, (x, y)| acc | (x ^ y)) == 0 +} + +fn text(reader: capnp::Result>) -> Result { + reader + .map_err(|e| capnp(&e))? + .to_string() + .map_err(|e| kj::failed!("{e}")) +} + +fn capnp(e: &capnp::Error) -> crate::Error { + kj::failed!("{e}") +} + +/// A Rust future C++ awaits (`result_task_run`). +pub struct ResultTask(pub LocalBoxFuture<'static, Result<()>>); + +pub async fn result_task_run(task: Box) -> Result<()> { + task.0.await +} + +/// What makes one actor's commits durable: `sync_actor` on its object. +pub struct Committer { + pub cluster: Rc, + pub namespace: String, + pub object: String, +} + +impl Committer { + pub fn commit(&self) -> Box { + let cluster = Rc::clone(&self.cluster); + let (namespace, object) = (self.namespace.clone(), self.object.clone()); + Box::new(ResultTask( + async move { + cluster + .sync_actor(&namespace, &object) + .await + .map_err(|e| kj::failed!("{namespace}/{object}: commit failed: {e}")) + } + .boxed_local(), + )) + } +} + +/// This instance's ownership of one Durable Object at an epoch. +/// +/// Its container holds it while the container is in its namespace's map. Dropping it stops capturing the object and keeps it +/// dormant here, uploading what the bucket lacks; the next request wakes it. +pub struct Owned { + node: Rc, + namespace: String, + actor: String, + epoch: u64, + /// The takeover, shared by the requests that wait for it. + restore: RefCell>>>>, +} + +impl Owned { + pub fn new(node: Rc, namespace: &str, actor: &str, epoch: u64) -> Self { + Self { + node, + namespace: namespace.to_owned(), + actor: actor.to_owned(), + epoch, + restore: RefCell::new(None), + } + } + + pub fn cluster(&self) -> &Rc { + &self.node.cluster + } + + /// Takes the object over into the database at `path`, once: recovers what earlier epochs + /// committed and opens this one. + pub fn restore(&self, path: String) -> impl Future> + use<> { + let mut restore = self.restore.borrow_mut(); + let cluster = Rc::clone(&self.node.cluster); + let (namespace, actor, epoch) = (self.namespace.clone(), self.actor.clone(), self.epoch); + restore + .get_or_insert_with(|| { + async move { + cluster + .restore_actor(&namespace, &actor, epoch, &path) + .await + .map_err(|e| kj::failed!("{namespace}/{actor}: takeover failed: {e}")) + } + .boxed_local() + .shared() + }) + .clone() + } +} + +impl Drop for Owned { + fn drop(&mut self) { + let cluster = Rc::clone(&self.node.cluster); + cluster.close_actor(&self.namespace, &self.actor); + let (namespace, actor, epoch) = (self.namespace.clone(), self.actor.clone(), self.epoch); + self.node.factory.spawn_detached(async move { + cluster.evict_actor(&namespace, &actor, epoch).await; + }); + } +} + +#[cfg(test)] +#[path = "cluster-test.rs"] +mod tests; diff --git a/src/workerd/server/server/config.rs b/src/workerd/server/server/config.rs index 31f38933477..9be554cc1e0 100644 --- a/src/workerd/server/server/config.rs +++ b/src/workerd/server/server/config.rs @@ -51,6 +51,7 @@ use crate::channels::ServerHandle; use crate::channels::SubrequestChannel; use crate::channels::TokenUsage; use crate::channels::WorkerInterface; +use crate::cluster::ClusterNode; use crate::listen::loopback::Loopback; use crate::loader::WorkerLoaderNamespace; use crate::services::DiskDirectoryService; @@ -312,6 +313,8 @@ pub struct Server { /// Durable namespaces by `uniqueKey`, for actor tokens. namespaces_by_unique_key: HashMap>, worker_loaders: RefCell, + /// This instance's cluster, in cluster mode. + cluster: Option>, /// Declared last, so that it is dropped last: the services (and their isolates) refer to it. factory: Rc, } @@ -332,6 +335,11 @@ impl Server { lookup_service(&self.services.0, &self.report, designator) } + #[must_use] + pub fn cluster(&self) -> Option<&Rc> { + self.cluster.as_ref() + } + /// Every service, in config order. pub fn services(&self) -> impl Iterator { self.services @@ -631,6 +639,7 @@ fn init_workflows( services: &LinkedHashMap, actor_configs: &ActorConfigs, report: &Reporter, + cluster: Option<&Rc>, namespaces_by_unique_key: &mut HashMap>, ) -> Result<()> { for service_conf in config.get_services().map_err(capnp_error)? { @@ -662,7 +671,7 @@ fn init_workflows( else { continue; }; - if let Some(namespace) = engine.init_workflow(workflow, key, config)? { + if let Some(namespace) = engine.init_workflow(workflow, key, config, cluster)? { namespaces_by_unique_key.insert(key.clone(), namespace); } } @@ -751,6 +760,7 @@ impl<'a> WorkflowEngine<'a> { workflow: &Workflow, key: &str, config: &ActorConfig, + cluster: Option<&Rc>, ) -> Result>> { let (workflow_name, class_name) = (workflow.name.as_str(), workflow.class_name.as_str()); if self.app.has_actor_class(key) { @@ -827,6 +837,7 @@ impl<'a> WorkflowEngine<'a> { actor_class, storage_path, self.engine.persistent_self_tokens(), + cluster, ) .map(Some) } @@ -1007,6 +1018,7 @@ impl Server { overrides: &Overrides, report: Reporter, mut inbound_listeners: HashMap>, + cluster_address: Option, ) -> Result> { let message = factory.config()?; let config = message.get_root::().map_err(capnp_error)?; @@ -1053,20 +1065,42 @@ impl Server { .insert("internet".to_owned(), Service::Leaf(internet)); } + let cluster = match cluster_address { + Some(address) if config.has_cluster() => { + make_cluster(&factory, config, address, &report)? + } + _ => None, + }; + if let Some(cluster) = &cluster { + // Actor databases open only under a lease. + cluster + .cluster + .join() + .await + .map_err(|e| kj::failed!("joining the cluster: {e}"))?; + } + // Every service exists: wire each worker's Workflows to the engine that runs them. init_workflows( config, &services.0, &actor_configs, &report, + cluster.as_ref(), &mut namespaces_by_unique_key, )?; + if let Some(cluster) = &cluster { + for namespace in namespaces_by_unique_key.values() { + namespace.set_cluster(Rc::clone(cluster)); + } + } let server = Rc::new(Self { report, services, namespaces_by_unique_key, worker_loaders: RefCell::new(WorkerLoaders::default()), + cluster, factory, }); let handle = Box::new(ServerHandle(Rc::downgrade(&server))); @@ -1475,3 +1509,23 @@ impl Server { #[cfg(test)] #[path = "config-test.rs"] mod tests; + +/// The cluster of `Config.cluster`, joined at `address`. `None` after reporting a config error. +fn make_cluster( + factory: &Rc, + config: config::Reader<'_>, + address: String, + report: &Reporter, +) -> Result>> { + if !factory.experimental() { + report.error("Cluster mode is experimental; it requires --experimental.".to_owned()); + return Ok(None); + } + let conf = config.get_cluster().map_err(capnp_error)?; + // The MAC's other inputs cross the wire, so a short key falls to an offline search. + if text(conf.get_key())?.len() < 32 { + report.error("cluster.key must be at least 32 bytes.".to_owned()); + return Ok(None); + } + Ok(Some(ClusterNode::new(Rc::clone(factory), conf, address)?)) +} diff --git a/src/workerd/server/server/lib.rs b/src/workerd/server/server/lib.rs index 04f9aeb34ab..913e3e6fdd3 100644 --- a/src/workerd/server/server/lib.rs +++ b/src/workerd/server/server/lib.rs @@ -15,6 +15,7 @@ pub mod bench; pub mod bindings; pub mod bridge; pub mod channels; +pub mod cluster; pub mod config; pub mod entry; pub mod in_process; diff --git a/src/workerd/server/server/run.rs b/src/workerd/server/server/run.rs index 58a72111dca..184b0b571a1 100644 --- a/src/workerd/server/server/run.rs +++ b/src/workerd/server/server/run.rs @@ -29,6 +29,8 @@ use crate::bindings::Designator; use crate::bindings::len_u32; use crate::bridge::ffi; use crate::channels::Channel; +use crate::cluster; +use crate::cluster::ClusterNode; use crate::config::Factory; use crate::config::Overrides; use crate::config::Reporter; @@ -61,6 +63,9 @@ pub struct RunOptions { pub control: Option, /// `--debug-port`. pub debug_port: Option, + /// `--cluster-addr` and `--cluster-advertise`. + pub cluster_address: Option, + pub cluster_advertise: Option, } /// The control channel (`--control-fd`), written line by line; a failed write is logged. The @@ -139,11 +144,21 @@ async fn start( control.listen_inspector(port); } + // The cluster port is bound before the services, like the sockets: its address goes in the + // lease record. + let cluster_port = if config.has_cluster() { + let conf = config.get_cluster().map_err(capnp_error)?; + let (address, advertise) = (&options.cluster_address, &options.cluster_advertise); + Some(cluster::bind(conf, address.as_deref(), advertise.as_deref()).await?) + } else { + None + }; let server = Server::start( Rc::clone(&factory), &options.overrides, report, bound.inbound, + cluster_port.as_ref().map(|(_, address)| address.clone()), ) .await?; report_unmatched_overrides(&server, &options.overrides, config, for_test)?; @@ -187,6 +202,9 @@ async fn start( for (name, port) in ports { control.listen(&name, port); } + if let (Some(node), Some((listener, _))) = (server.cluster(), cluster_port) { + start_cluster(&factory, node, listener); + } if let Some(address) = &options.debug_port { let listener = listen::listen(address, 0, factory.loopback()).await?; @@ -204,6 +222,22 @@ async fn start( })) } +/// Runs this instance's part in the cluster: the peer port, the lease, and compaction. They run for +/// the life of the process; a lease the instance cannot keep ends it. +fn start_cluster(factory: &Rc, node: &Rc, listener: tokio::net::TcpListener) { + factory.spawn_detached(cluster::serve(listener, Rc::clone(node))); + let cluster = Rc::clone(&node.cluster); + factory.spawn_detached(async move { + let reason = cluster.maintain().await; + // Peers may take this instance's objects over once its lease lapses: it must stop + // serving first. + tracing::error!("leaving the cluster: {reason}"); + std::process::exit(1); + }); + let cluster = Rc::clone(&node.cluster); + factory.spawn_detached(async move { cluster.tier_loop().await }); +} + /// The listener for one bound socket, or none for a socket whose type is unknown (reported). fn listener( context: Rc, @@ -371,6 +405,23 @@ pub async fn run( match futures::future::select(serving, drain).await { Either::Left((result, _)) => result?, Either::Right(((), serving)) => { + // A draining instance claims nothing more, lets its clustered objects go, + // `preventEviction` or not, and leaves the cluster, so peers take them over at their + // next listing while its connections finish. A request in flight on one could not + // commit anyway once a peer owns the object. + if let Some(node) = server.cluster() { + node.cluster.start_draining(); + let workers = server + .services() + .filter_map(|(_, service)| service.as_worker()); + let namespaces = workers.flat_map(|worker| worker.namespaces()); + for (_, namespace) in namespaces.filter(|(_, ns)| ns.cluster().is_some()) { + namespace.abort_all(None); + } + if let Err(e) = node.cluster.deregister().await { + tracing::warn!("leaving the cluster: {e}"); + } + } draining.send_replace(true); serving.await?; } diff --git a/src/workerd/server/server/worker.rs b/src/workerd/server/server/worker.rs index 2d526ac5d1f..73bbf974590 100644 --- a/src/workerd/server/server/worker.rs +++ b/src/workerd/server/server/worker.rs @@ -54,6 +54,7 @@ use crate::channels::WorkerEntrypoint; use crate::channels::WorkerInterface; use crate::channels::WorkerInterfaceList; use crate::channels::WorkerStub; +use crate::cluster::ClusterNode; use crate::config::Factory; use crate::config::Reporter; use crate::config::Server; @@ -462,6 +463,7 @@ impl WorkerService { }); let namespace = ActorNamespace::new( Rc::clone(&self.factory), + self.service_name.clone().unwrap_or_default(), class_name.clone(), config.clone(), actor_class, @@ -474,7 +476,8 @@ impl WorkerService { /// Creates the namespace backing one configured Workflow, keyed by `key`, its unique key. /// Its actors are `actor_class`, the engine's class with the Workflow's props bound, and its /// storage is in `storage_path`, the directory of the Workflow's `bindingService` worker. - /// `persistent_self_tokens` is the engine worker's. + /// `persistent_self_tokens` is the engine worker's. In cluster mode the namespace joins + /// `cluster` before its storage opens. pub fn init_workflow_namespace( &self, key: &str, @@ -482,14 +485,19 @@ impl WorkerService { actor_class: Rc, storage_path: Option<&str>, persistent_self_tokens: Persistent, + cluster: Option<&Rc>, ) -> Result> { let namespace = ActorNamespace::new( Rc::clone(&self.factory), + self.service_name.clone().unwrap_or_default(), key.to_owned(), config, actor_class, persistent_self_tokens, ); + if let Some(cluster) = cluster { + namespace.set_cluster(Rc::clone(cluster)); + } namespace.link(storage_path)?; self.namespaces .borrow_mut() diff --git a/src/workerd/server/tests/cluster/BUILD.bazel b/src/workerd/server/tests/cluster/BUILD.bazel new file mode 100644 index 00000000000..ee9543cf79c --- /dev/null +++ b/src/workerd/server/tests/cluster/BUILD.bazel @@ -0,0 +1,23 @@ +load("@aspect_rules_js//js:defs.bzl", "js_test") + +# Takeover waits out a dead owner's 20 s lease, several times over. +# TODO(cluster): under the 60 s "long" timeout. +js_test( + name = "cluster-test", + size = "large", + timeout = "eternal", + data = [ + ":cluster-test.wd-test", + ":cluster-worker.js", + ":fake-s3.js", + ":fake-s3.wd-test", + "//src/workerd/server:workerd", + ], + entry_point = "cluster-test.mjs", + env = { + "WORKERD_BINARY": "$(rootpath //src/workerd/server:workerd)", + "WD_TEST_CONFIG": "$(rootpath :cluster-test.wd-test)", + "FAKE_S3_CONFIG": "$(rootpath :fake-s3.wd-test)", + }, + tags = ["js-test"], +) diff --git a/src/workerd/server/tests/cluster/cluster-test.mjs b/src/workerd/server/tests/cluster/cluster-test.mjs new file mode 100644 index 00000000000..5591d1bdac1 --- /dev/null +++ b/src/workerd/server/tests/cluster/cluster-test.mjs @@ -0,0 +1,1197 @@ +// Copyright (c) 2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 +// +// Integration test for cluster mode. The test starts a fake S3 (fake-s3.js, in +// a workerd of its own) and several workerd instances whose cluster state lives +// in it, each with its SQLite files on a disk of its own, then exercises: +// 1. Consistent DO state across instances (single-writer correctness). +// 2. Transparent forwarding when a request arrives at the "wrong" instance. +// 3. Takeover after a node is killed. +// 4. Each running instance holds a lease record in the bucket. +// 5. Storing/loading a DO stub without calling it claims nothing. +// 6. A broken DO does not stay stuck while stubs still pin it: it can be +// re-instantiated instead of hanging requests. +// 7. Persistent stubs to restored RpcTargets rooted at a DO on one node can be +// stored and redeemed from a DO on another node; the restore runs on the +// root DO's owner. This holds whether the stub arrives as a token or was +// minted in-process and passed through props. +// 8. A stub that has resolved to a remote DO keeps failing once that DO +// breaks, while a fresh stub reaches the re-instantiated DO. + +import { spawn } from 'node:child_process'; +import { mkdir, mkdtemp, readdir, rm, writeFile } from 'node:fs/promises'; +import { tmpdir } from 'node:os'; +import { connect, createServer } from 'node:net'; +import { createHash, createHmac, randomBytes } from 'node:crypto'; +import { join } from 'node:path'; +import { env } from 'node:process'; +import { test } from 'node:test'; +import assert from 'node:assert'; +import { setTimeout as sleep } from 'node:timers/promises'; + +assert( + env.WORKERD_BINARY !== undefined, + 'You must set the WORKERD_BINARY environment variable.' +); +assert( + env.WD_TEST_CONFIG !== undefined, + 'You must set the WD_TEST_CONFIG environment variable.' +); +assert( + env.FAKE_S3_CONFIG !== undefined, + 'You must set the FAKE_S3_CONFIG environment variable.' +); + +const CONTROL_FD = 3; + +// Node's SIGTERM is TerminateProcess on Windows, so a node cannot drain there. +const SIGTERM_DRAINS = { + skip: process.platform === 'win32' && 'SIGTERM does not drain on Windows', +}; + +// A minimal harness specialized for the cluster test. Unlike server-harness.mjs +// it: +// - allows overriding bindings via process environment (NODE_ID), +// - points each instance at the fake S3 and at a disk of its own, +// - supports SIGKILL in addition to SIGTERM for the takeover test. +class ClusterNode { + #binary; + #config; + #localPath; + #s3Port; + #nodeId; + #child = null; + #httpPort = null; + #closed = null; + #stderrBuffer = ''; + #extraArgs; + + constructor({ binary, config, localPath, s3Port, nodeId, extraArgs = [] }) { + this.#extraArgs = extraArgs; + this.#binary = binary; + this.#config = config; + this.#localPath = localPath; + this.#s3Port = s3Port; + this.#nodeId = nodeId; + } + + get localPath() { + return this.#localPath; + } + + get nodeId() { + return this.#nodeId; + } + + get httpPort() { + assert(this.#httpPort !== null, 'node not started'); + return this.#httpPort; + } + + // Returns the contents of the node's stderr observed so far. + get stderr() { + return this.#stderrBuffer; + } + + async start() { + assert.strictEqual(this.#child, null); + + const args = [ + 'serve', + this.#config, + '--experimental', + '--verbose', + `--control-fd=${CONTROL_FD}`, + `--socket-addr=http=127.0.0.1:0`, + ]; + if (this.#localPath !== undefined) { + args.push(`--directory-path=local=${this.#localPath}`); + } + args.push(...this.#extraArgs); + + const child = spawn(this.#binary, args, { + stdio: ['ignore', 'inherit', 'pipe', 'pipe'], + env: { + ...env, + // Drop the runtime on exit, joining its threads, which TSan reports + // as leaked otherwise. + KJ_CLEAN_SHUTDOWN: '1', + NODE_ID: this.#nodeId, + AWS_ENDPOINT_URL: `http://127.0.0.1:${this.#s3Port}`, + }, + }); + this.#child = child; + + child.stderr.on('data', (data) => { + const chunk = data.toString('utf8'); + this.#stderrBuffer += chunk; + // Mirror to our own stderr so test output remains useful. + process.stderr.write(`[${this.#nodeId}] ${chunk}`); + }); + + // Watch the control fd for the assigned http port. + const portPromise = new Promise((resolve, reject) => { + let buffer = ''; + const onData = (data) => { + buffer += data.toString('utf8'); + let nl; + while ((nl = buffer.indexOf('\n')) !== -1) { + const line = buffer.slice(0, nl).trim(); + buffer = buffer.slice(nl + 1); + if (!line) continue; + try { + const parsed = JSON.parse(line); + if (parsed.event === 'listen' && parsed.socket === 'http') { + child.stdio[CONTROL_FD].off('data', onData); + resolve(parsed.port); + return; + } + } catch (err) { + reject( + new Error( + `Failed to parse control message from node ${this.#nodeId}: ${line}` + ) + ); + return; + } + } + }; + child.stdio[CONTROL_FD].on('data', onData); + child.once('error', reject); + child.once('exit', (code, signal) => { + if (this.#httpPort === null) { + reject( + new Error( + `Node ${this.#nodeId} exited before listening (code=${code}, signal=${signal})` + ) + ); + } + }); + }); + + this.#closed = new Promise((resolve) => { + child.once('exit', (code, signal) => resolve({ code, signal })); + }); + + await new Promise((resolve, reject) => { + child.once('spawn', resolve).once('error', reject); + }); + + this.#httpPort = await portPromise; + } + + async stop({ signal = 'SIGTERM', timeoutMs = 10_000 } = {}) { + if (this.#child === null) return null; + const child = this.#child; + this.#child = null; + child.kill(signal); + + const killTimer = setTimeout(() => { + try { + child.kill('SIGKILL'); + } catch (_) { + // process may already be gone. + } + }, timeoutMs); + + const result = await this.#closed; + clearTimeout(killTimer); + this.#httpPort = null; + return result; + } +} + +// Fetches `path` from the given node and parses the JSON response. When +// `timeoutMs` is given, a request that does not complete in time fails with a +// clear error instead of hanging the test (used where a bug would manifest as +// a request that never completes). +async function fetchJson(port, path, { timeoutMs } = {}) { + const url = `http://127.0.0.1:${port}${path}`; + let res; + try { + res = await fetch(url, { + signal: + timeoutMs === undefined ? undefined : AbortSignal.timeout(timeoutMs), + }); + } catch (err) { + if (err.name === 'TimeoutError') { + throw new Error( + `request to ${url} did not complete within ${timeoutMs}ms` + ); + } + throw err; + } + const text = await res.text(); + let body; + try { + body = JSON.parse(text); + } catch (err) { + throw new Error( + `Non-JSON response from ${url} (status ${res.status}): ${text}` + ); + } + return { status: res.status, body }; +} + +// The fake S3 the cluster's state lives in, in a workerd of its own. +class FakeS3 { + #node; + + constructor() { + // A cluster node's harness starts any workerd; this one has no cluster. + this.#node = new ClusterNode({ + binary: env.WORKERD_BINARY, + config: env.FAKE_S3_CONFIG, + nodeId: 's3', + }); + } + + get port() { + return this.#node.httpPort; + } + + start() { + return this.#node.start(); + } + + stop() { + return this.#node.stop({ timeoutMs: 5000 }); + } + + // The keys under `prefix` in the cluster's bucket, whose layout is under + // `cluster/v1/`. + async list(prefix) { + const query = new URLSearchParams({ + 'list-type': '2', + prefix: `cluster/v1/${prefix}`, + }); + const res = await fetch(`http://127.0.0.1:${this.port}/wdtest/?${query}`); + const xml = await res.text(); + return [...xml.matchAll(/([^<]*)<\/Key>/g)].map((m) => + m[1].slice('cluster/v1/'.length) + ); + } + + async get(key) { + const res = await fetch( + `http://127.0.0.1:${this.port}/wdtest/cluster/v1/${key}` + ); + return res.status === 404 ? null : await res.text(); + } + + // The key of the node that owns the Counter with this ID, or null when no + // owner record exists. + async owner(id) { + const record = await this.get(`${objectPrefix(id)}/own.json`); + return record === null ? null : JSON.parse(record).node; + } + + // How many GET, PUT and DELETE requests the bucket served for `key`. + async requests(key) { + const query = new URLSearchParams({ key }); + return ( + await fetch(`http://127.0.0.1:${this.port}/_requests?${query}`) + ).json(); + } + + // The keys of the nodes with a lease record. + async nodes() { + // Not the `.probe` a starting node writes and deletes. + return (await this.list('nodes/')) + .filter((key) => key.endsWith('.json')) + .map((key) => key.slice('nodes/'.length, -'.json'.length)); + } +} + +// Starts a fake S3 and `numNodes` workerd instances and runs `fn` with: +// nodes the ClusterNode instances, in start order +// bucket the FakeS3 +// nodeKeys Map from nodeId to the node's key (32-char hex), which is also +// what owner records name +async function withCluster(numNodes, fn, { extraArgs = () => [] } = {}) { + // Each test run gets its own bucket and disks, so state doesn't leak + // between tests. + const root = await mkdtemp(join(tmpdir(), 'workerd-cluster-')); + const bucket = new FakeS3(); + const nodes = []; + const nodeKeys = new Map(); + try { + await bucket.start(); + for (let i = 0; i < numNodes; i++) { + const localPath = join(root, `node${i}`); + await mkdir(localPath); + const node = new ClusterNode({ + binary: env.WORKERD_BINARY, + config: env.WD_TEST_CONFIG, + localPath, + s3Port: bucket.port, + nodeId: `node${i}`, + extraArgs: await extraArgs(i), + }); + await node.start(); + nodes.push(node); + + // A node writes its lease record in its first maintenance round. Since + // nodes start one at a time, the record that is new is this node's. + const known = new Set(nodeKeys.values()); + const deadline = Date.now() + 10_000; + let fresh = []; + while (Date.now() < deadline) { + fresh = (await bucket.nodes()).filter((key) => !known.has(key)); + if (fresh.length > 0) break; + await sleep(50); + } + assert.strictEqual( + fresh.length, + 1, + `expected exactly one new lease record after starting ${node.nodeId}, got: [${fresh.join(', ')}]` + ); + nodeKeys.set(node.nodeId, fresh[0]); + } + await fn({ nodes, bucket, nodeKeys }); + } finally { + // Stop all nodes (ignore errors -- some may already be stopped by the + // test itself). + for (const node of nodes) { + try { + await node.stop({ timeoutMs: 5000 }); + } catch (_) { + // ignore + } + } + await bucket.stop(); + await rm(root, { recursive: true, force: true }); + } +} + +// --------------------------------------------------------------------------- +// Tests + +test('cluster: DO state is consistent across instances (single writer)', async () => { + await withCluster(3, async ({ nodes }) => { + // Send a series of /increment requests to a round-robin of nodes, all for + // the same DO name. Track the maximum returned count -- it must be a + // strictly-increasing sequence (1, 2, 3, ...) because only one instance is + // the writer at any given time and the DO state is persistent. + const totalRequests = 12; + const expected = []; + for (let i = 1; i <= totalRequests; i++) expected.push(i); + + const observed = []; + const ownerIds = new Set(); + for (let i = 0; i < totalRequests; i++) { + const node = nodes[i % nodes.length]; + const { status, body } = await fetchJson( + node.httpPort, + '/increment?name=consistency' + ); + assert.strictEqual( + status, + 200, + `request to ${node.nodeId} failed: ${JSON.stringify(body)}` + ); + observed.push(body.count); + ownerIds.add(body.nodeId); + } + observed.sort((a, b) => a - b); + assert.deepStrictEqual( + observed, + expected, + `each /increment must be a unique strictly-increasing count` + ); + + // The DO is owned by exactly one node at a time. Even though we directed + // requests at all 3 nodes, every observed response should report the same + // nodeId -- the actual owner -- because incoming requests are forwarded. + assert.strictEqual( + ownerIds.size, + 1, + `expected exactly one DO owner across ${totalRequests} requests, got: ${[...ownerIds].join(', ')}` + ); + }); +}); + +test('cluster: requests to non-owner are transparently forwarded', async () => { + await withCluster(2, async ({ nodes }) => { + // Prime the DO by sending an increment to node 0. That node should claim + // ownership and respond. Then send a /get to node 1. Node 1 should forward + // to node 0 (the owner) and return the same state, and node 0 should be + // identified as the responder. + const first = await fetchJson( + nodes[0].httpPort, + '/increment?name=forwarding' + ); + assert.strictEqual(first.status, 200); + assert.strictEqual(first.body.count, 1); + const owner = first.body.nodeId; + + const second = await fetchJson(nodes[1].httpPort, '/get?name=forwarding'); + assert.strictEqual(second.status, 200); + assert.strictEqual( + second.body.count, + 1, + `forwarded /get must see prior increment` + ); + assert.strictEqual( + second.body.nodeId, + owner, + `forwarded request must be served by the original owner (${owner}), not the forwarding node (${nodes[1].nodeId})` + ); + + // Same id must be reported for both calls (sanity check for idFromName + // determinism across instances). + assert.strictEqual(first.body.id, second.body.id); + }); +}); + +test('cluster: killing the owner allows another node to take over', async () => { + await withCluster(2, async ({ nodes }) => { + // Prime the DO on whichever node ends up owning it. + const initial = await fetchJson( + nodes[0].httpPort, + '/increment?name=takeover' + ); + assert.strictEqual(initial.status, 200); + assert.strictEqual(initial.body.count, 1); + const originalOwner = initial.body.nodeId; + const ownerIndex = nodes.findIndex((n) => n.nodeId === originalOwner); + assert.notStrictEqual(ownerIndex, -1); + const survivorIndex = ownerIndex === 0 ? 1 : 0; + const survivor = nodes[survivorIndex]; + + // Hard-kill the owner. SIGKILL leaves its lease record in place, so the + // survivor takes over once that lease expires. + await nodes[ownerIndex].stop({ signal: 'SIGKILL', timeoutMs: 2000 }); + + // Hit the survivor repeatedly. Requests fail until the dead owner's lease + // expires; then the survivor takes ownership. The persisted count must be + // preserved. Each request is bounded: one that hung once held this test + // for nine minutes. + let response = null; + let lastErr = null; + const deadline = Date.now() + 30_000; + while (Date.now() < deadline) { + try { + response = await fetchJson( + survivor.httpPort, + '/increment?name=takeover', + { timeoutMs: 10_000 } + ); + if (response.status === 200) break; + } catch (err) { + lastErr = err; + } + await sleep(200); + } + assert( + response !== null && response.status === 200, + `survivor never succeeded; last error: ${lastErr}` + ); + assert.strictEqual( + response.body.nodeId, + survivor.nodeId, + `survivor (${survivor.nodeId}) must own the DO after takeover, got ${response.body.nodeId}` + ); + assert.strictEqual( + response.body.count, + 2, + `persisted state must be preserved across takeover` + ); + + // Further requests to the survivor continue to work and increment. + const followup = await fetchJson( + survivor.httpPort, + '/increment?name=takeover' + ); + assert.strictEqual(followup.status, 200); + assert.strictEqual(followup.body.count, 3); + assert.strictEqual(followup.body.nodeId, survivor.nodeId); + }); +}); + +test( + 'cluster: draining node still routes in-flight requests to a remote owner', + SIGTERM_DRAINS, + async () => { + await withCluster(2, async ({ nodes }) => { + // Prime the DO so that exactly one node owns it. + const initial = await fetchJson( + nodes[0].httpPort, + '/increment?name=drain' + ); + assert.strictEqual(initial.status, 200); + assert.strictEqual(initial.body.count, 1); + const owner = nodes.find((n) => n.nodeId === initial.body.nodeId); + const nonOwner = nodes.find((n) => n.nodeId !== initial.body.nodeId); + assert(owner !== undefined && nonOwner !== undefined); + + // Put a request in flight on the non-owner whose stateless handler waits + // before calling the DO, then SIGTERM the non-owner while it is waiting. + // Draining lets in-flight requests finish, so this request must still be + // able to route to the owner over cluster RPC after draining began. + const requestPromise = fetchJson( + nonOwner.httpPort, + '/increment?name=drain&delay=1000', + { timeoutMs: 10_000 } + ); + await sleep(250); + const exitPromise = nonOwner.stop({ + signal: 'SIGTERM', + timeoutMs: 10_000, + }); + + const response = await requestPromise; + assert.strictEqual( + response.status, + 200, + `in-flight request on draining node failed: ${JSON.stringify(response.body)}` + ); + assert.strictEqual(response.body.nodeId, owner.nodeId); + assert.strictEqual(response.body.count, 2); + + // The draining node must have exited cleanly once the request finished. + const exit = await exitPromise; + assert.deepStrictEqual(exit, { code: 0, signal: null }); + }); + } +); + +test( + 'cluster: draining node refuses to claim an unowned DO', + SIGTERM_DRAINS, + async () => { + await withCluster(2, async ({ nodes, bucket, nodeKeys }) => { + // Put a request in flight on node 0 for a DO that nobody owns yet, then + // SIGTERM node 0 while its stateless handler is still waiting. By the time + // the handler calls the DO, node 0 is draining: its cluster listener no + // longer accepts connections, so claiming the DO there would leave it + // unreachable from the rest of the cluster until node 0 exits. The claim + // must be refused instead. + const requestPromise = fetchJson( + nodes[0].httpPort, + '/increment?name=lame-duck&delay=1000', + { timeoutMs: 10_000 } + ); + await sleep(250); + const exitPromise = nodes[0].stop({ + signal: 'SIGTERM', + timeoutMs: 10_000, + }); + + const response = await requestPromise; + assert.strictEqual( + response.status, + 500, + `draining node must not claim the DO: ${JSON.stringify(response.body)}` + ); + // The refusal is a DISCONNECTED exception, which JS sees as a generic + // connection-loss error. + assert.match(response.body.error, /Network connection lost/); + + const exit = await exitPromise; + assert.deepStrictEqual(exit, { code: 0, signal: null }); + + // The DO was never instantiated anywhere: the survivor claims it fresh. + const survivor = await fetchJson( + nodes[1].httpPort, + '/increment?name=lame-duck' + ); + assert.strictEqual(survivor.status, 200, JSON.stringify(survivor.body)); + assert.strictEqual(survivor.body.count, 1); + assert.strictEqual(survivor.body.nodeId, nodes[1].nodeId); + assert.strictEqual( + await bucket.owner(survivor.body.id), + nodeKeys.get(nodes[1].nodeId) + ); + }); + } +); + +test( + "cluster: a peer takes over a draining owner's DO before its lease ends", + SIGTERM_DRAINS, + async () => { + await withCluster(2, async ({ nodes }) => { + const initial = await fetchJson( + nodes[0].httpPort, + '/increment?name=handoff' + ); + assert.strictEqual(initial.status, 200); + const owner = nodes.find((n) => n.nodeId === initial.body.nodeId); + const peer = nodes.find((n) => n.nodeId !== initial.body.nodeId); + + // A slow request holds the owner draining for longer than a peer + // needs to see it leave, but within its lease. + const holding = fetchJson( + owner.httpPort, + '/increment?name=hold&delay=12000', + { timeoutMs: 20_000 } + ); + await sleep(250); + const drainedAt = Date.now(); + const exitPromise = owner.stop({ signal: 'SIGTERM', timeoutMs: 20_000 }); + + // A request waits on the owner until the peer's next listing drops + // it, then the peer claims. One timed out would still increment. + let response = null; + while (Date.now() < drainedAt + 10_000) { + response = await fetchJson(peer.httpPort, '/increment?name=handoff', { + timeoutMs: drainedAt + 10_000 - Date.now(), + }).catch(() => null); + if (response?.status === 200) break; + await sleep(200); + } + assert.strictEqual(response?.status, 200, JSON.stringify(response?.body)); + assert.strictEqual(response.body.nodeId, peer.nodeId); + assert.strictEqual(response.body.count, 2); + await holding.catch(() => {}); + assert.deepStrictEqual(await exitPromise, { code: 0, signal: null }); + }); + } +); + +test('cluster: concurrent first requests to a DO read its owner record once per node', async () => { + await withCluster(2, async ({ nodes, bucket }) => { + // Without coalescing, each of these reads own.json and the losers retry + // their claims: the stampede that starved lease renewal on the fleet. + const results = await Promise.all( + Array.from({ length: 64 }, (_, i) => + fetchJson(nodes[i % 2].httpPort, '/increment?name=herd') + ) + ); + for (const r of results) + assert.strictEqual(r.status, 200, JSON.stringify(r.body)); + const counts = results.map((r) => r.body.count).sort((a, b) => a - b); + assert.deepStrictEqual( + counts, + Array.from({ length: 64 }, (_, i) => i + 1) + ); + const requests = await bucket.requests( + `${objectPrefix(results[0].body.id)}/own.json` + ); + assert(requests <= 8, `${requests} owner record requests for one cold DO`); + }); +}); + +// Where a Durable Object's records are in the bucket: a hash of its +// namespace's uniqueKey, then its ID. +function objectPrefix(id, uniqueKey = 'counter-test-namespace') { + const hash = createHash('sha256'); + for (const part of ['workerd cluster namespace v1', '', uniqueKey]) { + const length = Buffer.alloc(8); + length.writeBigUInt64BE(BigInt(Buffer.byteLength(part))); + hash.update(length).update(part); + } + return `do/${hash.digest().subarray(0, 16).toString('base64url')}/${id}`; +} + +// A port that was free a moment ago. +async function freePort() { + const server = createServer(); + await new Promise((resolve) => server.listen(0, '127.0.0.1', resolve)); + const { port } = server.address(); + await new Promise((resolve) => server.close(resolve)); + return port; +} + +test('cluster: an instance bound to a wildcard advertises --cluster-advertise', async () => { + const ports = []; + const extraArgs = async (i) => { + ports[i] = await freePort(); + return [ + `--cluster-addr=0.0.0.0:${ports[i]}`, + `--cluster-advertise=127.0.0.1:${ports[i]}`, + ]; + }; + await withCluster( + 2, + async ({ nodes, bucket, nodeKeys }) => { + for (const [i, node] of nodes.entries()) { + const record = JSON.parse( + await bucket.get(`nodes/${nodeKeys.get(node.nodeId)}.json`) + ); + assert.strictEqual(record.address, `127.0.0.1:${ports[i]}`); + } + // Peers reach each other at the advertised addresses. + const first = await fetchJson( + nodes[0].httpPort, + '/increment?name=advertised' + ); + const second = await fetchJson( + nodes[1].httpPort, + '/increment?name=advertised' + ); + assert.strictEqual(first.status, 200, JSON.stringify(first.body)); + assert.strictEqual(second.status, 200, JSON.stringify(second.body)); + assert.strictEqual(second.body.count, 2); + }, + { extraArgs } + ); +}); + +// The status line a CONNECT to the cluster port at `port` gets, with `auth` +// as its cf-cluster-auth header, if any. +async function clusterConnect(port, auth) { + const socket = connect(port, '127.0.0.1'); + await new Promise((resolve) => socket.once('connect', resolve)); + const header = auth === undefined ? '' : `cf-cluster-auth: ${auth}\r\n`; + socket.write( + `CONNECT cluster:1 HTTP/1.1\r\nhost: cluster:1\r\n${header}\r\n` + ); + const [data] = await Promise.race([ + new Promise((resolve) => socket.once('data', (d) => resolve([d]))), + sleep(5000).then(() => [Buffer.from('timeout')]), + ]); + socket.destroy(); + return data.toString().split('\r\n')[0]; +} + +// A cf-cluster-auth header for a CONNECT to `target`, as an instance with the +// test config's key makes it. +function clusterAuth(target, ms = Date.now(), nonce = randomBytes(16)) { + const n = nonce.toString('hex'); + const mac = createHmac('sha256', 'test-cluster-shared-secret-of-32-bytes') + .update(`workerd cluster connect v1\n${target}\n${ms}\n${n}`) + .digest('hex'); + return `${ms}.${n}.${mac}`; +} + +test('cluster: a CONNECT is refused when replayed, stale, misdirected or unsigned', async () => { + const port = await freePort(); + const extraArgs = async () => [`--cluster-addr=127.0.0.1:${port}`]; + await withCluster( + 1, + async () => { + const target = `127.0.0.1:${port}`; + const auth = clusterAuth(target); + assert.match(await clusterConnect(port, auth), / 200 /); + assert.match(await clusterConnect(port, auth), / 403 /, 'replayed'); + const stale = clusterAuth(target, Date.now() - 60_000); + assert.match(await clusterConnect(port, stale), / 403 /, 'stale'); + const elsewhere = clusterAuth(`127.0.0.1:${port + 1}`); + assert.match(await clusterConnect(port, elsewhere), / 403 /, 'target'); + assert.match(await clusterConnect(port), / 403 /, 'unsigned'); + }, + { extraArgs } + ); +}); + +test('cluster: a public cluster address is refused', async () => { + const local = await mkdtemp(join(tmpdir(), 'workerd-cluster-')); + for (const flag of ['--cluster-addr', '--cluster-advertise']) { + const child = spawn( + env.WORKERD_BINARY, + [ + 'serve', + env.WD_TEST_CONFIG, + '--experimental', + '--socket-addr=http=127.0.0.1:0', + `${flag}=8.8.8.8:7000`, + `--directory-path=local=${local}`, + ], + { stdio: ['ignore', 'ignore', 'pipe'] } + ); + let stderr = ''; + child.stderr.on('data', (d) => (stderr += d)); + const code = await new Promise((resolve) => child.once('exit', resolve)); + assert.notStrictEqual(code, 0, flag); + assert.match(stderr, /8\.8\.8\.8 is public/, flag); + } + await rm(local, { recursive: true }); +}); + +test('cluster: Durable Objects from before cluster mode are refused', async () => { + await withCluster(1, async ({ bucket }) => { + const local = await mkdtemp(join(tmpdir(), 'workerd-cluster-')); + await mkdir(join(local, 'counter-test-namespace')); + await writeFile(join(local, 'counter-test-namespace', 'a.sqlite'), ''); + const node = new ClusterNode({ + binary: env.WORKERD_BINARY, + config: env.WD_TEST_CONFIG, + localPath: local, + s3Port: bucket.port, + nodeId: 'old', + }); + await assert.rejects(node.start(), /exited before listening/); + assert.match(node.stderr, /from before cluster mode/); + await rm(local, { recursive: true }); + }); +}); + +test('cluster: each running instance holds a lease record', async () => { + await withCluster(2, async ({ nodes, bucket, nodeKeys }) => { + const records = await bucket.nodes(); + assert.deepStrictEqual( + records.sort(), + [...nodeKeys.values()].sort(), + `expected one lease record per instance` + ); + for (const key of records) { + const record = JSON.parse(await bucket.get(`nodes/${key}.json`)); + assert(record.expires_ms > Date.now(), `lease of ${key} has expired`); + } + }); +}); + +test('cluster: setting an alarm is refused', async () => { + await withCluster(1, async ({ nodes }) => { + const res = await fetchJson(nodes[0].httpPort, '/set-alarm?name=alarm'); + assert.match(res.body.error ?? '', /not yet supported in cluster mode/); + }); +}); + +test('cluster: a DO reopened where it was closed resumes its epoch', async () => { + await withCluster(1, async ({ nodes, bucket }) => { + const [node] = nodes; + await fetchJson(node.httpPort, '/increment?name=dormant'); + const before = await fetchJson(node.httpPort, '/increment?name=dormant'); + assert.strictEqual(before.body.count, 2, JSON.stringify(before.body)); + const id = before.body.id; + // Breaking it drops its container, as expiry does: the database + // closes, and the bucket gets what it lacks. + await fetchJson(node.httpPort, '/break?name=dormant'); + await sleep(500); + const after = await fetchJson(node.httpPort, '/increment?name=dormant'); + assert.strictEqual(after.body.count, 3, JSON.stringify(after.body)); + const prefix = objectPrefix(id); + const record = JSON.parse(await bucket.get(`${prefix}/own.json`)); + assert.strictEqual(record.epoch, 1, JSON.stringify(record)); + // No session 2, whose name is `zd`. + assert.deepStrictEqual(await bucket.list(`${prefix}/snapshot/zd/`), []); + assert.match(node.stderr, /epoch 1 resumes at \d+/); + }); +}); + +test('cluster: storing and loading a stub claims nothing', async () => { + await withCluster(2, async ({ nodes, bucket }) => { + // DO "holder" (on whichever node claims it) stores a stub to DO "lazy" and + // reads it back. Serializing and deserializing the stub must not claim + // "lazy": no owner record, and no storage for it. + const stored = await fetchJson( + nodes[1].httpPort, + '/store-stub?name=holder&target=lazy' + ); + assert.strictEqual(stored.status, 200, JSON.stringify(stored.body)); + assert.strictEqual(stored.body.hasStub, true); + const holderId = stored.body.id; + const targetId = stored.body.targetId; + assert.notStrictEqual(holderId, targetId); + assert.notStrictEqual( + await bucket.owner(holderId), + null, + `the holder DO itself must be claimed` + ); + assert.strictEqual( + await bucket.owner(targetId), + null, + `storing/loading a stub must not claim its target` + ); + for (const node of nodes) { + let entries = []; + try { + entries = await readdir(join(node.localPath, 'counter-test-namespace')); + } catch (_) { + // No storage at all on this node. + } + assert( + !entries.some((name) => name.startsWith(targetId)), + `storing/loading a stub must not create actor storage; found: ${entries.join(', ')}` + ); + } + + // Calling the loaded stub claims it. + const called = await fetchJson( + nodes[1].httpPort, + '/call-stored-stub?name=holder' + ); + assert.strictEqual(called.status, 200, JSON.stringify(called.body)); + assert.strictEqual(called.body.target.id, targetId); + assert.strictEqual(called.body.target.count, 0); + assert.notStrictEqual( + await bucket.owner(targetId), + null, + `calling the stub must claim its target` + ); + }); +}); + +// Requests that would hang forever if a broken actor kept its ownership lock +// are bounded by this timeout so the failure is a clear error. +const HANG_TIMEOUT_MS = 5000; + +test('cluster: a broken DO pinned by a stub does not keep its ownership lock', async () => { + await withCluster(2, async ({ nodes, bucket, nodeKeys }) => { + // Node 0 claims "pinned". DO "holder", served by node 1, then obtains a + // stub to it, uses it, and keeps it for as long as "holder" lives. Through + // node 0's bootstrap for that stub, node 0's container for "pinned" stays + // referenced for the rest of the test. + const first = await fetchJson(nodes[0].httpPort, '/increment?name=pinned'); + assert.strictEqual(first.status, 200, JSON.stringify(first.body)); + assert.strictEqual(first.body.count, 1); + const owner = first.body.nodeId; + const objectId = first.body.id; + assert.strictEqual(await bucket.owner(objectId), nodeKeys.get(owner)); + + const held = await fetchJson( + nodes[1].httpPort, + '/hold-stub?name=holder&target=pinned' + ); + assert.strictEqual(held.status, 200, JSON.stringify(held.body)); + assert.strictEqual(held.body.target.nodeId, owner); + + // Break the DO. Its container leaves the owner's map but lives on, pinned + // by the parked stub. + const broken = await fetchJson(nodes[1].httpPort, '/break?name=pinned'); + assert.strictEqual(broken.status, 500, JSON.stringify(broken.body)); + + // The lock must have been released along with the map entry. Otherwise a + // request from the non-owner routes to the owner, which finds no container + // and routes to itself, forever. + const other = nodes.find((n) => n.nodeId !== owner); + const after = await fetchJson(other.httpPort, '/increment?name=pinned', { + timeoutMs: HANG_TIMEOUT_MS, + }); + assert.strictEqual(after.status, 200, JSON.stringify(after.body)); + assert.strictEqual(after.body.count, 2, 'state must survive the break'); + assert.strictEqual( + await bucket.owner(objectId), + nodeKeys.get(after.body.nodeId), + 'the lock file must name the node that re-instantiated the DO' + ); + }); +}); + +test('cluster: a DO whose constructor fails releases its ownership lock', async () => { + await withCluster(2, async ({ nodes, bucket, nodeKeys }) => { + const first = await fetchJson(nodes[0].httpPort, '/increment?name=ctor'); + assert.strictEqual(first.status, 200, JSON.stringify(first.body)); + const objectId = first.body.id; + + // Arm the one-shot constructor failure and abort the running instance so + // the next request has to construct a new one. + const armed = await fetchJson( + nodes[1].httpPort, + '/arm-constructor-failure?name=ctor' + ); + assert.strictEqual(armed.status, 500, JSON.stringify(armed.body)); + + // Re-instantiate through a stub kept by DO "holder". The constructor + // throws, which breaks the new instance; the kept stub pins its hollowed + // container on whichever node claimed the DO. + const held = await fetchJson( + nodes[1].httpPort, + '/hold-stub?name=holder&target=ctor', + { timeoutMs: HANG_TIMEOUT_MS } + ); + assert.strictEqual(held.status, 500, JSON.stringify(held.body)); + assert.match(held.body.error, /constructor failed on purpose/); + + // A fresh request from the other node must not hang. The failed constructor + // committed the cleared flag before throwing, so this instantiation + // succeeds and sees the earlier increment. + const after = await fetchJson(nodes[0].httpPort, '/increment?name=ctor', { + timeoutMs: HANG_TIMEOUT_MS, + }); + assert.strictEqual(after.status, 200, JSON.stringify(after.body)); + assert.strictEqual(after.body.count, 2); + assert.strictEqual( + await bucket.owner(objectId), + nodeKeys.get(after.body.nodeId), + 'the lock file must name the node that re-instantiated the DO' + ); + }); +}); + +test('cluster: aborting a DO with a request in flight lets it be re-instantiated immediately', async () => { + await withCluster(2, async ({ nodes, bucket, nodeKeys }) => { + const first = await fetchJson( + nodes[0].httpPort, + '/increment?name=inflight' + ); + assert.strictEqual(first.status, 200, JSON.stringify(first.body)); + const owner = first.body.nodeId; + const objectId = first.body.id; + + // Start a request the DO holds open, give it time to reach the DO, then + // abort the DO underneath it. The in-flight request keeps a reference to + // the old Worker::Actor until it unwinds, but the storage must be closed + // and the lock released as soon as the actor breaks, not when that + // reference goes away. + const holding = fetchJson(nodes[0].httpPort, '/hold?name=inflight', { + timeoutMs: HANG_TIMEOUT_MS, + }); + await sleep(250); + const broken = await fetchJson(nodes[0].httpPort, '/break?name=inflight'); + assert.strictEqual(broken.status, 500, JSON.stringify(broken.body)); + + const other = nodes.find((n) => n.nodeId !== owner); + const after = await fetchJson(other.httpPort, '/increment?name=inflight', { + timeoutMs: HANG_TIMEOUT_MS, + }); + assert.strictEqual(after.status, 200, JSON.stringify(after.body)); + assert.strictEqual( + after.body.count, + 2, + 'the new instance must see the state committed before the abort' + ); + assert.strictEqual( + await bucket.owner(objectId), + nodeKeys.get(after.body.nodeId), + 'the lock file must name the node that re-instantiated the DO' + ); + + const held = await holding; + assert.strictEqual(held.status, 500, JSON.stringify(held.body)); + }); +}); + +// Sets up the cross-node persistent-stub scenario of the test below: DO "root" +// is claimed by node 0, then DO "holder", served through node 1 (and therefore +// claimed by node 1), asks "root" to vend a persistent stub of the given kind +// and stores it. Returns root's node ID. +async function vendAcrossNodes(nodes, kind) { + const first = await fetchJson(nodes[0].httpPort, '/increment?name=root'); + assert.strictEqual(first.status, 200, JSON.stringify(first.body)); + const rootOwner = first.body.nodeId; + + const vended = await fetchJson( + nodes[1].httpPort, + `/vend-stub?name=holder&kind=${kind}&target=root`, + { timeoutMs: HANG_TIMEOUT_MS } + ); + assert.strictEqual(vended.status, 200, JSON.stringify(vended.body)); + assert.strictEqual(vended.body.hasStub, true); + assert.notStrictEqual( + vended.body.nodeId, + rootOwner, + 'the test requires root and holder to live on different nodes' + ); + + // Vending ran [restore]() live. Storing and loading the stub must not have + // replayed it. + const state = await fetchJson(nodes[1].httpPort, '/get?name=root'); + assert.strictEqual(state.status, 200, JSON.stringify(state.body)); + assert.strictEqual(state.body.restores, 1); + + return rootOwner; +} + +// TODO(cluster): the token-carrying peer call (bucket-mvp's getChannelFromToken). +test( + 'cluster: a restored RpcTarget stub is redeemed on the root DO owner', + { todo: true }, + async () => { + await withCluster(2, async ({ nodes }) => { + const rootOwner = await vendAcrossNodes(nodes, 'rpc'); + + // The RpcTarget mutates root's own storage, so it must run in root's + // context on root's owner. + const used = await fetchJson( + nodes[1].httpPort, + '/use-stub?name=holder&kind=rpc&amount=10', + { timeoutMs: HANG_TIMEOUT_MS } + ); + assert.strictEqual(used.status, 200, JSON.stringify(used.body)); + assert.deepStrictEqual(used.body.result, { + count: 11, + nodeId: rootOwner, + }); + + const state = await fetchJson(nodes[0].httpPort, '/get?name=root'); + assert.strictEqual(state.body.count, 11); + assert.strictEqual(state.body.restores, 2); + }); + } +); + +test('cluster: a stub resolved to a remote DO keeps failing after that DO breaks', async () => { + await withCluster(2, async ({ nodes }) => { + const first = await fetchJson(nodes[0].httpPort, '/increment?name=sticky'); + assert.strictEqual(first.status, 200, JSON.stringify(first.body)); + const owner = first.body.nodeId; + + // DO "holder" on node 1 obtains a stub to "sticky", uses it once (resolving + // it to node 0's instance), and keeps it. + const held = await fetchJson( + nodes[1].httpPort, + '/hold-stub?name=holder&target=sticky' + ); + assert.strictEqual(held.status, 200, JSON.stringify(held.body)); + assert.strictEqual(held.body.target.nodeId, owner); + assert.notStrictEqual(held.body.nodeId, owner); + + const broken = await fetchJson(nodes[1].httpPort, '/break?name=sticky'); + assert.strictEqual(broken.status, 500, JSON.stringify(broken.body)); + + // The held stub points at the broken instance and must keep failing, even + // after a fresh stub has re-instantiated the DO. + const heldAfter = await fetchJson( + nodes[1].httpPort, + '/call-held-stub?name=holder&target=sticky', + { timeoutMs: HANG_TIMEOUT_MS } + ); + assert.strictEqual(heldAfter.status, 500, JSON.stringify(heldAfter.body)); + assert.match(heldAfter.body.error, /broken on purpose/); + + const fresh = await fetchJson(nodes[1].httpPort, '/increment?name=sticky', { + timeoutMs: HANG_TIMEOUT_MS, + }); + assert.strictEqual(fresh.status, 200, JSON.stringify(fresh.body)); + assert.strictEqual(fresh.body.count, 2); + + const heldAgain = await fetchJson( + nodes[1].httpPort, + '/call-held-stub?name=holder&target=sticky', + { timeoutMs: HANG_TIMEOUT_MS } + ); + assert.strictEqual(heldAgain.status, 500, JSON.stringify(heldAgain.body)); + assert.match(heldAgain.body.error, /broken on purpose/); + }); +}); + +test('cluster: a freshly minted RpcTarget stub passed via props reaches its DO after it breaks', async () => { + await withCluster(2, async ({ nodes }) => { + const first = await fetchJson(nodes[0].httpPort, '/increment?name=root'); + assert.strictEqual(first.status, 200, JSON.stringify(first.body)); + const originalOwner = first.body.nodeId; + + // Root mints an RpcTarget stub and hands it to a Consumer entrypoint on the + // same node via props. The Consumer holds the stub's channel only -- the + // very channel object ctx.restore() created, not a decoded token -- and + // waits for the go signal before using it. + const vended = await fetchJson( + nodes[0].httpPort, + '/vend-to-consumer?name=root' + ); + assert.strictEqual(vended.status, 200, JSON.stringify(vended.body)); + + // Break root, then re-instantiate it through the other node. A broken DO + // keeps its owner, which re-instantiates it. + const broken = await fetchJson(nodes[1].httpPort, '/break?name=root'); + assert.strictEqual(broken.status, 500, JSON.stringify(broken.body)); + const moved = await fetchJson(nodes[1].httpPort, '/increment?name=root', { + timeoutMs: HANG_TIMEOUT_MS, + }); + assert.strictEqual(moved.status, 200, JSON.stringify(moved.body)); + assert.strictEqual(moved.body.count, 2); + assert.strictEqual(moved.body.nodeId, originalOwner); + + // Now let the Consumer use its stub: it reaches the re-instantiated DO. + const go = await fetchJson(nodes[0].httpPort, '/set-go?name=sink'); + assert.strictEqual(go.status, 200, JSON.stringify(go.body)); + + const deadline = Date.now() + HANG_TIMEOUT_MS; + let recorded; + while (recorded === undefined) { + assert(Date.now() < deadline, 'Consumer never recorded a result'); + await sleep(50); + const sink = await fetchJson(nodes[0].httpPort, '/get?name=sink'); + recorded = sink.body.recorded; + } + assert.deepStrictEqual(recorded, { count: 3, nodeId: moved.body.nodeId }); + + const state = await fetchJson(nodes[1].httpPort, '/get?name=root'); + assert.strictEqual(state.body.restores, 2); + }); +}); diff --git a/src/workerd/server/tests/cluster/cluster-test.wd-test b/src/workerd/server/tests/cluster/cluster-test.wd-test new file mode 100644 index 00000000000..147a47732a2 --- /dev/null +++ b/src/workerd/server/tests/cluster/cluster-test.wd-test @@ -0,0 +1,53 @@ +# Copyright (c) 2026 Cloudflare, Inc. +# Licensed under the Apache 2.0 license found in the LICENSE file or at: +# https://opensource.org/licenses/Apache-2.0 + +# Each instance keeps its SQLite files on its own disk ("local", one directory per instance), and +# membership and Durable Object ownership live in a bucket: a fake S3 (fake-s3.wd-test) that the +# test starts, whose address it passes in AWS_ENDPOINT_URL. + +using Workerd = import "/workerd/workerd.capnp"; + +const config :Workerd.Config = ( + services = [ + ( name = "local", + disk = ( + writable = true, + allowDotfiles = true, + ) + ), + + ( name = "main", + worker = ( + modules = [ + ( name = "cluster-worker.js", esModule = embed "cluster-worker.js" ), + ], + compatibilityDate = "2026-01-01", + compatibilityFlags = ["nodejs_compat", "allow_irrevocable_stub_storage"], + bindings = [ + ( name = "COUNTER", durableObjectNamespace = "Counter" ), + ( name = "NODE_ID", fromEnvironment = "NODE_ID" ), + ], + durableObjectNamespaces = [ + ( className = "Counter", uniqueKey = "counter-test-namespace" ), + ], + durableObjectStorage = ( localDisk = "local" ), + ) + ), + ], + + sockets = [ + ( name = "http", address = "*:0", http = (), service = "main" ), + ], + + cluster = ( + bucket = "s3://wdtest/cluster", + bucketOptions = [ + "aws_region=us-east-1", + "aws_access_key_id=wdtest", + "aws_secret_access_key=wdtest-secret", + ], + address = "127.0.0.1:0", + key = "test-cluster-shared-secret-of-32-bytes", + ), +); diff --git a/src/workerd/server/tests/cluster/cluster-worker.js b/src/workerd/server/tests/cluster/cluster-worker.js new file mode 100644 index 00000000000..fc07787b697 --- /dev/null +++ b/src/workerd/server/tests/cluster/cluster-worker.js @@ -0,0 +1,314 @@ +// Copyright (c) 2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 + +// Test worker for the cluster integration test. Exposes a Counter Durable Object +// and a top-level fetch() that routes requests to the DO by name. Requests that +// fail are reported as JSON `{ error }` with status 500 rather than letting the +// exception propagate, so the test can assert on the message. +// +// The DO supports the following request paths: +// /increment Increments and returns the current counter value. +// /get Returns the current counter value without incrementing it, +// along with `restores`, the number of times this DO's +// [restore]() method has run. +// /identity Returns information identifying this instance (NODE_ID env) +// along with the DO id, so the test can verify which instance +// served the request. +// /store-stub Stores a stub to the DO named by the `target` query param in +// this DO's storage, then reads it back without calling it. +// Returns the target's id. +// /call-stored-stub +// Loads the stub stored by /store-stub and calls /get on it. +// Returns the target's response. +// /hold-stub Obtains a stub to the DO named by the `target` query param, +// calls /identity on it, and keeps the stub on this instance +// for as long as this instance lives. (A stub is owned by the +// request context that created it, so a module-level variable +// would not keep it alive past the request; an instance field +// of a long-lived DO does.) Returns the target's response. +// /call-held-stub Calls /identity on the stub kept by /hold-stub for the DO +// named by the `target` query param. Returns the target's +// response. +// /hold Waits one second before responding, so that the test can +// break the DO while this request is still in flight. +// /break Aborts the DO via ctx.abort(). The request fails. +// /arm-constructor-failure +// Sets a storage flag that makes the *next* instantiation of +// this DO throw from its constructor (the flag is cleared and +// committed before throwing, so the instantiation after that +// succeeds), then aborts the DO so that the next request +// re-instantiates it. +// +// Persistent-stub paths. Kind `rpc` names a CounterAdder RpcTarget, vended by +// the DO given by the `target` query param via its [restore]() method, that +// mutates the target's counter: +// /vend-stub?kind=K&target=T +// Asks the target (over RPC) to vend a persistent stub of the +// given kind, stores it in this DO's storage under `kind`, and +// reads it back without calling it. +// /use-stub?kind=K[&amount=N] +// Loads the stub stored by /vend-stub and calls add(amount) +// on it. Returns the result. +// /vend-to-consumer +// Vends an `rpc` stub of *this* DO and hands it, via props, +// to a Consumer entrypoint in the same process, which keeps +// only the stub's channel (props carry no live capability). +// The Consumer waits until DO "sink" has its `go` flag set, +// then calls the stub and records the result in "sink". +// /set-go Sets this DO's `go` flag (read by Consumer via /get). +// /record?value=V Stores V (JSON) as this DO's `recorded` value. +// +// The top-level fetch() additionally honors a `delay` query parameter (ms): it +// waits that long *before* calling the DO. This lets a test put a request in +// flight on a node and SIGTERM that node while the request has not yet routed +// to the DO's owner. + +import { + DurableObject, + RpcTarget, + WorkerEntrypoint, + restore, +} from 'cloudflare:workers'; + +export default { + async fetch(request, env) { + const url = new URL(request.url); + + // The DO name to act on is given via the `name` query parameter (defaults + // to "default"). All instances using the same name will route to the same + // DO id. + const name = url.searchParams.get('name') ?? 'default'; + + // Synthesize a stub URL for the DO. + const id = env.COUNTER.idFromName(name); + const stub = env.COUNTER.get(id); + + const delay = url.searchParams.get('delay'); + if (delay !== null) { + await scheduler.wait(Number(delay)); + } + + // Forward to the DO with the same path and query. + const forwardUrl = new URL(url.pathname + url.search, 'http://do/'); + try { + return await stub.fetch(forwardUrl.toString(), { + method: request.method, + headers: request.headers, + }); + } catch (err) { + return Response.json({ error: String(err) }, { status: 500 }); + } + }, +}; + +// RpcTarget vended by Counter's [restore]() that mutates the counter's storage. +// It runs in the Counter's context on whichever node owns the Counter. +class CounterAdder extends RpcTarget { + constructor(counter) { + super(); + this.counter = counter; + } + + async add(amount) { + const count = + ((await this.counter.state.storage.get('count')) ?? 0) + amount; + await this.counter.state.storage.put('count', count); + return { count, nodeId: this.counter.env.NODE_ID ?? '' }; + } +} + +// See /vend-to-consumer. The stub in `props` was minted by ctx.restore() in this +// process and arrives here without its live capability, so the first call on it +// restores through the channel object the minting DO created, not through a +// decoded token. +export class Consumer extends WorkerEntrypoint { + useLater() { + const adder = this.ctx.props.adder; + const sink = this.env.COUNTER.get(this.env.COUNTER.idFromName('sink')); + this.ctx.waitUntil( + (async () => { + for (;;) { + const state = await (await sink.fetch('http://do/get')).json(); + if (state.go) break; + await scheduler.wait(50); + } + let result; + try { + result = await adder.add(1); + } catch (err) { + result = { error: String(err) }; + } + await sink.fetch( + 'http://do/record?value=' + encodeURIComponent(JSON.stringify(result)) + ); + })() + ); + } +} + +const BREAK_ON_CONSTRUCT = 'break-on-construct'; + +export class Counter extends DurableObject { + constructor(state, env) { + super(state, env); + this.state = state; + this.env = env; + // See /hold-stub. Maps target DO name -> stub. + this.heldStubs = new Map(); + + state.blockConcurrencyWhile(async () => { + if (await state.storage.get(BREAK_ON_CONSTRUCT)) { + // One-shot: make sure the cleared flag is durable before breaking, so + // the next instantiation succeeds. + await state.storage.delete(BREAK_ON_CONSTRUCT); + await state.storage.sync(); + throw new Error('constructor failed on purpose'); + } + }); + } + + async fetch(request) { + const url = new URL(request.url); + const nodeId = this.env.NODE_ID ?? ''; + const idHex = this.state.id.toString(); + + if (url.pathname === '/increment') { + let count = (await this.state.storage.get('count')) ?? 0; + count += 1; + await this.state.storage.put('count', count); + return Response.json({ + count, + nodeId, + id: idHex, + }); + } else if (url.pathname === '/get') { + const count = (await this.state.storage.get('count')) ?? 0; + const restores = (await this.state.storage.get('restores')) ?? 0; + const go = (await this.state.storage.get('go')) ?? false; + const recorded = await this.state.storage.get('recorded'); + return Response.json({ + count, + restores, + go, + recorded, + nodeId, + id: idHex, + }); + } else if (url.pathname === '/identity') { + return Response.json({ nodeId, id: idHex }); + } else if (url.pathname === '/store-stub') { + // Only stubs from the ctx.exports self-binding are storable. + const target = url.searchParams.get('target'); + const ns = this.state.exports.Counter; + const targetId = ns.idFromName(target); + await this.state.storage.put('stub', ns.get(targetId)); + // Read it back so that the stub is also deserialized, but do not call it. + const stub = await this.state.storage.get('stub'); + return Response.json({ + nodeId, + id: idHex, + targetId: targetId.toString(), + hasStub: typeof stub?.fetch === 'function', + }); + } else if (url.pathname === '/call-stored-stub') { + const stub = await this.state.storage.get('stub'); + const response = await stub.fetch('http://do/get'); + return Response.json({ + nodeId, + id: idHex, + target: await response.json(), + }); + } else if (url.pathname === '/hold-stub') { + const target = url.searchParams.get('target'); + const stub = this.env.COUNTER.get(this.env.COUNTER.idFromName(target)); + this.heldStubs.set(target, stub); + const response = await stub.fetch('http://do/identity'); + return Response.json({ + nodeId, + id: idHex, + target: await response.json(), + }); + } else if (url.pathname === '/call-held-stub') { + const target = url.searchParams.get('target'); + const stub = this.heldStubs.get(target); + if (stub === undefined) { + return Response.json({ error: 'no held stub' }, { status: 400 }); + } + const response = await stub.fetch('http://do/identity'); + return Response.json({ + nodeId, + id: idHex, + target: await response.json(), + }); + } else if (url.pathname === '/vend-stub') { + const kind = url.searchParams.get('kind'); + const target = url.searchParams.get('target'); + const ns = this.state.exports.Counter; + const stub = await ns.get(ns.idFromName(target)).vend(kind); + await this.state.storage.put(kind, stub); + // Read it back so that the stub is also deserialized, but do not call it. + const loaded = await this.state.storage.get(kind); + return Response.json({ + nodeId, + id: idHex, + hasStub: loaded !== undefined, + }); + } else if (url.pathname === '/use-stub') { + const kind = url.searchParams.get('kind'); + const stub = await this.state.storage.get(kind); + const result = await stub.add( + Number(url.searchParams.get('amount') ?? '1') + ); + return Response.json({ nodeId, id: idHex, result }); + } else if (url.pathname === '/vend-to-consumer') { + const adder = await this.state.restore({ kind: 'rpc' }); + const consumer = this.state.exports.Consumer({ props: { adder } }); + await consumer.useLater(); + return Response.json({ nodeId, id: idHex }); + } else if (url.pathname === '/set-go') { + await this.state.storage.put('go', true); + return Response.json({ nodeId, id: idHex }); + } else if (url.pathname === '/record') { + await this.state.storage.put( + 'recorded', + JSON.parse(url.searchParams.get('value')) + ); + return Response.json({ nodeId, id: idHex }); + } else if (url.pathname === '/hold') { + await scheduler.wait(1000); + return Response.json({ nodeId, id: idHex }); + } else if (url.pathname === '/set-alarm') { + try { + await this.state.storage.setAlarm(Date.now() + 60_000); + return Response.json({ set: true }); + } catch (e) { + return Response.json({ error: String(e) }); + } + } else if (url.pathname === '/break') { + this.state.abort('broken on purpose'); + } else if (url.pathname === '/arm-constructor-failure') { + await this.state.storage.put(BREAK_ON_CONSTRUCT, true); + await this.state.storage.sync(); + this.state.abort('armed constructor failure'); + } + + return new Response('Not found', { status: 404 }); + } + + // Present so `setAlarm` reaches the cluster's refusal. + async alarm() {} + + // RPC method used by /vend-stub on the *target* DO. + vend(kind) { + return this.state.restore({ kind }); + } + + async [restore](params) { + const restores = ((await this.state.storage.get('restores')) ?? 0) + 1; + await this.state.storage.put('restores', restores); + + if (params.kind === 'rpc') return new CounterAdder(this); + throw new Error(`unexpected restore params: ${JSON.stringify(params)}`); + } +} diff --git a/src/workerd/server/tests/cluster/fake-s3.js b/src/workerd/server/tests/cluster/fake-s3.js new file mode 100644 index 00000000000..5712066236d --- /dev/null +++ b/src/workerd/server/tests/cluster/fake-s3.js @@ -0,0 +1,115 @@ +// Copyright (c) 2026 Cloudflare, Inc. +// Licensed under the Apache 2.0 license found in the LICENSE file or at: +// https://opensource.org/licenses/Apache-2.0 +// +// The S3 calls a bucket cluster makes, for tests: GET, PUT (conditional on +// If-Match or If-None-Match: *), DELETE, and ListObjectsV2, path-style +// (`//`). One Durable Object holds every key, so a conditional +// write is atomic. Requests are not authenticated. + +import { DurableObject } from 'cloudflare:workers'; + +const PAGE = 1000; +// Storage values are capped at 128 KiB: a body is stored in chunks, under keys +// outside every listed prefix. +const CHUNK = 64 * 1024; +const chunkKey = (key, i) => `\0${key}\0${i}`; + +function escape(s) { + return s.replace(/&/g, '&').replace(//g, '>'); +} + +export class Bucket extends DurableObject { + next = 0; + // Requests per object key, for tests that bound a cluster's bucket traffic. + requests = new Map(); + + async fetch(request) { + const storage = this.ctx.storage; + const url = new URL(request.url); + if (url.pathname === '/_requests') { + return Response.json(this.requests.get(url.searchParams.get('key')) ?? 0); + } + const path = decodeURIComponent(url.pathname).slice(1); + const slash = path.indexOf('/'); + const key = slash < 0 ? '' : path.slice(slash + 1); + const headers = { date: new Date().toUTCString() }; + + if (request.method === 'GET' && key === '') { + const prefix = url.searchParams.get('prefix') ?? ''; + const after = url.searchParams.get('continuation-token'); + const listed = await storage.list({ + prefix, + ...(after === null ? {} : { startAfter: after }), + limit: PAGE + 1, + }); + const page = [...listed].slice(0, PAGE); + const truncated = listed.size > PAGE; + const xml = + '' + + `${truncated}` + + page + .map( + ([k, v]) => + `${escape(k)}${v.size}` + ) + .join('') + + (truncated + ? `${escape(page.at(-1)[0])}` + : '') + + ''; + return new Response(xml, { headers }); + } + + // Read the body before the object: storage calls keep other requests + // out, but awaiting the body would let one in between the check and the + // write. + const counted = key.slice(key.indexOf('/') + 1); + this.requests.set(counted, (this.requests.get(counted) ?? 0) + 1); + const body = request.method === 'PUT' ? await request.arrayBuffer() : null; + const object = await storage.get(key); + switch (request.method) { + case 'GET': + if (!object) return new Response(null, { status: 404, headers }); + const keys = Array.from({ length: object.chunks }, (_, i) => + chunkKey(key, i) + ); + const chunks = keys.length === 0 ? new Map() : await storage.get(keys); + const out = new Uint8Array(object.size); + keys.forEach((k, i) => + out.set(new Uint8Array(chunks.get(k)), i * CHUNK) + ); + return new Response(out, { + headers: { ...headers, etag: object.etag }, + }); + case 'PUT': { + const ifMatch = request.headers.get('if-match'); + const ifNoneMatch = request.headers.get('if-none-match'); + if ( + (ifNoneMatch === '*' && object) || + (ifMatch !== null && object?.etag !== ifMatch) + ) { + return new Response(null, { status: 412, headers }); + } + const etag = `"${Date.now()}-${this.next++}"`; + const chunks = Math.ceil(body.byteLength / CHUNK); + const entries = {}; + for (let i = 0; i < chunks; i++) + entries[chunkKey(key, i)] = body.slice(i * CHUNK, (i + 1) * CHUNK); + entries[key] = { etag, size: body.byteLength, chunks }; + await storage.put(entries); + return new Response(null, { headers: { ...headers, etag } }); + } + case 'DELETE': + await storage.delete(key); + return new Response(null, { status: 204, headers }); + } + return new Response(null, { status: 405, headers }); + } +} + +export default { + fetch(request, env) { + return env.BUCKET.get(env.BUCKET.idFromName('bucket')).fetch(request); + }, +}; diff --git a/src/workerd/server/tests/cluster/fake-s3.wd-test b/src/workerd/server/tests/cluster/fake-s3.wd-test new file mode 100644 index 00000000000..da369a03ef0 --- /dev/null +++ b/src/workerd/server/tests/cluster/fake-s3.wd-test @@ -0,0 +1,31 @@ +# Copyright (c) 2026 Cloudflare, Inc. +# Licensed under the Apache 2.0 license found in the LICENSE file or at: +# https://opensource.org/licenses/Apache-2.0 + +# The bucket for cluster-test: fake-s3.js on an ephemeral port, in memory. + +using Workerd = import "/workerd/workerd.capnp"; + +const config :Workerd.Config = ( + services = [ + ( name = "s3", + worker = ( + modules = [ + ( name = "fake-s3.js", esModule = embed "fake-s3.js" ), + ], + compatibilityDate = "2026-01-01", + bindings = [ + ( name = "BUCKET", durableObjectNamespace = "Bucket" ), + ], + durableObjectNamespaces = [ + ( className = "Bucket", uniqueKey = "fake-s3-bucket" ), + ], + durableObjectStorage = ( inMemory = void ), + ) + ), + ], + + sockets = [ + ( name = "http", address = "*:0", http = (), service = "s3" ), + ], +); diff --git a/src/workerd/server/workerd.capnp b/src/workerd/server/workerd.capnp index 94030079525..63457f6613c 100644 --- a/src/workerd/server/workerd.capnp +++ b/src/workerd/server/workerd.capnp @@ -94,6 +94,68 @@ struct Config { logging @6 : LoggingOptions; # Console and Stdio logging configuration options. + + cluster @7 :ClusterConfig; + # Set when this instance should join a cluster. Requires `--experimental`. +} + +struct ClusterConfig { + # workerd instances can work together in a cluster, to distribute Durable Objects among them. + # Each Durable Object runs on one instance at a time, its owner; any instance routes requests + # for it there. + # + # The cluster's shared state lives in an S3-compatible bucket: each instance renews a lease + # record naming its address, and each Durable Object's owner is a record claimed by a conditional + # write. Taking an object over from an owner whose lease expired bumps its ownership epoch. + # Instances need no shared filesystem: `durableObjectStorage.localDisk` is each instance's own + # disk. Durable Objects stored in memory stay local to each instance. The bucket's copy of an + # object replaces the instance's, so a `localDisk` already holding Durable Objects from before + # cluster mode is refused at startup; they are not imported. + # + # All instances in the cluster MUST have identical configuration: any instance may forward any + # request to any other and it must produce the same result. During a rolling upgrade, instances + # may differ in backwards-compatible ways. + + bucket @0 :Text; + # The bucket that holds the cluster's shared state: `s3://bucket/prefix`. + + bucketOptions @1 :List(Text); + # `key=value` settings for `bucket`: `aws_endpoint=https://...`, `aws_region=auto`, + # `aws_access_key_id=...`, `aws_secret_access_key=...`, and `aws_token=...`. In place of the + # last three, `aws_credentials_file=` names a file of `AWS_ACCESS_KEY_ID=`, + # `AWS_SECRET_ACCESS_KEY=` and `AWS_SESSION_TOKEN=` lines, reread whenever it changes, so + # short-lived credentials rotate without a restart. Replace it by rename, not in place. Each + # setting not given falls back to an environment variable: `AWS_ENDPOINT_URL`, `AWS_REGION`, + # `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, `AWS_SESSION_TOKEN`, `AWS_CREDENTIALS_FILE`. + + address @2 :Text; + # Where this instance listens for other instances: `host:port`, a port of 0 for an ephemeral + # one. Overridden by `--cluster-addr`. Holders of `key` get full access to every service in the + # process through it, and its traffic is not encrypted: only the cluster's instances should + # reach it, on a private network (a VPC, WireGuard, Tailscale). A public IP is refused unless + # `unsafePublicAddress` is set; a wildcard (`0.0.0.0`) listens on every interface, so firewall + # the port. + + advertise @4 :Text; + # The address other instances reach this one at, which goes in its lease record in the bucket. + # Optional: by default, the address bound, with the port it got. Required when that is a + # wildcard (`0.0.0.0:7000`) or not what peers can reach (NAT, a container). Overridden by + # `--cluster-advertise`. A public IP is refused unless `unsafePublicAddress` is set. + + key @3 :Text; + # A secret known only to members of the cluster, at least 32 bytes: random, such as the output + # of `openssl rand -hex 32`. Each connection between instances opens with + # a MAC keyed by it, over the target, a timestamp and a nonce, so a header seen on the wire + # cannot be replayed; the instances' clocks must agree within 30 seconds. It also keys channel + # tokens, which pass service and Durable Object stubs over RPC. Traffic is not encrypted: run + # the cluster on a private network (a VPC, WireGuard, Tailscale). + # + # TODO(someday): Support some way to gradually roll out a rotated key. + + unsafePublicAddress @5 :Bool = false; + # Allow a public IP as `address` or `advertise`. Refused by default, since traffic between + # instances is not encrypted. Private, loopback, link-local and shared (100.64.0.0/10, as + # Tailscale uses) addresses, wildcards and hostnames are allowed. } struct LoggingOptions { @@ -816,8 +878,8 @@ struct Worker { # extensions `.sqlite-wal`, and `.sqlite-shm` may also be present.) } - # TODO(someday): Support distributing objects across a cluster. At present, objects are always - # local to one instance of the runtime. + # In a cluster (`Config.cluster`), objects stored on `localDisk` are distributed across the + # instances; see `ClusterConfig`. moduleFallback @13 :Text; diff --git a/src/workerd/util/sqlite-kv-test.c++ b/src/workerd/util/sqlite-kv-test.c++ index 57f2e24b9f9..b092050ca53 100644 --- a/src/workerd/util/sqlite-kv-test.c++ +++ b/src/workerd/util/sqlite-kv-test.c++ @@ -338,5 +338,25 @@ KJ_TEST("SQLite-KV multi-put with allowUnconfirmed") { KJ_EXPECT(called); } +KJ_TEST("SQLite-KV reads throw after close()") { + auto dir = kj::newInMemoryDirectory(kj::nullClock()); + SqliteDatabase::Vfs vfs(*dir); + SqliteDatabase db(vfs, kj::Path({"foo"}), kj::WriteMode::CREATE | kj::WriteMode::MODIFY); + SqliteKv kv(db); + + kv.put("foo", "abc"_kj.asBytes()); + kv.putExternals("foo", kj::arr(kj::heapArray("tok"_kj.asBytes()))); + + db.close(); + + // Unlike reset(), close() must not make the data appear deleted: the file still has it. + KJ_EXPECT_THROW_MESSAGE("database has been closed", + kv.get("foo", [](kj::ArrayPtr) { KJ_FAIL_EXPECT("should not be called"); })); + KJ_EXPECT_THROW_MESSAGE( + "database has been closed", kv.list(nullptr, kj::none, kj::none, SqliteKv::FORWARD)->next()); + KJ_EXPECT_THROW_MESSAGE("database has been closed", kv.getExternals("foo")); + KJ_EXPECT_THROW_MESSAGE("database has been closed", kv.put("bar", "def"_kj.asBytes())); +} + } // namespace } // namespace workerd diff --git a/src/workerd/util/sqlite-kv.c++ b/src/workerd/util/sqlite-kv.c++ index 6dd044b8b30..047e8bf1e33 100644 --- a/src/workerd/util/sqlite-kv.c++ +++ b/src/workerd/util/sqlite-kv.c++ @@ -276,10 +276,16 @@ void SqliteKv::putExternals(kj::StringPtr key, kj::Array> tokens } } -void SqliteKv::beforeSqliteReset() { - // We'll need to recreate the tables on the next operation. - tableCreated = false; - externalsTableCreated = false; +void SqliteKv::beforeSqliteClose(SqliteDatabase::CloseReason reason) { + if (reason == SqliteDatabase::CloseReason::RESET) { + // The tables are deleted along with the database and will need to be recreated on the next + // operation. + tableCreated = false; + externalsTableCreated = false; + } + // On close(), leave the flags alone: the tables still exist in the file. Clearing them would + // make get()/list()/getExternals() report an empty database without consulting the (closed) + // connection; with the flags intact those reads go through a Statement and throw instead. } void SqliteKv::rollbackMultiPut(Initialized& stmts, WriteOptions options) { diff --git a/src/workerd/util/sqlite-kv.h b/src/workerd/util/sqlite-kv.h index a2e85670a41..815b25e302f 100644 --- a/src/workerd/util/sqlite-kv.h +++ b/src/workerd/util/sqlite-kv.h @@ -226,7 +226,7 @@ class SqliteKv: private SqliteDatabase::ResetListener { // with `allowUnconfirmed = true` since the paired KV write decides confirmation semantics. void clearExternalsIfPresent(KeyPtr key, SqliteQueryId queryId); - void beforeSqliteReset() override; + void beforeSqliteClose(SqliteDatabase::CloseReason reason) override; // Returns `queryId` if set, otherwise mints a fresh one. Used so every statement of a single // top-level KV operation shares one grouping id, while recursive operations (e.g. multi-put) diff --git a/src/workerd/util/sqlite-metadata-test.c++ b/src/workerd/util/sqlite-metadata-test.c++ index b153a44e781..f899f8c545f 100644 --- a/src/workerd/util/sqlite-metadata-test.c++ +++ b/src/workerd/util/sqlite-metadata-test.c++ @@ -65,5 +65,23 @@ KJ_TEST("SQLite-METADATA") { KJ_EXPECT(metadata.getAlarm() == anAlarmTime2); } +KJ_TEST("SQLite-METADATA reads throw after close()") { + auto dir = kj::newInMemoryDirectory(kj::nullClock()); + SqliteDatabase::Vfs vfs(*dir); + SqliteDatabase db(vfs, kj::Path({"foo"}), kj::WriteMode::CREATE | kj::WriteMode::MODIFY); + SqliteMetadata metadata(db); + + constexpr kj::Date anAlarmTime = kj::UNIX_EPOCH + 1734099316 * kj::SECONDS; + metadata.setAlarm(anAlarmTime, /*allowUnconfirmed=*/false); + KJ_EXPECT(metadata.getAlarm() == anAlarmTime); + + db.close(); + + // Unlike reset(), close() must not make the alarm appear unset: the file still has it. + KJ_EXPECT_THROW_MESSAGE("database has been closed", metadata.getAlarm()); + KJ_EXPECT_THROW_MESSAGE( + "database has been closed", metadata.setAlarm(kj::none, /*allowUnconfirmed=*/false)); +} + } // namespace } // namespace workerd diff --git a/src/workerd/util/sqlite-metadata.c++ b/src/workerd/util/sqlite-metadata.c++ index 6acdbdb2528..7cc4d5625a0 100644 --- a/src/workerd/util/sqlite-metadata.c++ +++ b/src/workerd/util/sqlite-metadata.c++ @@ -100,10 +100,18 @@ SqliteMetadata::Initialized& SqliteMetadata::ensureInitialized(bool allowUnconfi KJ_UNREACHABLE; } -void SqliteMetadata::beforeSqliteReset() { - // We'll need to recreate the table on the next operation. - tableCreated = false; +void SqliteMetadata::beforeSqliteClose(SqliteDatabase::CloseReason reason) { + // Drop the alarm cache in both cases. After a reset() the alarm is gone; after a close() the + // next read must reach the closed connection and throw rather than report a stale value. cacheState = kj::none; + + if (reason == SqliteDatabase::CloseReason::RESET) { + // The table is deleted along with the database and will need to be recreated on the next + // operation. Leave `tableCreated` alone on close(): the table still exists in the file, and + // clearing the flag would make getAlarmUncached() report "no alarm" without consulting the + // (closed) database. + tableCreated = false; + } } } // namespace workerd diff --git a/src/workerd/util/sqlite-metadata.h b/src/workerd/util/sqlite-metadata.h index c365166a213..9df50dbca6d 100644 --- a/src/workerd/util/sqlite-metadata.h +++ b/src/workerd/util/sqlite-metadata.h @@ -75,7 +75,7 @@ class SqliteMetadata final: private SqliteDatabase::ResetListener { // first write. // ResetListener interface: - void beforeSqliteReset() override; + void beforeSqliteClose(SqliteDatabase::CloseReason reason) override; }; } // namespace workerd diff --git a/src/workerd/util/sqlite.c++ b/src/workerd/util/sqlite.c++ index 9aef534579b..fc4380d5614 100644 --- a/src/workerd/util/sqlite.c++ +++ b/src/workerd/util/sqlite.c++ @@ -696,7 +696,11 @@ SqliteDatabase::~SqliteDatabase() noexcept(false) { } SqliteDatabase::operator sqlite3*() { - return &KJ_ASSERT_NONNULL(maybeDb, "previous reset() failed"); + KJ_IF_SOME(db, maybeDb) { + return &db; + } + KJ_REQUIRE(!closed, "database has been closed"); + KJ_FAIL_ASSERT("previous reset() failed"); } SqliteMemoryScope SqliteDatabase::enterMemoryScope() { @@ -722,7 +726,7 @@ void SqliteDatabase::handleCriticalError(kj::Maybe errorCode, if (code == SQLITE_FULL || code == SQLITE_IOERR || code == SQLITE_NOMEM || code == SQLITE_INTERRUPT) { - sqlite3* db = &KJ_ASSERT_NONNULL(maybeDb, "previous reset() failed"); + sqlite3* db = *this; // We are in a transaction if (inTransaction || !savepoints.empty()) { // The transaction was auto-rolledback, re-enabling the auto commit mode, so we should fail @@ -847,7 +851,7 @@ SqliteDatabase::StatementAndEffect SqliteDatabase::prepareSql(StaticRegulator re Multi multi, kj::Maybe&> prelude, kj::Maybe queryId) { - sqlite3* db = &KJ_ASSERT_NONNULL(maybeDb, "previous reset() failed"); + sqlite3* db = *this; ParseContext parseContext; KJ_ASSERT(currentParseContext == kj::none, "recursive prepareSql()?"); @@ -1054,6 +1058,7 @@ void SqliteDatabase::executeWithRegulator( void SqliteDatabase::reset() { KJ_REQUIRE(!readOnly, "can't reset() read-only database"); + KJ_REQUIRE(!closed, "can't reset() a database that has been closed"); // If transactions are open during reset(), whatever had the transaction open is going to get // confused at best, or lose data at worst. Let's just not allow this. @@ -1067,7 +1072,7 @@ void SqliteDatabase::reset() { KJ_IF_SOME(db, maybeDb) { for (auto& listener: resetListeners) { - listener.beforeSqliteReset(); + listener.beforeSqliteClose(CloseReason::RESET); } auto err = sqlite3_close(&db); @@ -1086,6 +1091,44 @@ void SqliteDatabase::reset() { } } +void SqliteDatabase::close() noexcept { + if (closed) return; + + // If we're currently executing a SQLite statement, there's no safe way for us to close the DB. + // But, callers of close() rely on the fact that the DB will in fact be closed, and they often + // sit on error-handling paths that cannot tolerate further exceptions. These asserts, if they + // fail, will abort the process, due to close() being noexcept. As of this writing it is believed + // that no real code path is likely to hit these, so these are just safety checks / internal + // invariants. + KJ_ASSERT(currentStatement == kj::none, "close() called while a query is executing"); + KJ_ASSERT(currentRegulator == kj::none, "close() called while a query is compiling"); + + auto memoryScope = enterMemoryScope(); + + KJ_IF_SOME(db, maybeDb) { + for (auto& listener: resetListeners) { + listener.beforeSqliteClose(CloseReason::CLOSE); + } + + // Every prepared statement was finalized by the listener walk above, so SQLITE_BUSY here + // indicates a statement that isn't tracked by a ResetListener. Do not fall back to + // sqlite3_close_v2(): a lazily-closed handle keeps the file open, which is exactly what this + // method exists to prevent. + auto err = sqlite3_close(&db); + KJ_ASSERT( + err == SQLITE_OK, "close() found dependent objects that still exist", sqlite3_errstr(err)); + + maybeDb = kj::none; + } + + // sqlite3_close() rolled back any open transaction. Nothing may read the database again, so the + // rollback callbacks (which exist to keep in-memory caches in sync with it) are simply dropped. + inTransaction = false; + savepoints.clear(); + rollbackCallbacks.clear(); + closed = true; +} + bool SqliteDatabase::isAuthorized(int actionCode, kj::Maybe param1, kj::Maybe param2, @@ -1550,10 +1593,22 @@ SqliteDatabase::StatementAndEffect& SqliteDatabase::Statement::prepareForExecuti return KJ_ASSERT_NONNULL(stmt.tryGet()); } -void SqliteDatabase::Statement::beforeSqliteReset() { +void SqliteDatabase::Statement::beforeSqliteClose(CloseReason reason) { + // Regardless of `reason`, the prepared statement must be finalized (by destroying the + // StatementAndEffect) so that the connection can be closed. KJ_IF_SOME(prepared, stmt.tryGet()) { - // Pull the original SQL code out of the statement and store it. - stmt = kj::str(sqlite3_sql(prepared.statement)); + switch (reason) { + case CloseReason::RESET: + // Keep the SQL text so that the statement can be reprepared against the new database. + stmt = kj::str(sqlite3_sql(prepared.statement)); + break; + case CloseReason::CLOSE: + // The statement can never be reprepared, so don't bother copying the SQL. An empty string + // (no allocation) suffices: prepareForExecution() will attempt to prepare it and fail + // with "database has been closed" before ever looking at the text. + stmt = kj::String(); + break; + } } } @@ -1856,15 +1911,20 @@ bool SqliteDatabase::Query::isNull(uint column) { SqliteDatabase::StatementAndEffect& SqliteDatabase::Query::getStatementAndEffect() { return KJ_UNWRAP_OR(maybeStatement, { - regulator->onError(kj::none, "SQLite query was canceled because the database was deleted."); - KJ_FAIL_REQUIRE("query canceled because reset() was called on the database"); + if (db.closed) { + regulator->onError(kj::none, "SQLite query was canceled because the database was closed."); + KJ_FAIL_REQUIRE("query canceled because the database has been closed"); + } else { + regulator->onError(kj::none, "SQLite query was canceled because the database was deleted."); + KJ_FAIL_REQUIRE("query canceled because reset() was called on the database"); + } }); } -void SqliteDatabase::Query::beforeSqliteReset() { +void SqliteDatabase::Query::beforeSqliteClose(CloseReason reason) { // Note that if we don't own the statement, then `maybeStatement` is probably already dangling // here. Luckily, we don't need to reset it or anything because the statement will be destroyed - // by Statement::beforeSqliteReset(). + // by Statement::beforeSqliteClose(). maybeStatement = kj::none; ownStatement = {}; } diff --git a/src/workerd/util/sqlite.h b/src/workerd/util/sqlite.h index bbe1e538948..c51fe1e46f8 100644 --- a/src/workerd/util/sqlite.h +++ b/src/workerd/util/sqlite.h @@ -307,7 +307,31 @@ class SqliteDatabase { // may throw if they depend on tables that haven't been recreated yet). void reset(); - // Objects that need to be notified when reset() is called may inherit `ResetListener`. + // Closes the underlying SQLite connection without reopening it. Every subsequent attempt to + // use the database, including through existing Statements and Queries, throws. Any open + // transaction is rolled back (never committed). Idempotent. + // + // This exists so that an actor's storage can be severed at the moment the actor is shut down, + // even if references to the SqliteDatabase live on: closing the last connection checkpoints and + // deletes the WAL and releases any file locks, which the next opener of the same file needs. + // + // This is declared noexcept because it is expected to be used in shutdown paths which are not + // prepared to catch further errors and strongly depend on the database actually becoming closed + // (so that they can safely release file locks). If close() is unable to complete (e.g. because + // a sqlite statement is currently being executed) then we have no choice but to abort the + // process. + void close() noexcept; + + // Why a connection is being closed. Passed to `ResetListener::beforeSqliteClose()`. + enum class CloseReason { + // reset(): the database file is deleted and a fresh connection opened. + RESET, + // close(): the connection is gone for good. + CLOSE, + }; + + // Objects that need to be notified when reset() or close() is called may inherit + // `ResetListener`. class ResetListener { public: ResetListener(SqliteDatabase& db): db(db) { @@ -321,9 +345,16 @@ class SqliteDatabase { db.resetListeners.add(*this); } - // When the database's `reset()` method is called, all listeners' `beforeSqliteReset()` will be - // called before actually resetting the database. - virtual void beforeSqliteReset() = 0; + // When the database's `reset()` or `close()` method is called, all listeners' + // `beforeSqliteClose()` will be called before actually closing the connection. Any prepared + // statements must be finalized here so that `sqlite3_close()` can succeed. + // + // The `reason` matters for listeners that cache facts about the database's contents: after + // RESET the database is empty and will be reopened, so such caches must be cleared; after + // CLOSE nothing may read the database again, so such caches should be left alone so that + // attempted reads reach the closed connection and throw rather than reporting an empty + // database. + virtual void beforeSqliteClose(CloseReason reason) = 0; protected: // so that subclasses don't have to store their own copy of the `db` reference SqliteDatabase& db; @@ -357,6 +388,11 @@ class SqliteDatabase { // When a rollback occurs, callbacks are invoked in the reverse of the order in which they were // registered. The database content is rolled back first, before invoking any callbacks. // Callbacks may read from the database, but must not write to it. + // + // NOTE: close() technically rolls back all outstanding transactions, but does NOT call rollback + // callbacks. Instead, callers who need to handle the close() case should register a + // ResetListener. In practices, in-memory caches are generally moot after close() since the + // actor has shut down anyway. void onRollback(kj::Function callback) { if (inTransaction || !savepoints.empty()) { rollbackCallbacks.add(kj::mv(callback)); @@ -388,9 +424,12 @@ class SqliteDatabase { kj::Maybe actorAccountLimits; - // This pointer can be left null if a call to reset() failed to re-open the database. + // Null after a call to reset() failed to re-open the database, or after close(). kj::Maybe maybeDb; + // Set by close(). Distinguishes the closed state from a failed reset() in error messages. + bool closed = false; + // Set while a query is compiling. kj::Maybe currentRegulator; @@ -560,7 +599,7 @@ class SqliteDatabase::Statement final: private ResetListener { regulator(regulator), stmt(kj::mv(sqlCode)) {} - void beforeSqliteReset() override; + void beforeSqliteClose(CloseReason reason) override; // Get the underlying StatementAndEffect, which the caller will then execute. If `prelude` is // non-empty, prepareForExecution() actually executes the prelude. @@ -833,7 +872,7 @@ class SqliteDatabase::Query final: private ResetListener { return getStatementAndEffect().statement; } - void beforeSqliteReset() override; + void beforeSqliteClose(CloseReason reason) override; void nextRow(bool first); };