From c6e7f71b6f28f8f6c331491469648037a3ea05e9 Mon Sep 17 00:00:00 2001 From: lb <542828+lukebuehler@users.noreply.github.com> Date: Mon, 2 Nov 2026 16:38:00 +0100 Subject: [PATCH] fixes for aks --- Cargo.lock | 13 +- Cargo.toml | 2 +- crates/store-pg/Cargo.toml | 4 +- crates/store-pg/src/azure.rs | 87 ++++++ crates/store-pg/src/lib.rs | 73 +---- crates/store-pg/src/s3.rs | 83 ++++++ crates/store-pg/tests/azure_blob_live.rs | 163 +++++++++++ crates/store-pg/tests/postgres_tls_live.rs | 61 ++++ crates/temporal-runtime/Cargo.toml | 2 +- crates/temporal-runtime/src/config.rs | 56 +--- .../src/config/object_storage.rs | 277 ++++++++++++++++++ .../documentation/deployment/configuration.md | 35 ++- docs/documentation/deployment/self-hosting.md | 4 +- .../deployment/troubleshooting.md | 2 +- .../reference/environment-variables.md | 29 +- .../roadmap/azure-storage-and-postgres-tls.md | 54 ++++ release/metadata.env | 2 +- release/notes/v0.5.1.md | 21 ++ 18 files changed, 824 insertions(+), 144 deletions(-) create mode 100644 crates/store-pg/src/azure.rs create mode 100644 crates/store-pg/src/s3.rs create mode 100644 crates/store-pg/tests/azure_blob_live.rs create mode 100644 crates/store-pg/tests/postgres_tls_live.rs create mode 100644 crates/temporal-runtime/src/config/object_storage.rs create mode 100644 docs/roadmap/azure-storage-and-postgres-tls.md create mode 100644 release/notes/v0.5.1.md diff --git a/Cargo.lock b/Cargo.lock index 96e00e0be..18e1a0a37 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -687,7 +687,7 @@ checksum = "1c133bc6a41be0d194c306b5506d15e6feeea7b1d6604bd3f8310dfb2ca96486" [[package]] name = "cli" -version = "0.5.0" +version = "0.5.1" dependencies = [ "anyhow", "api", @@ -1316,7 +1316,7 @@ dependencies = [ [[package]] name = "environment-daemon" -version = "0.5.0" +version = "0.5.1" dependencies = [ "anyhow", "clap", @@ -1371,7 +1371,7 @@ dependencies = [ [[package]] name = "environment-provider-incus" -version = "0.5.0" +version = "0.5.1" dependencies = [ "anyhow", "async-trait", @@ -3161,6 +3161,7 @@ dependencies = [ "futures-util", "http", "http-body-util", + "httparse", "humantime", "hyper", "itertools 0.15.0", @@ -4167,7 +4168,7 @@ dependencies = [ [[package]] name = "release-info" -version = "0.5.0" +version = "0.5.1" dependencies = [ "tempfile", ] @@ -4857,6 +4858,8 @@ dependencies = [ "log", "memchr", "percent-encoding", + "rustls", + "rustls-native-certs", "serde", "serde_json", "sha2 0.10.9", @@ -5209,7 +5212,7 @@ dependencies = [ [[package]] name = "temporal-runtime" -version = "0.5.0" +version = "0.5.1" dependencies = [ "anyhow", "api", diff --git a/Cargo.toml b/Cargo.toml index 41a871766..2b1447c36 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,4 +30,4 @@ members = [ resolver = "2" [workspace.package] -version = "0.5.0" +version = "0.5.1" diff --git a/crates/store-pg/Cargo.toml b/crates/store-pg/Cargo.toml index e2d42aa48..aa4c42cfd 100644 --- a/crates/store-pg/Cargo.toml +++ b/crates/store-pg/Cargo.toml @@ -18,13 +18,13 @@ environment-protocol = { path = "../environment-protocol" } futures-util = "0.3" async-trait = "0.1" mcp = { path = "../mcp" } -object_store = { version = "0.14", features = ["aws"] } +object_store = { version = "0.14", features = ["aws", "azure"] } profiles = { path = "../profiles" } rand = "0.10" sha2 = "0.11" serde = { version = "1", features = ["derive"] } serde_json = "1" -sqlx = { version = "0.9", default-features = false, features = ["runtime-tokio", "postgres", "uuid", "json"] } +sqlx = { version = "0.9", default-features = false, features = ["runtime-tokio", "tls-rustls-ring-native-roots", "postgres", "uuid", "json"] } thiserror = "2" tracing = "0.1" tokio = { version = "1", features = ["rt"] } diff --git a/crates/store-pg/src/azure.rs b/crates/store-pg/src/azure.rs new file mode 100644 index 000000000..50516d540 --- /dev/null +++ b/crates/store-pg/src/azure.rs @@ -0,0 +1,87 @@ +use std::{fmt, sync::Arc}; + +use object_store::{ + ObjectStore, + azure::{AzureConfigKey, MicrosoftAzureBuilder}, +}; + +/// Static Azure credentials supplied by the deployment. Debug output is redacted. +#[derive(Clone)] +pub enum AzureObjectStoreCredential { + AccessKey(String), + SasToken(String), +} + +impl fmt::Debug for AzureObjectStoreCredential { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str(match self { + Self::AccessKey(_) => "AccessKey([REDACTED])", + Self::SasToken(_) => "SasToken([REDACTED])", + }) + } +} + +#[derive(Clone)] +pub struct AzureObjectStoreConfig { + pub account: String, + pub container: String, + pub credential: AzureObjectStoreCredential, + /// Account endpoint, including the account path when using an emulator. + pub endpoint: Option, +} + +impl fmt::Debug for AzureObjectStoreConfig { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("AzureObjectStoreConfig") + .field("account", &self.account) + .field("container", &self.container) + .field("credential", &self.credential) + .field("endpoint_configured", &self.endpoint.is_some()) + .finish() + } +} + +/// Build the native Azure backend without changing CAS keys or metadata. +/// Credentials are explicit so a missing Secret cannot fall back to another identity. +pub fn build_azure_object_store( + config: AzureObjectStoreConfig, +) -> Result, object_store::Error> { + let mut builder = MicrosoftAzureBuilder::new() + .with_account(config.account) + .with_container_name(config.container); + builder = match config.credential { + AzureObjectStoreCredential::AccessKey(key) => builder.with_access_key(key), + AzureObjectStoreCredential::SasToken(token) => { + builder.with_config(AzureConfigKey::SasKey, token.trim_start_matches('?')) + } + }; + if let Some(endpoint) = config.endpoint { + builder = builder + .with_allow_http(endpoint.starts_with("http://")) + .with_endpoint(endpoint); + } + Ok(Arc::new(builder.build()?)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn azure_config_debug_redacts_both_credential_types_and_endpoint() { + for credential in [ + AzureObjectStoreCredential::AccessKey("secret-key".into()), + AzureObjectStoreCredential::SasToken("sig=secret-sas".into()), + ] { + let config = AzureObjectStoreConfig { + account: "account".into(), + container: "container".into(), + credential, + endpoint: Some("https://secret-endpoint.example".into()), + }; + let debug = format!("{config:?}"); + assert!(!debug.contains("secret-")); + assert!(debug.contains("[REDACTED]")); + } + } +} diff --git a/crates/store-pg/src/lib.rs b/crates/store-pg/src/lib.rs index c666ec378..0e4977f90 100644 --- a/crates/store-pg/src/lib.rs +++ b/crates/store-pg/src/lib.rs @@ -6,6 +6,7 @@ mod access; mod api_keys; mod auth; +mod azure; mod blob; mod blob_cache; mod bots; @@ -21,6 +22,7 @@ mod oauth; mod object; mod profile; mod providers; +mod s3; mod session; pub use session::PurgeSessionError; mod shared; @@ -31,14 +33,15 @@ use std::sync::Arc; use base64::Engine as _; use object_store::ObjectStore; -use object_store::aws::AmazonS3Builder; use sqlx::{PgPool, postgres::PgPoolOptions}; use thiserror::Error; use uuid::Uuid; pub use access::AccessFilter; pub use access::{AccessStoreError, PgAccessStore, ResourceAccess}; +pub use azure::{AzureObjectStoreConfig, AzureObjectStoreCredential, build_azure_object_store}; pub use model_defaults::ModelDefaultsStoreError; +pub use s3::{S3ObjectStoreConfig, build_s3_object_store}; /// A session page with the access summary each view carries. #[derive(Clone, Debug)] @@ -139,74 +142,6 @@ impl PgStoreConfig { } } -#[derive(Clone, Debug, PartialEq, Eq)] -pub struct S3ObjectStoreConfig { - pub bucket: String, - pub endpoint: Option, - pub region: String, - pub access_key_id: Option, - pub secret_access_key: Option, - pub force_path_style: bool, -} - -impl S3ObjectStoreConfig { - pub fn new(bucket: impl Into) -> Self { - Self { - bucket: bucket.into(), - endpoint: None, - region: "us-east-1".to_owned(), - access_key_id: None, - secret_access_key: None, - force_path_style: false, - } - } - - pub fn with_endpoint(mut self, endpoint: impl Into) -> Self { - self.endpoint = Some(endpoint.into()); - self - } - - pub fn with_region(mut self, region: impl Into) -> Self { - self.region = region.into(); - self - } - - pub fn with_access_key_id(mut self, access_key_id: impl Into) -> Self { - self.access_key_id = Some(access_key_id.into()); - self - } - - pub fn with_secret_access_key(mut self, secret_access_key: impl Into) -> Self { - self.secret_access_key = Some(secret_access_key.into()); - self - } - - pub fn with_force_path_style(mut self, force_path_style: bool) -> Self { - self.force_path_style = force_path_style; - self - } -} - -pub fn build_s3_object_store( - config: S3ObjectStoreConfig, -) -> Result, object_store::Error> { - let mut builder = AmazonS3Builder::new() - .with_bucket_name(config.bucket) - .with_region(config.region) - .with_virtual_hosted_style_request(!config.force_path_style); - if let Some(endpoint) = config.endpoint { - let allow_http = endpoint.starts_with("http://"); - builder = builder.with_endpoint(endpoint).with_allow_http(allow_http); - } - if let Some(access_key_id) = config.access_key_id { - builder = builder.with_access_key_id(access_key_id); - } - if let Some(secret_access_key) = config.secret_access_key { - builder = builder.with_secret_access_key(secret_access_key); - } - Ok(Arc::new(builder.build()?)) -} - #[derive(Clone)] pub struct PgStore { pub(crate) pool: PgPool, diff --git a/crates/store-pg/src/s3.rs b/crates/store-pg/src/s3.rs new file mode 100644 index 000000000..161f40586 --- /dev/null +++ b/crates/store-pg/src/s3.rs @@ -0,0 +1,83 @@ +use std::{fmt, sync::Arc}; + +use object_store::{ObjectStore, aws::AmazonS3Builder}; + +#[derive(Clone, PartialEq, Eq)] +pub struct S3ObjectStoreConfig { + pub bucket: String, + pub endpoint: Option, + pub region: String, + pub access_key_id: Option, + pub secret_access_key: Option, + pub force_path_style: bool, +} + +impl fmt::Debug for S3ObjectStoreConfig { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("S3ObjectStoreConfig") + .field("bucket", &self.bucket) + .field("region", &self.region) + .field("endpoint_configured", &self.endpoint.is_some()) + .field("credentials_configured", &self.secret_access_key.is_some()) + .field("force_path_style", &self.force_path_style) + .finish() + } +} + +impl S3ObjectStoreConfig { + pub fn new(bucket: impl Into) -> Self { + Self { + bucket: bucket.into(), + endpoint: None, + region: "us-east-1".to_owned(), + access_key_id: None, + secret_access_key: None, + force_path_style: false, + } + } + + pub fn with_endpoint(mut self, endpoint: impl Into) -> Self { + self.endpoint = Some(endpoint.into()); + self + } + + pub fn with_region(mut self, region: impl Into) -> Self { + self.region = region.into(); + self + } + + pub fn with_access_key_id(mut self, access_key_id: impl Into) -> Self { + self.access_key_id = Some(access_key_id.into()); + self + } + + pub fn with_secret_access_key(mut self, secret_access_key: impl Into) -> Self { + self.secret_access_key = Some(secret_access_key.into()); + self + } + + pub fn with_force_path_style(mut self, force_path_style: bool) -> Self { + self.force_path_style = force_path_style; + self + } +} + +pub fn build_s3_object_store( + config: S3ObjectStoreConfig, +) -> Result, object_store::Error> { + let mut builder = AmazonS3Builder::new() + .with_bucket_name(config.bucket) + .with_region(config.region) + .with_virtual_hosted_style_request(!config.force_path_style); + if let Some(endpoint) = config.endpoint { + let allow_http = endpoint.starts_with("http://"); + builder = builder.with_endpoint(endpoint).with_allow_http(allow_http); + } + if let Some(access_key_id) = config.access_key_id { + builder = builder.with_access_key_id(access_key_id); + } + if let Some(secret_access_key) = config.secret_access_key { + builder = builder.with_secret_access_key(secret_access_key); + } + Ok(Arc::new(builder.build()?)) +} diff --git a/crates/store-pg/tests/azure_blob_live.rs b/crates/store-pg/tests/azure_blob_live.rs new file mode 100644 index 000000000..6fdade59d --- /dev/null +++ b/crates/store-pg/tests/azure_blob_live.rs @@ -0,0 +1,163 @@ +//! Explicit Azure/Azurite and PostgreSQL integration. Never reads .env. +use std::{env, panic::AssertUnwindSafe, str::FromStr, sync::Arc}; + +use futures_util::{FutureExt as _, TryStreamExt as _}; +use harness::{ + BlobRef, + storage::{BlobStore as _, BlobStoreError}, +}; +use object_store::{ObjectStore, ObjectStoreExt as _}; +use sqlx::{ + Executor as _, + postgres::{PgConnectOptions, PgPoolOptions}, +}; +use store_pg::{ + AzureObjectStoreConfig, AzureObjectStoreCredential, PgStore, PgStoreConfig, + build_azure_object_store, delete_objects_under_prefix, universe_cas_object_prefix, +}; +use uuid::Uuid; + +fn azure_config() -> AzureObjectStoreConfig { + let credential = match ( + env::var("LIGHTSPEED_TEST_AZURE_ACCESS_KEY").ok(), + env::var("LIGHTSPEED_TEST_AZURE_SAS_TOKEN").ok(), + ) { + (Some(key), None) => AzureObjectStoreCredential::AccessKey(key), + (None, Some(token)) => AzureObjectStoreCredential::SasToken(token), + _ => panic!( + "set exactly one of LIGHTSPEED_TEST_AZURE_ACCESS_KEY or LIGHTSPEED_TEST_AZURE_SAS_TOKEN" + ), + }; + AzureObjectStoreConfig { + account: env::var("LIGHTSPEED_TEST_AZURE_ACCOUNT") + .expect("explicit test Azure account required"), + container: env::var("LIGHTSPEED_TEST_AZURE_CONTAINER") + .expect("explicit test Azure container required"), + endpoint: env::var("LIGHTSPEED_TEST_AZURE_ENDPOINT").ok(), + credential, + } +} + +#[tokio::test(flavor = "current_thread")] +#[ignore = "requires explicit LIGHTSPEED_TEST_POSTGRES_URL and LIGHTSPEED_TEST_AZURE_*; uses a unique schema and object prefix"] +async fn azure_blobs_survive_reconnection_and_remain_isolated_between_universes() { + let url = + env::var("LIGHTSPEED_TEST_POSTGRES_URL").expect("explicit test PostgreSQL URL required"); + let config = azure_config(); + let objects = build_azure_object_store(config.clone()).unwrap(); + let admin = PgPoolOptions::new() + .max_connections(1) + .connect(&url) + .await + .unwrap(); + let id = Uuid::new_v4().simple().to_string(); + let schema = format!("azure_blobs_{id}"); + let prefix = format!("lightspeed-tests/{id}"); + admin + .execute(sqlx::raw_sql(sqlx::AssertSqlSafe(format!( + "CREATE SCHEMA \"{schema}\"" + )))) + .await + .unwrap(); + let options = PgConnectOptions::from_str(&url) + .unwrap() + .options([("search_path", schema.as_str())]); + + let outcome = AssertUnwindSafe(exercise(options, config, objects.clone(), &prefix)) + .catch_unwind() + .await; + delete_objects_under_prefix(objects.as_ref(), &prefix) + .await + .expect("clean up test blobs"); + admin + .execute(sqlx::raw_sql(sqlx::AssertSqlSafe(format!( + "DROP SCHEMA \"{schema}\" CASCADE" + )))) + .await + .unwrap(); + admin.close().await; + if let Err(panic) = outcome { + std::panic::resume_unwind(panic); + } +} + +async fn exercise( + options: PgConnectOptions, + azure: AzureObjectStoreConfig, + objects: Arc, + prefix: &str, +) { + let pool = PgPoolOptions::new() + .max_connections(2) + .connect_with(options.clone()) + .await + .unwrap(); + PgStore::migrate(&pool).await.unwrap(); + let first_config = PgStoreConfig::new(Uuid::new_v4()).with_object_prefix(prefix); + let second_config = PgStoreConfig::new(Uuid::new_v4()).with_object_prefix(prefix); + let first = PgStore::with_object_store(pool.clone(), objects.clone(), first_config.clone()); + let second = PgStore::with_object_store(pool.clone(), objects.clone(), second_config.clone()); + first.ensure_universe().await.unwrap(); + second.ensure_universe().await.unwrap(); + let payload = vec![b'x'; 128 * 1024]; + let reference = first.put_bytes(payload.clone()).await.unwrap(); + assert_eq!(first.read_bytes(&reference).await.unwrap(), payload); + assert!(matches!( + second.read_bytes(&reference).await, + Err(BlobStoreError::NotFound { .. }) + )); + assert_eq!(second.put_bytes(payload.clone()).await.unwrap(), reference); + let kinds: Vec = + sqlx::query_scalar("SELECT storage_kind FROM cas_blobs WHERE digest = $1") + .bind(reference.as_str().strip_prefix("sha256:").unwrap()) + .fetch_all(&pool) + .await + .unwrap(); + assert_eq!(kinds, vec!["object", "object"]); + drop(first); + drop(second); + pool.close().await; + + // Recreate both database and Azure clients; no previous process cache may satisfy the read. + let objects = build_azure_object_store(azure).unwrap(); + let pool = PgPoolOptions::new() + .max_connections(2) + .connect_with(options) + .await + .unwrap(); + let first = PgStore::with_object_store(pool.clone(), objects.clone(), first_config.clone()); + let second = PgStore::with_object_store(pool.clone(), objects.clone(), second_config.clone()); + assert_eq!(first.read_bytes(&reference).await.unwrap(), payload); + assert_eq!(second.read_bytes(&reference).await.unwrap(), payload); + assert!(matches!( + first.read_bytes(&BlobRef::from_bytes(b"missing")).await, + Err(BlobStoreError::NotFound { .. }) + )); + + let first_prefix = universe_cas_object_prefix(prefix, first_config.universe_id); + let second_prefix = universe_cas_object_prefix(prefix, second_config.universe_id); + let listed = objects + .list(Some(&object_store::path::Path::from(first_prefix.as_str()))) + .try_collect::>() + .await + .unwrap(); + assert_eq!(listed.len(), 1); + assert_eq!( + delete_objects_under_prefix(objects.as_ref(), &first_prefix) + .await + .unwrap(), + 1 + ); + assert!(matches!( + objects.head(&listed[0].location).await, + Err(object_store::Error::NotFound { .. }) + )); + assert_eq!(second.read_bytes(&reference).await.unwrap(), payload); + assert_eq!( + delete_objects_under_prefix(objects.as_ref(), &second_prefix) + .await + .unwrap(), + 1 + ); + pool.close().await; +} diff --git a/crates/store-pg/tests/postgres_tls_live.rs b/crates/store-pg/tests/postgres_tls_live.rs new file mode 100644 index 000000000..3e0b323c0 --- /dev/null +++ b/crates/store-pg/tests/postgres_tls_live.rs @@ -0,0 +1,61 @@ +//! Explicit TLS-only PostgreSQL verification. Never reads .env or changes schemas. +use std::{env, str::FromStr, time::Duration}; + +use sqlx::postgres::{PgConnectOptions, PgPoolOptions, PgSslMode}; + +#[tokio::test(flavor = "current_thread")] +#[ignore = "requires LIGHTSPEED_TEST_POSTGRES_TLS_URL, LIGHTSPEED_TEST_POSTGRES_WRONG_CA_FILE, and LIGHTSPEED_TEST_POSTGRES_WRONG_HOST"] +async fn verify_full_requires_a_trusted_ca_and_matching_hostname() { + let url = env::var("LIGHTSPEED_TEST_POSTGRES_TLS_URL") + .expect("explicit TLS-required test PostgreSQL URL required"); + let options = PgConnectOptions::from_str(&url).unwrap(); + assert!(matches!(options.get_ssl_mode(), PgSslMode::VerifyFull)); + let pool = PgPoolOptions::new() + .max_connections(1) + .connect_with(options.clone()) + .await + .expect("connect over verified TLS"); + let encrypted: bool = + sqlx::query_scalar("SELECT ssl FROM pg_stat_ssl WHERE pid = pg_backend_pid()") + .fetch_one(&pool) + .await + .unwrap(); + assert!(encrypted); + pool.close().await; + + let wrong_ca = env::var("LIGHTSPEED_TEST_POSTGRES_WRONG_CA_FILE") + .expect("path to a valid but unrelated CA certificate required"); + assert!( + std::fs::read_to_string(&wrong_ca) + .unwrap() + .contains("BEGIN CERTIFICATE") + ); + let error = PgPoolOptions::new() + .acquire_timeout(Duration::from_secs(10)) + .connect_with(options.clone().ssl_root_cert(wrong_ca)) + .await + .unwrap_err(); + assert_certificate_failure(&error); + + let wrong_host = env::var("LIGHTSPEED_TEST_POSTGRES_WRONG_HOST") + .expect("test server address not covered by its certificate required"); + let error = PgPoolOptions::new() + .acquire_timeout(Duration::from_secs(10)) + .connect_with(options.host(&wrong_host)) + .await + .unwrap_err(); + assert_certificate_failure(&error); +} + +fn assert_certificate_failure(error: &sqlx::Error) { + // Handshake validation errors may be wrapped by SQLx as invalid-data I/O. + let certificate_failure = match error { + sqlx::Error::Tls(_) => true, + sqlx::Error::Io(error) => error.kind() == std::io::ErrorKind::InvalidData, + _ => false, + }; + assert!( + certificate_failure, + "expected certificate validation failure: {error:?}" + ); +} diff --git a/crates/temporal-runtime/Cargo.toml b/crates/temporal-runtime/Cargo.toml index c6ffa87fc..4d4776635 100644 --- a/crates/temporal-runtime/Cargo.toml +++ b/crates/temporal-runtime/Cargo.toml @@ -35,7 +35,7 @@ llm-clients = { path = "../llm-clients" } llm-runtime = { path = "../llm-runtime" } mcp = { path = "../mcp" } object_store = "0.14" -sqlx = { version = "0.9", default-features = false, features = ["runtime-tokio", "postgres", "uuid"] } +sqlx = { version = "0.9", default-features = false, features = ["runtime-tokio", "tls-rustls-ring-native-roots", "postgres", "uuid"] } profiles = { path = "../profiles" } release-info = { path = "../release-info" } reqwest = { version = "0.13", default-features = false, features = ["json", "rustls", "stream"] } diff --git a/crates/temporal-runtime/src/config.rs b/crates/temporal-runtime/src/config.rs index 9ee9f059f..3a4744ff0 100644 --- a/crates/temporal-runtime/src/config.rs +++ b/crates/temporal-runtime/src/config.rs @@ -2,13 +2,12 @@ use std::{env, sync::Arc, time::Duration}; use object_store::ObjectStore; use sqlx::{PgPool, postgres::PgPoolOptions}; -use store_pg::{ - BlobCache, PgStore, PgStoreConfig, PgStoreError, S3ObjectStoreConfig, SchemaStatus, - SecretsMasterKey, build_s3_object_store, -}; +use store_pg::{BlobCache, PgStore, PgStoreConfig, PgStoreError, SchemaStatus, SecretsMasterKey}; use temporal_workflow::{DEFAULT_TASK_QUEUE, bots::DEFAULT_BOTS_TASK_QUEUE}; use uuid::Uuid; +mod object_storage; + /// Retired deployment model policy must be migrated to universe defaults. /// Credentials and provider transport still use their existing configuration. pub fn validate_model_environment() -> anyhow::Result<()> { @@ -245,12 +244,11 @@ pub struct DeploymentStores { impl DeploymentStores { pub async fn from_env() -> anyhow::Result { let allow_unledgered_schema = allow_unledgered_schema_from_env()?; + let object_store = object_storage::from_env()? + .map(|config| config.build()) + .transpose()?; let pool = postgres_pool_from_env().await?; verify_runtime_schema(&pool, allow_unledgered_schema).await?; - let object_store = match object_store_config_from_env()? { - Some(object_config) => Some(build_s3_object_store(object_config)?), - None => None, - }; let secrets_master_key = match optional_env("LIGHTSPEED_SECRETS_MASTER_KEY") { Some(master_key) => { Some(SecretsMasterKey::from_base64(&master_key).map_err(|error| { @@ -391,48 +389,6 @@ pub async fn pg_store_from_env() -> anyhow::Result> { Ok(store) } -fn object_store_config_from_env() -> anyhow::Result> { - let object_env_present = [ - "LIGHTSPEED_OBJECT_STORE_BUCKET", - "LIGHTSPEED_OBJECT_STORE_ENDPOINT", - "LIGHTSPEED_OBJECT_STORE_REGION", - "LIGHTSPEED_OBJECT_STORE_PREFIX", - "LIGHTSPEED_OBJECT_STORE_FORCE_PATH_STYLE", - ] - .into_iter() - .any(|key| env::var_os(key).is_some()); - let Some(bucket) = optional_env("LIGHTSPEED_OBJECT_STORE_BUCKET") else { - return if object_env_present { - Err(anyhow::anyhow!( - "LIGHTSPEED_OBJECT_STORE_BUCKET must be set when object store env is configured" - )) - } else { - Ok(None) - }; - }; - - let mut config = S3ObjectStoreConfig::new(bucket); - if let Some(endpoint) = optional_env("LIGHTSPEED_OBJECT_STORE_ENDPOINT") { - config = config.with_endpoint(endpoint); - } - config = config.with_region( - optional_env("LIGHTSPEED_OBJECT_STORE_REGION").unwrap_or_else(|| "us-east-1".to_owned()), - ); - if let Some(access_key_id) = optional_env("AWS_ACCESS_KEY_ID") { - config = config.with_access_key_id(access_key_id); - } - if let Some(secret_access_key) = optional_env("AWS_SECRET_ACCESS_KEY") { - config = config.with_secret_access_key(secret_access_key); - } - if let Some(force_path_style) = optional_env("LIGHTSPEED_OBJECT_STORE_FORCE_PATH_STYLE") { - config = - config.with_force_path_style(force_path_style.parse::().map_err(|error| { - anyhow::anyhow!("invalid LIGHTSPEED_OBJECT_STORE_FORCE_PATH_STYLE: {error}") - })?); - } - Ok(Some(config)) -} - fn optional_env(key: &str) -> Option { env::var(key).ok().filter(|value| !value.is_empty()) } diff --git a/crates/temporal-runtime/src/config/object_storage.rs b/crates/temporal-runtime/src/config/object_storage.rs new file mode 100644 index 000000000..abad2f736 --- /dev/null +++ b/crates/temporal-runtime/src/config/object_storage.rs @@ -0,0 +1,277 @@ +use std::{env, sync::Arc}; + +use object_store::ObjectStore; +use store_pg::{ + AzureObjectStoreConfig, AzureObjectStoreCredential, S3ObjectStoreConfig, + build_azure_object_store, build_s3_object_store, +}; + +const BACKEND: &str = "LIGHTSPEED_OBJECT_STORE_BACKEND"; +const BUCKET: &str = "LIGHTSPEED_OBJECT_STORE_BUCKET"; +const ENDPOINT: &str = "LIGHTSPEED_OBJECT_STORE_ENDPOINT"; +const REGION: &str = "LIGHTSPEED_OBJECT_STORE_REGION"; +const PREFIX: &str = "LIGHTSPEED_OBJECT_STORE_PREFIX"; +const PATH_STYLE: &str = "LIGHTSPEED_OBJECT_STORE_FORCE_PATH_STYLE"; +const ACCOUNT: &str = "LIGHTSPEED_OBJECT_STORE_AZURE_ACCOUNT"; +const CONTAINER: &str = "LIGHTSPEED_OBJECT_STORE_AZURE_CONTAINER"; +const ACCESS_KEY: &str = "LIGHTSPEED_OBJECT_STORE_AZURE_ACCESS_KEY"; +const SAS_TOKEN: &str = "LIGHTSPEED_OBJECT_STORE_AZURE_SAS_TOKEN"; +const AZURE_KEYS: [&str; 4] = [ACCOUNT, CONTAINER, ACCESS_KEY, SAS_TOKEN]; +const S3_KEYS: [&str; 3] = [BUCKET, REGION, PATH_STYLE]; + +#[derive(Debug)] +pub(super) enum ObjectStorageConfig { + S3(S3ObjectStoreConfig), + Azure(AzureObjectStoreConfig), +} + +impl ObjectStorageConfig { + pub(super) fn build(self) -> anyhow::Result> { + match self { + ObjectStorageConfig::S3(config) => Ok(build_s3_object_store(config)?), + ObjectStorageConfig::Azure(config) => build_azure_object_store(config).map_err(|_| { + // Builder errors can include malformed SAS values. Report setting names only. + anyhow::anyhow!( + "invalid Azure object-store configuration; check {ACCOUNT}, {CONTAINER}, \ + {ENDPOINT}, and the credential in {ACCESS_KEY} or {SAS_TOKEN}" + ) + }), + } + } +} + +pub(super) fn from_env() -> anyhow::Result> { + parse(|key| env::var(key).ok()) +} + +fn parse(get: impl Fn(&str) -> Option) -> anyhow::Result> { + let value = |key: &str| get(key).filter(|value| !value.is_empty()); + let required = |key: &str| { + value(key) + .ok_or_else(|| anyhow::anyhow!("{key} must be set when object storage is configured")) + }; + let configured = [BACKEND, ENDPOINT, PREFIX] + .into_iter() + .chain(S3_KEYS) + .chain(AZURE_KEYS) + .any(|key| get(key).is_some()); + if !configured { + return Ok(None); + } + + let backend = match value(BACKEND).as_deref().unwrap_or("s3") { + "s3" => { + for key in AZURE_KEYS { + anyhow::ensure!(get(key).is_none(), "{key} requires {BACKEND}=azure"); + } + let mut config = S3ObjectStoreConfig::new(required(BUCKET)?); + config.endpoint = value(ENDPOINT); + config.region = value(REGION).unwrap_or_else(|| "us-east-1".into()); + config.access_key_id = value("AWS_ACCESS_KEY_ID"); + config.secret_access_key = value("AWS_SECRET_ACCESS_KEY"); + config.force_path_style = value(PATH_STYLE) + .map(|value| value.parse::()) + .transpose() + .map_err(|_| anyhow::anyhow!("{PATH_STYLE} must be true or false"))? + .unwrap_or(false); + ObjectStorageConfig::S3(config) + } + "azure" => { + for key in S3_KEYS { + anyhow::ensure!( + get(key).is_none(), + "{key} is only supported with {BACKEND}=s3" + ); + } + let credential = match (value(ACCESS_KEY), value(SAS_TOKEN)) { + (Some(key), None) => AzureObjectStoreCredential::AccessKey(key), + (None, Some(token)) => AzureObjectStoreCredential::SasToken(token), + _ => anyhow::bail!( + "set exactly one of {ACCESS_KEY} or {SAS_TOKEN} for Azure storage" + ), + }; + let endpoint = value(ENDPOINT); + if let Some(endpoint) = &endpoint { + validate_azure_endpoint(endpoint)?; + } + ObjectStorageConfig::Azure(AzureObjectStoreConfig { + account: required(ACCOUNT)?, + container: required(CONTAINER)?, + credential, + endpoint, + }) + } + _ => anyhow::bail!("{BACKEND} must be s3 or azure"), + }; + Ok(Some(backend)) +} + +fn validate_azure_endpoint(endpoint: &str) -> anyhow::Result<()> { + let url = url::Url::parse(endpoint) + .map_err(|_| anyhow::anyhow!("{ENDPOINT} must be a valid account endpoint URL"))?; + anyhow::ensure!( + url.host_str().is_some() + && url.username().is_empty() + && url.password().is_none() + && url.query().is_none() + && url.fragment().is_none(), + "{ENDPOINT} must have a host and contain no credentials, query, or fragment; use {SAS_TOKEN} for SAS" + ); + let loopback = url.host_str().is_some_and(|host| { + host.eq_ignore_ascii_case("localhost") + || host + .trim_matches(['[', ']']) + .parse::() + .is_ok_and(|ip| ip.is_loopback()) + }); + anyhow::ensure!( + url.scheme() == "https" || (url.scheme() == "http" && loopback), + "{ENDPOINT} must use HTTPS; HTTP is supported only for loopback emulators" + ); + Ok(()) +} + +#[cfg(test)] +mod tests { + use super::*; + + fn config(entries: &[(&str, &str)]) -> anyhow::Result> { + parse(|key| { + entries + .iter() + .find(|(name, _)| *name == key) + .map(|(_, value)| (*value).into()) + }) + } + + fn azure() -> Vec<(&'static str, &'static str)> { + vec![ + (BACKEND, "azure"), + (ACCOUNT, "account"), + (CONTAINER, "container"), + (ACCESS_KEY, "secret-key"), + ] + } + + #[test] + fn storage_stays_optional_and_aws_credentials_do_not_enable_it() { + assert!(config(&[]).unwrap().is_none()); + assert!( + config(&[ + ("AWS_ACCESS_KEY_ID", "key"), + ("AWS_SECRET_ACCESS_KEY", "secret") + ]) + .unwrap() + .is_none() + ); + } + + #[test] + fn legacy_s3_settings_preserve_defaults_and_credentials() { + let config = config(&[ + (BUCKET, "bucket"), + (ENDPOINT, "http://minio:9000"), + (PATH_STYLE, "true"), + ("AWS_ACCESS_KEY_ID", "key"), + ("AWS_SECRET_ACCESS_KEY", "secret"), + ]) + .unwrap() + .unwrap(); + let debug = format!("{config:?}"); + assert!(!debug.contains("secret")); + let ObjectStorageConfig::S3(config) = config else { + panic!("expected S3") + }; + assert_eq!(config.bucket, "bucket"); + assert_eq!(config.region, "us-east-1"); + assert_eq!(config.access_key_id.as_deref(), Some("key")); + assert_eq!(config.secret_access_key.as_deref(), Some("secret")); + assert!(config.force_path_style); + } + + #[test] + fn incomplete_or_invalid_configuration_is_rejected_without_values() { + for entries in [ + vec![(PREFIX, "prefix")], + vec![(BUCKET, "")], + vec![(BACKEND, "unknown-secret")], + vec![(BUCKET, "bucket"), (PATH_STYLE, "secret-bool")], + vec![(ACCOUNT, "account")], + ] { + let error = config(&entries).unwrap_err().to_string(); + assert!(!error.contains("secret")); + } + } + + #[test] + fn azure_requires_account_container_and_exactly_one_credential() { + for key in [ACCOUNT, CONTAINER, ACCESS_KEY] { + let mut entries = azure(); + entries.retain(|(name, _)| *name != key); + assert!(config(&entries).is_err(), "missing {key}"); + } + let mut entries = azure(); + entries.push((SAS_TOKEN, "sig=secret-sas")); + let error = config(&entries).unwrap_err().to_string(); + assert!(!error.contains("secret-sas")); + assert!(!error.contains("secret-key")); + } + + #[test] + fn azure_selects_account_key_or_sas_and_keeps_shared_settings() { + for (key, secret) in [(ACCESS_KEY, "secret-key"), (SAS_TOKEN, "?sig=secret-sas")] { + let entries = [ + (BACKEND, "azure"), + (ACCOUNT, "account"), + (CONTAINER, "container"), + (key, secret), + (PREFIX, "deployment"), + ]; + let config = config(&entries).unwrap().unwrap(); + assert!(!format!("{config:?}").contains(secret)); + assert!(matches!(config, ObjectStorageConfig::Azure(_))); + } + } + + #[test] + fn azure_rejects_s3_settings_and_unsafe_endpoints() { + for key in S3_KEYS { + let mut entries = azure(); + entries.push((key, "value")); + assert!(config(&entries).is_err()); + } + for endpoint in [ + "http://blob.example", + "file:///tmp/blob", + "https://blob.example?sig=secret-sas", + "https://user:secret-key@blob.example", + "https://blob.example/#secret", + ] { + let mut entries = azure(); + entries.push((ENDPOINT, endpoint)); + let error = config(&entries).unwrap_err().to_string(); + assert!(!error.contains("secret")); + } + for endpoint in [ + "https://account.blob.core.windows.net", + "http://127.0.0.1:10000/account", + "http://[::1]:10000/account", + ] { + let mut entries = azure(); + entries.push((ENDPOINT, endpoint)); + assert!(config(&entries).is_ok()); + } + } + + #[test] + fn malformed_sas_builder_error_does_not_expose_credential() { + let entries = [ + (BACKEND, "azure"), + (ACCOUNT, "account"), + (CONTAINER, "container"), + (SAS_TOKEN, "invalid-secret-sas"), + ]; + let error = config(&entries).unwrap().unwrap().build().unwrap_err(); + assert!(!format!("{error:#}").contains("invalid-secret-sas")); + } +} diff --git a/docs/documentation/deployment/configuration.md b/docs/documentation/deployment/configuration.md index 580b73f9d..5cdc26061 100644 --- a/docs/documentation/deployment/configuration.md +++ b/docs/documentation/deployment/configuration.md @@ -134,7 +134,7 @@ for setup and precedence. With all `LIGHTSPEED_OBJECT_STORE_*` variables absent, blobs up to 64 KiB are stored inline in PostgreSQL. Larger writes fail. This supports an initial -small-text walkthrough; configure S3-compatible storage before working with +small-text walkthrough; configure S3-compatible or Azure Blob storage before working with larger files, attachments, or other payloads. To use S3-compatible storage for blobs, configure the full group. For example: @@ -151,15 +151,42 @@ Supply the credentials required by the object-store client. Add path-style addressing. The [reference](../reference/environment-variables.md#object-storage) lists the exact settings. +Existing S3 settings remain valid without a backend selector. To use native +Azure Blob storage instead: + +```dotenv +LIGHTSPEED_OBJECT_STORE_BACKEND=azure +LIGHTSPEED_OBJECT_STORE_AZURE_ACCOUNT=yourstorageaccount +LIGHTSPEED_OBJECT_STORE_AZURE_CONTAINER=lightspeed-content +LIGHTSPEED_OBJECT_STORE_PREFIX=production +``` + +Supply exactly one of `LIGHTSPEED_OBJECT_STORE_AZURE_ACCESS_KEY` or +`LIGHTSPEED_OBJECT_STORE_AZURE_SAS_TOKEN` through the deployment's secret +mechanism. Create the container first. Do not carry S3 bucket, region, or +path-style settings into this configuration. The client uses the account's +standard HTTPS Blob endpoint unless `LIGHTSPEED_OBJECT_STORE_ENDPOINT` is set. +For an emulator, the endpoint can be a loopback HTTP account URL, such as +`http://127.0.0.1:10000/account`; remote Azure endpoints require HTTPS. Put SAS +tokens in their dedicated secret setting rather than the endpoint URL. + Setting any variable in this group activates its configuration requirements, -even if the value is empty. An endpoint or prefix without a nonempty bucket -causes startup to fail. Remove the entire group when intentionally using only -PostgreSQL. +even if the value is empty. S3 requires a nonempty bucket; Azure requires its +account, container, and credential. Remove the entire group when intentionally +using only PostgreSQL. With object storage configured, small blobs still remain inline in PostgreSQL; larger blobs go to the object store. PostgreSQL keeps their catalog and exact physical object keys. A prefix change affects new keys, while existing records continue to reference their original keys. Keep those objects accessible. +Changing the backend or container does not copy existing objects. Plan a storage +migration separately rather than switching an existing deployment's settings. + +Remote PostgreSQL connections support TLS in the release build. Use +`sslmode=verify-full` in both Runtime and Platform database URLs, keeping +username/password authentication when that is the deployment's chosen method. +Normal HTTPS/TLS uses the clients' existing CA trust; manual certificate files +are needed only if a private authority must be added to that trust. Changing a bucket or endpoint does not copy existing content, so plan that change as a data migration and test old attachments and workspace files as well as newly written content. diff --git a/docs/documentation/deployment/self-hosting.md b/docs/documentation/deployment/self-hosting.md index d70ae0d51..3dc7fe044 100644 --- a/docs/documentation/deployment/self-hosting.md +++ b/docs/documentation/deployment/self-hosting.md @@ -180,9 +180,9 @@ routes. Leave all `LIGHTSPEED_OBJECT_STORE_*` settings absent for the initial small-blob PostgreSQL setup. Writes above 64 KiB fail without object storage. Even setting part of that configuration to an empty value can activate object-store -configuration and require a bucket. Follow +configuration and require backend-specific settings. Follow [Choose the blob backend](configuration.md#choose-the-blob-backend) to add the -complete S3-compatible configuration. +complete S3-compatible or Azure Blob configuration. The Platform needs a deployment-scoped runtime key with every method group and permission to assert actors. The migration procedure below creates it. Prepare diff --git a/docs/documentation/deployment/troubleshooting.md b/docs/documentation/deployment/troubleshooting.md index cf7a028d9..afb31756b 100644 --- a/docs/documentation/deployment/troubleshooting.md +++ b/docs/documentation/deployment/troubleshooting.md @@ -46,7 +46,7 @@ If the runtime exits before serving, check the error against these dependencies: | PostgreSQL connection | The runtime database URL, credentials, network path, and required connection options. Container-local `localhost` is not the database host. | | Schema verification | Use the deployed release's `schema-version` diagnostic. The runtime never migrates implicitly; follow the explicit migration procedure. | | Temporal connection | The frontend address and an existing namespace, reachable from the process. | -| Object-store configuration | A nonempty bucket when any `LIGHTSPEED_OBJECT_STORE_*` variable is set, plus the intended endpoint and credentials. | +| Object-store configuration | S3 requires a bucket; `LIGHTSPEED_OBJECT_STORE_BACKEND=azure` requires account, container, and exactly one access key or SAS token. Check for settings mixed between backends and use the intended endpoint. | | Role or environment routing | Valid roles and the internal environment gateway URL/token on every process without that role. | | Secret configuration | A base64 master key decoding to 32 bytes, matching the stored encrypted state. | | Retired model variables | Remove `LIGHTSPEED_CHAT_PROVIDER` and `LIGHTSPEED_CHAT_MODEL`; configure each universe's model defaults through Models or `lightspeed model defaults`. | diff --git a/docs/documentation/reference/environment-variables.md b/docs/documentation/reference/environment-variables.md index ae0f1045c..e0a77a084 100644 --- a/docs/documentation/reference/environment-variables.md +++ b/docs/documentation/reference/environment-variables.md @@ -38,7 +38,7 @@ They do not configure the TypeScript Platform backend. | Variable | Requirement/default | Purpose | | --- | --- | --- | -| `LIGHTSPEED_POSTGRES_URL` | **Required**; falls back to `LIGHTSPEED_TEST_POSTGRES_URL` | PostgreSQL connection URL used by the runtime, migration commands, and schema diagnostics. Production must use this name. | +| `LIGHTSPEED_POSTGRES_URL` | **Required**; falls back to `LIGHTSPEED_TEST_POSTGRES_URL` | PostgreSQL connection URL used by the runtime, migration commands, and schema diagnostics. Production must use this name. TLS is supported; use `sslmode=verify-full` for remote databases to verify the certificate and hostname. | | `LIGHTSPEED_PG_UNIVERSE_ID` | **Required in `single` auth mode** | UUID of the sole universe in a single-tenant deployment. Not used to auto-create universes in multi-tenant modes. | | `LIGHTSPEED_ROLES` | `gateway,environment-gateway,sessions,bots,channels` | Roles this process runs, a comma-separated subset of `gateway` (JSON-RPC, OAuth callbacks, webhook hooks), `environment-gateway` (worker environment routes, the public daemon registration routes, the lifecycle reconciler that creates and closes provisioned environments, and the registered-environment sweep), `sessions` (session workflows plus the promise-repair reaper, the session-retention reaper, both every five minutes, and the hourly CAS blob sweeper), `bots`, `channels` (also `--roles`). Each worker role polls its own task queue with that subsystem's workflows, activities, and background loops. Run exactly one `environment-gateway` process per deployment. | | `LIGHTSPEED_TASK_QUEUE` | `lightspeed-sessions` | Sessions task queue shared by the gateway and the `sessions` role. Deployments sharing a Temporal namespace must use distinct queues. | @@ -113,20 +113,33 @@ does not supply an unset model default. ### Object storage -If any `LIGHTSPEED_OBJECT_STORE_*` variable is set, -`LIGHTSPEED_OBJECT_STORE_BUCKET` becomes required. Blobs up to 64 KiB remain -inline in PostgreSQL; larger blobs require object storage. Without this group, -writes above that inline limit fail. +Blobs up to 64 KiB remain inline in PostgreSQL; larger blobs require object +storage. With the group absent, writes above that inline limit fail. Setting +any variable in the group activates validation, even if its value is empty. +Existing configurations default to S3; Azure must be selected explicitly. | Variable | Requirement/default | Purpose | | --- | --- | --- | -| `LIGHTSPEED_OBJECT_STORE_BUCKET` | Conditional | S3 bucket for CAS payloads. | -| `LIGHTSPEED_OBJECT_STORE_ENDPOINT` | Provider default | Optional S3-compatible endpoint such as MinIO. | +| `LIGHTSPEED_OBJECT_STORE_BACKEND` | `s3` | `s3` or `azure`. Azure settings require the explicit `azure` selection. | +| `LIGHTSPEED_OBJECT_STORE_BUCKET` | Required for S3 | S3 bucket for CAS payloads. Not accepted with Azure. | +| `LIGHTSPEED_OBJECT_STORE_ENDPOINT` | Provider default | Optional S3-compatible endpoint or Azure account endpoint. Azure endpoints must use HTTPS; HTTP is allowed only for loopback emulators. Do not include credentials or a SAS query in an Azure endpoint. | | `LIGHTSPEED_OBJECT_STORE_REGION` | `us-east-1` | S3 region. | -| `LIGHTSPEED_OBJECT_STORE_PREFIX` | Unset | Key prefix inside the bucket. | +| `LIGHTSPEED_OBJECT_STORE_PREFIX` | Unset | Key prefix inside the S3 bucket or Azure container. | | `LIGHTSPEED_OBJECT_STORE_FORCE_PATH_STYLE` | `false` | Boolean path-style addressing override; use `true` for the local MinIO stack. | | `AWS_ACCESS_KEY_ID` | Credential-chain dependent | S3 access key used when explicit object-store credentials are needed. | | `AWS_SECRET_ACCESS_KEY` | Credential-chain dependent | S3 secret key. | +| `LIGHTSPEED_OBJECT_STORE_AZURE_ACCOUNT` | Required for Azure | Azure storage account name. | +| `LIGHTSPEED_OBJECT_STORE_AZURE_CONTAINER` | Required for Azure | Existing Blob container name. | +| `LIGHTSPEED_OBJECT_STORE_AZURE_ACCESS_KEY` | Exactly one Azure credential | Storage account access key. Supply through a Secret. | +| `LIGHTSPEED_OBJECT_STORE_AZURE_SAS_TOKEN` | Exactly one Azure credential | SAS token, with or without a leading `?`. Supply through a Secret; permit the read/write/list/delete operations the blob store uses. | + +Azure requires exactly one of access key or SAS token; it does not fall back to +managed identity. S3-only bucket, region, and path-style settings are rejected +with Azure. Both backends use normal HTTPS certificate trust by default; no +certificate file is required for a server trusted by the container's CA roots. +Keep the backend, account/container, and existing objects consistent across +restarts: PostgreSQL records physical keys, not enough information to migrate +objects between backends or containers automatically. ### Audio preprocessing diff --git a/docs/roadmap/azure-storage-and-postgres-tls.md b/docs/roadmap/azure-storage-and-postgres-tls.md new file mode 100644 index 000000000..9b34b931b --- /dev/null +++ b/docs/roadmap/azure-storage-and-postgres-tls.md @@ -0,0 +1,54 @@ +# Azure Blob storage and PostgreSQL TLS + +**Status:** Implemented and verified locally, 2026-11-02. + +The Runtime release enables SQLx's Rustls backend with native CA roots in both +store and Runtime dependency declarations. The same pool configuration serves +normal Runtime connections, explicit migrations, and API-key provisioning. +Database authentication remains username/password. Platform and LLM clients +retain their existing TLS behaviour; this change introduces no certificate-file +requirement or LLM-specific trust override. + +Storage backend configuration now selects native Azure Blob or S3. Existing +S3 variables retain their defaults. Azure requires an explicit selector, +account/container, and exactly one account key or SAS token. Credentials and +invalid Azure credential errors are redacted. Missing or mixed backend settings +fail before the Runtime attempts database setup. + +Both implementations remain behind the existing `ObjectStore` interface, with +separate `s3.rs` and `azure.rs` modules in `store-pg`. Existing public S3 exports, +CAS hashes, universe prefixes, database schema, and inline storage are preserved. +Changing backend/container still requires a deliberate data migration. + +Verification includes configuration and redaction unit tests, plus explicit +ignored suites for external services: + +- `cargo test -p store-pg --test azure_blob_live -- --ignored` requires + `LIGHTSPEED_TEST_POSTGRES_URL`, `LIGHTSPEED_TEST_AZURE_ACCOUNT`, + `LIGHTSPEED_TEST_AZURE_CONTAINER`, and exactly one of + `LIGHTSPEED_TEST_AZURE_ACCESS_KEY` or `LIGHTSPEED_TEST_AZURE_SAS_TOKEN`. + `LIGHTSPEED_TEST_AZURE_ENDPOINT` optionally selects an emulator or private + account endpoint. The suite uses a unique schema and prefix and cleans up its + blobs. It checks payloads above the inline threshold, fresh-client reads, + missing blobs, universe isolation, listing, and deletion. +- `cargo test -p store-pg --test postgres_tls_live -- --ignored` requires + `LIGHTSPEED_TEST_POSTGRES_TLS_URL` with `sslmode=verify-full`, + `LIGHTSPEED_TEST_POSTGRES_WRONG_CA_FILE` pointing at a valid unrelated CA, and + `LIGHTSPEED_TEST_POSTGRES_WRONG_HOST` naming another address of that server + which its certificate does not cover. It checks encrypted connections and + rejection of incorrect trust/hostname settings without changing schemas. + +Neither suite reads `.env` or silently succeeds when configuration is missing. +Local test certificates are generated in temporary directories, not stored in +the repository. + +Validation passed for the affected Rust unit suites, verified PostgreSQL TLS +(including rejected CA/hostname cases), and Azure CAS against Azurite with both +account-key and SAS authentication. Platform migrations also passed against the +TLS test database and rejected an incorrect hostname. The built Runtime migrated +the database, verified its schema, and provisioned/reused a Platform API key with +the Azure settings loaded. Full workspace Clippy with all targets and warnings +denied, formatting, and documentation checks passed. + +Actual AKS, Azure PostgreSQL, and Azure Blob validation remains part of deployment +acceptance; emulator tests do not establish bank connectivity. diff --git a/release/metadata.env b/release/metadata.env index e9a20fbef..1d30e9c24 100644 --- a/release/metadata.env +++ b/release/metadata.env @@ -4,7 +4,7 @@ LIGHTSPEED_RELEASE_TARGET=x86_64-unknown-linux-gnu # The environment daemon alone is published as a static binary so it runs on # any Linux image regardless of its glibc. LIGHTSPEED_ENVD_TARGET=x86_64-unknown-linux-musl -LIGHTSPEED_PRODUCT_VERSION=0.5.0 +LIGHTSPEED_PRODUCT_VERSION=0.5.1 LIGHTSPEED_RELEASE_RUST_VERSION=1.99.0 LIGHTSPEED_RELEASE_BUILD_BASE_IMAGE=rust:1.99.0-bookworm@sha256:59037199c44290f2befcdd58dcc540164763fc296950255aaefeef096a1866b0 LIGHTSPEED_API_PROTOCOL_VERSION=lightspeed.agent.api.v1 diff --git a/release/notes/v0.5.1.md b/release/notes/v0.5.1.md new file mode 100644 index 000000000..5d46bde5c --- /dev/null +++ b/release/notes/v0.5.1.md @@ -0,0 +1,21 @@ +Lightspeed v0.5.1 adds native Azure Blob storage and enables verified PostgreSQL TLS in the Rust Runtime, supporting deployments on Azure Kubernetes Service with username/password database authentication. + +**PostgreSQL** + +- Runtime, migration, and API-key commands include SQLx TLS support with native CA roots. Use `sslmode=verify-full` to verify the database certificate and hostname. +- Platform keeps its existing PostgreSQL TLS behaviour. No new database authentication mode or LLM-specific certificate configuration is introduced. + +**Blob storage** + +- Select native Azure Blob with `LIGHTSPEED_OBJECT_STORE_BACKEND=azure`, configure the account and existing container, and supply exactly one account access key or SAS token through the deployment's secret mechanism. +- Existing S3 configurations retain their defaults and public Rust exports. S3 and Azure configuration/builders live in separate modules. +- Incomplete or mixed backend settings fail at startup, and credential debug output is redacted. Both backends use their normal HTTPS certificate trust. +- CAS identities, universe isolation, inline storage, database schemas, and protocol revisions are unchanged. Changing backend/container does not migrate existing objects; retain access to old objects or plan a separate storage migration. + +**Upgrade and validation** + +Use matching Runtime and Platform images from this release. The fresh-install requirements for installations predating v0.5.0 still apply; this patch does not provide a conversion from those databases or Temporal histories. See the [v0.5.0 release notes](https://github.com/smartcomputer-ai/lightspeed/releases/tag/v0.5.0) and [deployment configuration](https://ls.bot/docs/deployment/configuration/). + +Local validation covered the affected Rust unit suites, PostgreSQL TLS certificate/hostname rejection, Azure CAS against Azurite using account-key and SAS authentication, and Runtime/Platform migration paths. Full workspace Clippy, formatting, and documentation checks passed. Actual AKS and managed Azure service connectivity remains deployment acceptance work. + +[Full changelog: v0.5.0...v0.5.1](https://github.com/smartcomputer-ai/lightspeed/compare/v0.5.0...v0.5.1)