From 76ea0e155347c4fa6af356c8de07928bc97a061a Mon Sep 17 00:00:00 2001 From: lb <542828+lukebuehler@users.noreply.github.com> Date: Thu, 5 Nov 2026 14:11:16 +0100 Subject: [PATCH 1/3] blob storage with azuire identity --- Cargo.lock | 153 +++- crates/store-pg/Cargo.toml | 6 +- crates/store-pg/src/azure.rs | 50 +- .../store-pg/src/azure/workload_identity.rs | 291 +++++++ .../src/azure/workload_identity/tests.rs | 734 ++++++++++++++++++ crates/store-pg/src/lib.rs | 5 +- crates/store-pg/tests/azure_blob_live.rs | 44 +- .../src/config/object_storage.rs | 268 ++++++- .../documentation/deployment/configuration.md | 45 +- .../deployment/troubleshooting.md | 2 +- .../reference/environment-variables.md | 20 +- .../roadmap/azure-storage-and-postgres-tls.md | 36 + 12 files changed, 1606 insertions(+), 48 deletions(-) create mode 100644 crates/store-pg/src/azure/workload_identity.rs create mode 100644 crates/store-pg/src/azure/workload_identity/tests.rs diff --git a/Cargo.lock b/Cargo.lock index 089b6c5c..1a98b4ea 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -189,6 +189,45 @@ dependencies = [ "num-traits", ] +[[package]] +name = "asn1-rs" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7f43a50ac4fdca5df8e885c21b835997f0a1cdee65494a6847694a98652d9d8" +dependencies = [ + "asn1-rs-derive", + "asn1-rs-impl", + "displaydoc", + "nom", + "num-traits", + "rusticata-macros", + "thiserror 2.0.21", + "time", +] + +[[package]] +name = "asn1-rs-derive" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3109e49b1e4909e9db6515a30c633684d68cdeaa252f215214cb4fa1a5bfee2c" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", + "synstructure 0.13.2", +] + +[[package]] +name = "asn1-rs-impl" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "async-trait" version = "0.1.92" @@ -389,6 +428,15 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" +[[package]] +name = "bit-vec" +version = "0.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b71798fca2c1fe1086445a7258a4bc81e6e49dcd24c8d0dd9a1e57395b603f51" +dependencies = [ + "serde", +] + [[package]] name = "bitflags" version = "1.3.2" @@ -1081,6 +1129,20 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5729f5117e208430e437df2f4843f5e5952997175992d1414f94c57d61e270b4" +[[package]] +name = "der-parser" +version = "10.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07da5016415d5a3c4dd39b11ed26f915f52fc4e0dc197d87908bc916e51bc1a6" +dependencies = [ + "asn1-rs", + "displaydoc", + "nom", + "num-bigint", + "num-traits", + "rusticata-macros", +] + [[package]] name = "deranged" version = "0.5.8" @@ -2551,7 +2613,7 @@ dependencies = [ "base64 0.22.1", "getrandom 0.2.17", "js-sys", - "pem", + "pem 3.0.6", "serde", "serde_json", "signature 2.2.0", @@ -3186,6 +3248,15 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "oid-registry" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12f40cff3dde1b6087cc5d5f5d4d65712f34016a03ed60e9c08dcc392736b5b7" +dependencies = [ + "asn1-rs", +] + [[package]] name = "once_cell" version = "1.21.4" @@ -3325,6 +3396,16 @@ dependencies = [ "serde_core", ] +[[package]] +name = "pem" +version = "4.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d354a98a3d1251555de99e8fdd8afda05573c31b82f59063a7b0a29b5527f120" +dependencies = [ + "base64 0.23.1", + "serde_core", +] + [[package]] name = "percent-encoding" version = "2.3.2" @@ -4072,6 +4153,20 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "rcgen" +version = "0.14.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8774e05a7d0de114588e6a28fe7e71694b82614ed569d86d8b389dfbc98b8ad8" +dependencies = [ + "aws-lc-rs", + "pem 4.0.0", + "rustls-pki-types", + "time", + "x509-parser", + "yasna", +] + [[package]] name = "redox_syscall" version = "0.5.18" @@ -4312,6 +4407,15 @@ dependencies = [ "semver", ] +[[package]] +name = "rusticata-macros" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "faf0c4a6ece9950b9abdb62b1cfcf2a68b3b67a10ba445b3bb85be2a293d0632" +dependencies = [ + "nom", +] + [[package]] name = "rustix" version = "1.1.5" @@ -5056,13 +5160,17 @@ dependencies = [ "object_store", "profiles", "rand 0.10.3", + "rcgen", "serde", "serde_json", "sha2 0.11.0", "sqlx", + "tempfile", "thiserror 2.0.21", "tokio", + "tokio-rustls", "tracing", + "url", "uuid", "vfs", ] @@ -5153,6 +5261,17 @@ dependencies = [ "futures-core", ] +[[package]] +name = "synstructure" +version = "0.13.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "synstructure" version = "0.14.0" @@ -6962,6 +7081,34 @@ version = "0.6.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3ad82d2a33cdc9674dc7465672f271e096168fcdbe0f799d9e6db8c5892679dc" +[[package]] +name = "x509-parser" +version = "0.18.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d43b0f71ce057da06bc0851b23ee24f3f86190b07203dd8f567d0b706a185202" +dependencies = [ + "asn1-rs", + "aws-lc-rs", + "data-encoding", + "der-parser", + "lazy_static", + "nom", + "oid-registry", + "rusticata-macros", + "thiserror 2.0.21", + "time", +] + +[[package]] +name = "yasna" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5f6765e852b9b4dc8e2a76843e4d64d1cea8e79bcde0b6901aea8e7c7f08282" +dependencies = [ + "bit-vec 0.9.1", + "time", +] + [[package]] name = "yoke" version = "0.8.3" @@ -6982,7 +7129,7 @@ dependencies = [ "proc-macro2", "quote", "syn 3.0.6", - "synstructure", + "synstructure 0.14.0", ] [[package]] @@ -7023,7 +7170,7 @@ dependencies = [ "proc-macro2", "quote", "syn 3.0.6", - "synstructure", + "synstructure 0.14.0", ] [[package]] diff --git a/crates/store-pg/Cargo.toml b/crates/store-pg/Cargo.toml index aa4c42cf..5bcbae78 100644 --- a/crates/store-pg/Cargo.toml +++ b/crates/store-pg/Cargo.toml @@ -28,8 +28,12 @@ sqlx = { version = "0.9", default-features = false, features = ["runtime-tokio", thiserror = "2" tracing = "0.1" tokio = { version = "1", features = ["rt"] } +url = "2" uuid = { version = "1", features = ["v4"] } vfs = { path = "../vfs" } [dev-dependencies] -tokio = { version = "1", features = ["macros", "rt", "sync", "time"] } +rcgen = { version = "0.14", default-features = false, features = ["aws_lc_rs", "pem"] } +tempfile = "3" +tokio = { version = "1", features = ["io-util", "macros", "net", "rt", "sync", "time"] } +tokio-rustls = { version = "0.26", default-features = false, features = ["aws_lc_rs"] } diff --git a/crates/store-pg/src/azure.rs b/crates/store-pg/src/azure.rs index 50516d54..31e33ee8 100644 --- a/crates/store-pg/src/azure.rs +++ b/crates/store-pg/src/azure.rs @@ -1,23 +1,31 @@ use std::{fmt, sync::Arc}; use object_store::{ - ObjectStore, + ClientOptions, ObjectStore, RetryConfig, azure::{AzureConfigKey, MicrosoftAzureBuilder}, }; -/// Static Azure credentials supplied by the deployment. Debug output is redacted. +mod workload_identity; + +pub use workload_identity::{AzureWorkloadIdentity, AzureWorkloadIdentityError}; + +/// Azure credentials supplied by the deployment. Debug output is redacted. #[derive(Clone)] pub enum AzureObjectStoreCredential { AccessKey(String), SasToken(String), + WorkloadIdentity(AzureWorkloadIdentity), } impl fmt::Debug for AzureObjectStoreCredential { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str(match self { - Self::AccessKey(_) => "AccessKey([REDACTED])", - Self::SasToken(_) => "SasToken([REDACTED])", - }) + match self { + Self::AccessKey(_) => f.write_str("AccessKey([REDACTED])"), + Self::SasToken(_) => f.write_str("SasToken([REDACTED])"), + Self::WorkloadIdentity(identity) => { + f.debug_tuple("WorkloadIdentity").field(identity).finish() + } + } } } @@ -45,21 +53,39 @@ impl fmt::Debug for AzureObjectStoreConfig { /// Credentials are explicit so a missing Secret cannot fall back to another identity. pub fn build_azure_object_store( config: AzureObjectStoreConfig, +) -> Result, object_store::Error> { + build(config, ClientOptions::new(), RetryConfig::default()) +} + +fn build( + config: AzureObjectStoreConfig, + client_options: ClientOptions, + retry: RetryConfig, ) -> Result, object_store::Error> { let mut builder = MicrosoftAzureBuilder::new() .with_account(config.account) - .with_container_name(config.container); + .with_container_name(config.container) + .with_client_options(client_options) + .with_retry(retry); + // Bearer tokens never travel in plaintext, even to a loopback endpoint. + let bearer = matches!( + config.credential, + AzureObjectStoreCredential::WorkloadIdentity(_) + ); + if let Some(endpoint) = config.endpoint { + builder = builder + .with_allow_http(!bearer && endpoint.starts_with("http://")) + .with_endpoint(endpoint); + } builder = match config.credential { AzureObjectStoreCredential::AccessKey(key) => builder.with_access_key(key), AzureObjectStoreCredential::SasToken(token) => { builder.with_config(AzureConfigKey::SasKey, token.trim_start_matches('?')) } + AzureObjectStoreCredential::WorkloadIdentity(identity) => { + workload_identity::configure(builder, identity)? + } }; - if let Some(endpoint) = config.endpoint { - builder = builder - .with_allow_http(endpoint.starts_with("http://")) - .with_endpoint(endpoint); - } Ok(Arc::new(builder.build()?)) } diff --git a/crates/store-pg/src/azure/workload_identity.rs b/crates/store-pg/src/azure/workload_identity.rs new file mode 100644 index 00000000..dceebe35 --- /dev/null +++ b/crates/store-pg/src/azure/workload_identity.rs @@ -0,0 +1,291 @@ +//! Microsoft Entra workload identity federation for Azure Blob storage. +//! +//! `object_store` performs the exchange, caches access tokens, and re-reads the +//! projected token file for each exchange. This module selects that provider +//! explicitly and closes two gaps that matter for credential privacy: +//! +//! - The provider's derived `Debug` prints its cached access token, so it is held +//! behind [`FederatedCredentialProvider`], whose `Debug` shows only the file path. +//! - Request errors embed raw response bodies, so [`ScrubbingConnector`] reduces +//! every error body to allowlisted diagnostics before the retry layer reads it. +use std::{error::Error as StdError, fmt, io, path::Path, sync::Arc}; + +use async_trait::async_trait; +use object_store::{ + CredentialProvider, HeaderMap, + azure::{AzureCredential, AzureCredentialProvider, MicrosoftAzureBuilder}, + client::{ + HttpClient, HttpConnector, HttpError, HttpRequest, HttpResponse, HttpService, + ReqwestConnector, + }, +}; +use thiserror::Error; + +const STORE: &str = "MicrosoftAzure"; +/// Longer values are not diagnostic codes or identifiers, so they are withheld. +const MAX_DIAGNOSTIC_LEN: usize = 128; + +/// Settings for exchanging a projected token for Azure Storage access tokens. +/// None of these values is secret; the token itself stays in the file. +#[derive(Clone, Debug)] +pub struct AzureWorkloadIdentity { + pub client_id: String, + pub tenant_id: String, + /// Read again for each exchange and never canonicalized, so a projected + /// volume can repoint its symlinks when the kubelet rotates the token. + pub federated_token_file: String, + /// HTTPS authority. `None` selects the public Azure cloud. + pub authority_host: Option, +} + +#[derive(Debug, Error, PartialEq, Eq)] +pub enum AzureWorkloadIdentityError { + #[error("AZURE_CLIENT_ID must not be empty")] + EmptyClientId, + #[error("AZURE_TENANT_ID must be a tenant ID or domain name")] + InvalidTenantId, + #[error("AZURE_FEDERATED_TOKEN_FILE must be an absolute path")] + InvalidTokenFile, + #[error( + "AZURE_AUTHORITY_HOST must be an HTTPS authority with a host and no credentials, path, query, or fragment" + )] + InvalidAuthorityHost, +} + +impl AzureWorkloadIdentity { + /// Check the settings without touching the file or the network, so commands + /// that never reach Blob storage do not depend on either. + pub fn validate(&self) -> Result<(), AzureWorkloadIdentityError> { + if self.client_id.trim().is_empty() { + return Err(AzureWorkloadIdentityError::EmptyClientId); + } + // The tenant becomes a path segment of the token URL. + let tenant = self.tenant_id.as_str(); + if !tenant.starts_with(|c: char| c.is_ascii_alphanumeric()) + || !tenant + .chars() + .all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '.') + { + return Err(AzureWorkloadIdentityError::InvalidTenantId); + } + if !Path::new(&self.federated_token_file).is_absolute() { + return Err(AzureWorkloadIdentityError::InvalidTokenFile); + } + self.normalized_authority_host().map(|_| ()) + } + + /// The library joins the authority and tenant with a slash, so a configured + /// trailing slash must not reach it. + fn normalized_authority_host(&self) -> Result, AzureWorkloadIdentityError> { + let Some(authority) = &self.authority_host else { + return Ok(None); + }; + let url = url::Url::parse(authority) + .map_err(|_| AzureWorkloadIdentityError::InvalidAuthorityHost)?; + if url.scheme() != "https" + || url.host_str().is_none() + || !url.username().is_empty() + || url.password().is_some() + || url.path() != "/" + || url.query().is_some() + || url.fragment().is_some() + { + return Err(AzureWorkloadIdentityError::InvalidAuthorityHost); + } + Ok(Some(url.origin().ascii_serialization())) + } +} + +/// Install the federated credential on `builder`. Building creates the provider +/// without reading the token file or opening a connection. +pub(super) fn configure( + builder: MicrosoftAzureBuilder, + identity: AzureWorkloadIdentity, +) -> Result { + identity.validate().map_err(generic)?; + let authority_host = identity.normalized_authority_host().map_err(generic)?; + let token_file = identity.federated_token_file; + let builder = builder.with_http_connector(ScrubbingConnector); + + // The library's provider is reachable only through a built store. Its token + // client shares this builder's connector, certificate trust, and retry policy. + let mut federated = builder + .clone() + .with_credential_type("workload_identity") + .with_client_id(identity.client_id) + .with_tenant_id(identity.tenant_id) + .with_federated_token_file(token_file.clone()); + if let Some(authority_host) = authority_host { + federated = federated.with_authority_host(authority_host); + } + let inner = federated.build()?.credentials().clone(); + Ok(builder.with_credentials(Arc::new(FederatedCredentialProvider { inner, token_file }))) +} + +fn generic(error: impl StdError + Send + Sync + 'static) -> object_store::Error { + object_store::Error::Generic { + store: STORE, + source: Box::new(error), + } +} + +/// The stage at which an access token could not be obtained. Holds rendered text +/// rather than the library's error so that nothing else is reachable as a source. +#[derive(Debug, Error)] +pub(super) enum TokenAcquisitionError { + #[error( + "could not read the projected token file '{path}' named by AZURE_FEDERATED_TOKEN_FILE{}", + reason.map(|kind| format!(": {kind}")).unwrap_or_default() + )] + TokenFile { + path: String, + reason: Option, + }, + #[error("Microsoft Entra token exchange failed: {detail}")] + Exchange { detail: String }, + #[error("Microsoft Entra returned a token response that could not be read")] + InvalidResponse, +} + +struct FederatedCredentialProvider { + inner: AzureCredentialProvider, + token_file: String, +} + +impl fmt::Debug for FederatedCredentialProvider { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("FederatedCredentialProvider") + .field("token_file", &self.token_file) + .finish_non_exhaustive() + } +} + +#[async_trait] +impl CredentialProvider for FederatedCredentialProvider { + type Credential = AzureCredential; + + async fn get_credential(&self) -> object_store::Result> { + self.inner + .get_credential() + .await + .map_err(|error| generic(classify_token_error(&error, &self.token_file))) + } +} + +/// The library's credential error type is private, but its variants differ in +/// their source: none for the token file, `HttpError` for an unreadable token +/// response, and an opaque retry error for the exchange request. +fn classify_token_error(error: &object_store::Error, token_file: &str) -> TokenAcquisitionError { + let object_store::Error::Generic { source, .. } = error else { + return TokenAcquisitionError::Exchange { + detail: "details withheld".into(), + }; + }; + match source.source() { + None => TokenAcquisitionError::TokenFile { + path: token_file.to_owned(), + reason: std::fs::metadata(token_file).err().map(|e| e.kind()), + }, + Some(inner) if inner.is::() => TokenAcquisitionError::InvalidResponse, + Some(_) => TokenAcquisitionError::Exchange { + detail: render_chain(source.as_ref()), + }, + } +} + +/// Keep transport causes such as certificate rejections. Response bodies in this +/// text have already passed through [`ScrubbingConnector`]. +fn render_chain(error: &(dyn StdError + 'static)) -> String { + let mut rendered = error.to_string(); + let mut current = error.source(); + while let Some(source) = current { + let text = source.to_string(); + if !rendered.contains(&text) { + rendered.push_str(": "); + rendered.push_str(&text); + } + current = source.source(); + } + rendered +} + +#[derive(Debug)] +struct ScrubbingConnector; + +impl HttpConnector for ScrubbingConnector { + fn connect(&self, options: &object_store::ClientOptions) -> object_store::Result { + Ok(HttpClient::new(ScrubbingService( + ReqwestConnector::default().connect(options)?, + ))) + } +} + +#[derive(Debug)] +struct ScrubbingService(HttpClient); + +#[async_trait] +impl HttpService for ScrubbingService { + async fn call(&self, request: HttpRequest) -> Result { + let response = self.0.execute(request).await?; + if !(response.status().is_client_error() || response.status().is_server_error()) { + return Ok(response); + } + let (parts, body) = response.into_parts(); + let body = body.bytes().await?; + let summary = diagnostic_summary(&parts.headers, &body); + Ok(HttpResponse::from_parts(parts, summary.into())) + } +} + +/// Reduce an Entra or Azure Storage error to fields that identify the failure +/// and locate it in the service's logs. Free text is dropped because it can +/// repeat request content; Entra's `AADSTS` code identifies the same failure. +fn diagnostic_summary(headers: &HeaderMap, body: &[u8]) -> String { + let mut fields = Vec::new(); + let mut push = |name: &str, value: Option<&str>| { + if let Some(value) = value.filter(is_diagnostic_value) { + fields.push(format!("{name}={value}")); + } + }; + let header = |name: &str| headers.get(name).and_then(|value| value.to_str().ok()); + let text = String::from_utf8_lossy(body); + + if let Ok(json) = serde_json::from_slice::(body) { + push("error", json["error"].as_str()); + for code in json["error_codes"].as_array().into_iter().flatten() { + if let Some(code) = code.as_u64() { + push("code", Some(&format!("AADSTS{code}"))); + } + } + push("trace_id", json["trace_id"].as_str()); + push("correlation_id", json["correlation_id"].as_str()); + } + push( + "storage_error", + header("x-ms-error-code").or_else(|| xml_element(&text, "Code")), + ); + push("request_id", header("x-ms-request-id")); + + if fields.is_empty() { + "details withheld".into() + } else { + fields.join(" ") + } +} + +fn is_diagnostic_value(value: &&str) -> bool { + !value.is_empty() + && value.len() <= MAX_DIAGNOSTIC_LEN + && value + .chars() + .all(|c| c.is_ascii_alphanumeric() || c == '-' || c == '_') +} + +fn xml_element<'a>(text: &'a str, name: &str) -> Option<&'a str> { + let (_, rest) = text.split_once(&format!("<{name}>"))?; + let (value, _) = rest.split_once(&format!(""))?; + Some(value) +} + +#[cfg(test)] +mod tests; diff --git a/crates/store-pg/src/azure/workload_identity/tests.rs b/crates/store-pg/src/azure/workload_identity/tests.rs new file mode 100644 index 00000000..f71b0950 --- /dev/null +++ b/crates/store-pg/src/azure/workload_identity/tests.rs @@ -0,0 +1,734 @@ +//! Runs the library's real exchange and Blob client over TLS against a local +//! fake of Entra and Azure Storage, so certificate checks and wire content are +//! the ones production uses. Certificates are generated per test. +use std::{ + collections::HashMap, + error::Error as StdError, + path::{Path, PathBuf}, + sync::{Arc, Mutex}, + time::Duration, +}; + +use object_store::{Certificate, ClientOptions, ObjectStore, ObjectStoreExt as _, RetryConfig}; +use rcgen::{BasicConstraints, CertificateParams, IsCa, Issuer, KeyPair}; +use tokio::{ + io::{AsyncReadExt as _, AsyncWriteExt as _}, + net::TcpListener, + task::JoinHandle, +}; +use tokio_rustls::{ + TlsAcceptor, + rustls::{ + ServerConfig, + crypto::aws_lc_rs, + pki_types::{CertificateDer, PrivateKeyDer}, + }, +}; + +use super::{super::*, *}; + +const CLIENT_ID: &str = "client-id"; +const TENANT_ID: &str = "tenant-id"; +const ASSERTION: &str = "secret-assertion.jwt.value"; +const ROTATED_ASSERTION: &str = "secret-rotated-assertion.jwt.value"; +const LAST_MODIFIED: &str = "Wed, 21 Oct 2015 07:28:00 GMT"; + +struct TestCa { + certificate: Certificate, + issuer: Issuer<'static, KeyPair>, + der: CertificateDer<'static>, +} + +impl TestCa { + fn new() -> Self { + let mut params = CertificateParams::new(Vec::new()).unwrap(); + params.is_ca = IsCa::Ca(BasicConstraints::Unconstrained); + let key = KeyPair::generate().unwrap(); + let certificate = params.self_signed(&key).unwrap(); + Self { + certificate: Certificate::from_der(certificate.der()).unwrap(), + der: certificate.der().clone(), + issuer: Issuer::new(params, key), + } + } + + fn acceptor(&self, host: &str) -> TlsAcceptor { + let params = CertificateParams::new(vec![host.to_owned()]).unwrap(); + let key = KeyPair::generate().unwrap(); + let certificate = params.signed_by(&key, &self.issuer).unwrap(); + let config = ServerConfig::builder_with_provider(Arc::new(aws_lc_rs::default_provider())) + .with_safe_default_protocol_versions() + .unwrap() + .with_no_client_auth() + .with_single_cert( + vec![certificate.der().clone(), self.der.clone()], + PrivateKeyDer::Pkcs8(key.serialize_der().into()), + ) + .unwrap(); + TlsAcceptor::from(Arc::new(config)) + } + + /// Only this CA is trusted, so results do not depend on the machine's store. + fn client_options(&self) -> ClientOptions { + ClientOptions::new() + .with_root_certificate(self.certificate.clone()) + .with_no_system_certificates(true) + } +} + +#[derive(Clone, Debug)] +struct Request { + method: String, + path: String, + headers: HashMap, + body: Vec, +} + +impl Request { + fn form(&self) -> HashMap { + url::form_urlencoded::parse(&self.body) + .into_owned() + .collect() + } +} + +struct Response { + status: u16, + headers: Vec<(&'static str, String)>, + body: Vec, +} + +impl Response { + fn new(status: u16, body: impl Into>) -> Self { + Self { + status, + headers: Vec::new(), + body: body.into(), + } + } +} + +type Handler = dyn Fn(&Request) -> Response + Send + Sync; + +/// One request per connection. A rejected TLS handshake records nothing, which +/// is how the rejection tests prove that no credential reached the server. +struct Server { + port: u16, + requests: Arc>>, + task: JoinHandle<()>, +} + +impl Drop for Server { + fn drop(&mut self) { + self.task.abort(); + } +} + +impl Server { + async fn start(acceptor: TlsAcceptor, handler: Arc) -> Self { + let listener = TcpListener::bind(("127.0.0.1", 0)).await.unwrap(); + let port = listener.local_addr().unwrap().port(); + let requests = Arc::new(Mutex::new(Vec::new())); + let recorded = requests.clone(); + let task = tokio::spawn(async move { + loop { + let (stream, _) = listener.accept().await.unwrap(); + let (acceptor, handler, recorded) = + (acceptor.clone(), handler.clone(), recorded.clone()); + tokio::spawn(async move { + let Ok(mut stream) = acceptor.accept(stream).await else { + return; + }; + let Some(request) = read_request(&mut stream).await else { + return; + }; + let response = handler(&request); + recorded.lock().unwrap().push(request); + let mut head = format!( + "HTTP/1.1 {} X\r\nConnection: close\r\nContent-Length: {}\r\n", + response.status, + response.body.len() + ); + for (name, value) in response.headers { + head.push_str(&format!("{name}: {value}\r\n")); + } + head.push_str("\r\n"); + let _ = stream.write_all(head.as_bytes()).await; + let _ = stream.write_all(&response.body).await; + let _ = stream.shutdown().await; + }); + } + }); + Self { + port, + requests, + task, + } + } + + fn url(&self) -> String { + format!("https://localhost:{}", self.port) + } + + fn requests(&self) -> Vec { + self.requests.lock().unwrap().clone() + } +} + +async fn read_request(stream: &mut (impl tokio::io::AsyncRead + Unpin)) -> Option { + let mut buffer = Vec::new(); + let head_end = loop { + if let Some(end) = buffer.windows(4).position(|w| w == b"\r\n\r\n") { + break end; + } + let mut chunk = [0; 4096]; + match stream.read(&mut chunk).await.ok()? { + 0 => return None, + read => buffer.extend_from_slice(&chunk[..read]), + } + }; + let head = String::from_utf8(buffer[..head_end].to_vec()).ok()?; + let mut lines = head.split("\r\n"); + let mut request_line = lines.next()?.split(' '); + let (method, path) = (request_line.next()?, request_line.next()?); + let headers: HashMap = lines + .filter_map(|line| line.split_once(':')) + .map(|(name, value)| (name.to_ascii_lowercase(), value.trim().to_owned())) + .collect(); + let length: usize = headers + .get("content-length") + .map_or(Some(0), |value| value.parse().ok())?; + let mut body = buffer.split_off(head_end + 4); + while body.len() < length { + let mut chunk = [0; 4096]; + match stream.read(&mut chunk).await.ok()? { + 0 => return None, + read => body.extend_from_slice(&chunk[..read]), + } + } + Some(Request { + method: method.into(), + path: path.into(), + headers, + body, + }) +} + +/// Issues `secret-access-token-` valid for `expires_in` seconds. +fn entra(expires_in: u64) -> Arc { + let issued = Mutex::new(0); + Arc::new(move |_| { + let mut issued = issued.lock().unwrap(); + *issued += 1; + Response::new( + 200, + format!( + r#"{{"access_token":"secret-access-token-{issued}","token_type":"Bearer","expires_in":{expires_in}}}"# + ), + ) + }) +} + +fn storage() -> Arc { + let blobs = Mutex::new(HashMap::>::new()); + Arc::new(move |request| { + let mut blobs = blobs.lock().unwrap(); + let mut response = match request.method.as_str() { + "PUT" => { + blobs.insert(request.path.clone(), request.body.clone()); + Response::new(201, Vec::new()) + } + _ => match blobs.get(&request.path) { + Some(body) => Response::new(200, body.clone()), + None => Response::new(404, "BlobNotFound"), + }, + }; + response.headers.push(("ETag", "\"etag\"".into())); + response + .headers + .push(("Last-Modified", LAST_MODIFIED.into())); + response + }) +} + +fn failing(status: u16, headers: Vec<(&'static str, String)>, body: &str) -> Arc { + let body = body.to_owned(); + Arc::new(move |_| Response { + status, + headers: headers.clone(), + body: body.clone().into_bytes(), + }) +} + +fn write_token(directory: &Path, contents: &str) -> PathBuf { + let path = directory.join("azure-identity-token"); + std::fs::write(&path, contents).unwrap(); + path +} + +fn no_retries() -> RetryConfig { + RetryConfig { + max_retries: 0, + retry_timeout: Duration::from_secs(5), + ..RetryConfig::default() + } +} + +fn store( + authority: &str, + blob_endpoint: &str, + token_file: &Path, + options: ClientOptions, +) -> Arc { + build( + AzureObjectStoreConfig { + account: "account".into(), + container: "container".into(), + endpoint: Some(format!("{blob_endpoint}/account")), + credential: AzureObjectStoreCredential::WorkloadIdentity(AzureWorkloadIdentity { + client_id: CLIENT_ID.into(), + tenant_id: TENANT_ID.into(), + federated_token_file: token_file.to_str().unwrap().into(), + authority_host: Some(authority.into()), + }), + }, + options, + no_retries(), + ) + .unwrap() +} + +fn location() -> object_store::path::Path { + object_store::path::Path::from("prefix/blob") +} + +fn full_chain(error: &object_store::Error) -> String { + let mut text = format!("{error} | {error:?}"); + let mut current = error.source(); + while let Some(source) = current { + text.push_str(&format!(" | {source} | {source:?}")); + current = source.source(); + } + text +} + +fn assert_private(text: &str) { + assert!(!text.contains("secret-"), "credential exposed: {text}"); +} + +fn token_error(error: &object_store::Error) -> &TokenAcquisitionError { + let object_store::Error::Generic { source, .. } = error else { + panic!("expected a generic error: {error:?}") + }; + source + .downcast_ref() + .unwrap_or_else(|| panic!("expected a token acquisition error: {error:?}")) +} + +#[tokio::test(flavor = "current_thread")] +async fn exchange_presents_the_projected_token_and_blobs_use_the_bearer_token() { + let ca = TestCa::new(); + let directory = tempfile::tempdir().unwrap(); + let token_file = write_token(directory.path(), ASSERTION); + let login = Server::start(ca.acceptor("localhost"), entra(3600)).await; + let blobs = Server::start(ca.acceptor("localhost"), storage()).await; + // The trailing slash must not produce a double slash in the token path. + let authority = format!("{}/", login.url()); + let store = store(&authority, &blobs.url(), &token_file, ca.client_options()); + + store.put(&location(), vec![1, 2, 3].into()).await.unwrap(); + let read = store.get(&location()).await.unwrap().bytes().await.unwrap(); + assert_eq!(read.as_ref(), [1, 2, 3]); + + let exchanges = login.requests(); + assert_eq!(exchanges.len(), 1, "the provider caches the access token"); + let exchange = &exchanges[0]; + assert_eq!(exchange.method, "POST"); + assert_eq!(exchange.path, format!("/{TENANT_ID}/oauth2/v2.0/token")); + let form = exchange.form(); + assert_eq!(form["grant_type"], "client_credentials"); + assert_eq!(form["client_id"], CLIENT_ID); + assert_eq!(form["client_assertion"], ASSERTION); + assert_eq!( + form["client_assertion_type"], + "urn:ietf:params:oauth:client-assertion-type:jwt-bearer" + ); + assert_eq!(form["scope"], "https://storage.azure.com/.default"); + + let requests = blobs.requests(); + assert_eq!(requests.len(), 2); + for request in requests { + assert_eq!( + request.headers["authorization"], + "Bearer secret-access-token-1" + ); + assert!(request.path.starts_with("/account/container/prefix/blob")); + assert!(!request.path.contains("sig=")); + } +} + +#[tokio::test(flavor = "current_thread")] +async fn each_exchange_reads_the_current_token_through_a_rotated_symlink() { + let ca = TestCa::new(); + let directory = tempfile::tempdir().unwrap(); + // Mirror a projected volume: the path is a symlink into a versioned directory. + let version = |name: &str, contents: &str| { + let path = directory.path().join(name); + std::fs::create_dir(&path).unwrap(); + std::fs::write(path.join("token"), contents).unwrap(); + path + }; + let first = version("version-1", ASSERTION); + let second = version("version-2", ROTATED_ASSERTION); + let data = directory.path().join("data"); + std::os::unix::fs::symlink(&first, &data).unwrap(); + let token_file = directory.path().join("token"); + std::os::unix::fs::symlink(data.join("token"), &token_file).unwrap(); + + // Immediately expired tokens force an exchange for every request. + let login = Server::start(ca.acceptor("localhost"), entra(0)).await; + let blobs = Server::start(ca.acceptor("localhost"), storage()).await; + let store = store(&login.url(), &blobs.url(), &token_file, ca.client_options()); + + store.put(&location(), vec![1].into()).await.unwrap(); + + let staged = directory.path().join("data-next"); + std::os::unix::fs::symlink(&second, &staged).unwrap(); + std::fs::rename(&staged, &data).unwrap(); + std::fs::remove_dir_all(&first).unwrap(); + + store.put(&location(), vec![2].into()).await.unwrap(); + + let assertions: Vec<_> = login + .requests() + .iter() + .map(|request| request.form()["client_assertion"].clone()) + .collect(); + assert_eq!(assertions, [ASSERTION, ROTATED_ASSERTION]); + let bearers: Vec<_> = blobs + .requests() + .iter() + .map(|request| request.headers["authorization"].clone()) + .collect(); + assert_eq!( + bearers, + [ + "Bearer secret-access-token-1", + "Bearer secret-access-token-2" + ] + ); +} + +#[tokio::test(flavor = "current_thread")] +async fn a_replaced_token_file_is_read_by_the_next_exchange() { + let ca = TestCa::new(); + let directory = tempfile::tempdir().unwrap(); + let token_file = write_token(directory.path(), ASSERTION); + let login = Server::start(ca.acceptor("localhost"), entra(0)).await; + let blobs = Server::start(ca.acceptor("localhost"), storage()).await; + let store = store(&login.url(), &blobs.url(), &token_file, ca.client_options()); + + store.put(&location(), vec![1].into()).await.unwrap(); + let staged = directory.path().join("staged"); + std::fs::write(&staged, ROTATED_ASSERTION).unwrap(); + std::fs::rename(&staged, &token_file).unwrap(); + store.put(&location(), vec![2].into()).await.unwrap(); + + let exchanges = login.requests(); + assert_eq!(exchanges[0].form()["client_assertion"], ASSERTION); + assert_eq!(exchanges[1].form()["client_assertion"], ROTATED_ASSERTION); +} + +#[tokio::test(flavor = "current_thread")] +async fn untrusted_or_misnamed_token_endpoints_receive_no_assertion() { + let trusted = TestCa::new(); + let untrusted = TestCa::new(); + let directory = tempfile::tempdir().unwrap(); + let token_file = write_token(directory.path(), ASSERTION); + let blobs = Server::start(trusted.acceptor("localhost"), storage()).await; + + for acceptor in [ + untrusted.acceptor("localhost"), + trusted.acceptor("other.example"), + ] { + let login = Server::start(acceptor, entra(3600)).await; + let store = store( + &login.url(), + &blobs.url(), + &token_file, + trusted.client_options(), + ); + let error = store.put(&location(), vec![1].into()).await.unwrap_err(); + assert!(matches!( + token_error(&error), + TokenAcquisitionError::Exchange { .. } + )); + assert!(error.to_string().contains("certificate"), "{error}"); + assert_private(&full_chain(&error)); + assert!(login.requests().is_empty()); + } + assert!(blobs.requests().is_empty()); +} + +#[tokio::test(flavor = "current_thread")] +async fn untrusted_or_misnamed_blob_endpoints_receive_no_bearer_token() { + let trusted = TestCa::new(); + let untrusted = TestCa::new(); + let directory = tempfile::tempdir().unwrap(); + let token_file = write_token(directory.path(), ASSERTION); + let login = Server::start(trusted.acceptor("localhost"), entra(3600)).await; + + for acceptor in [ + untrusted.acceptor("localhost"), + trusted.acceptor("other.example"), + ] { + let blobs = Server::start(acceptor, storage()).await; + let store = store( + &login.url(), + &blobs.url(), + &token_file, + trusted.client_options(), + ); + let error = store.put(&location(), vec![1].into()).await.unwrap_err(); + assert!(full_chain(&error).contains("certificate"), "{error:?}"); + assert_private(&full_chain(&error)); + assert!(blobs.requests().is_empty()); + } +} + +#[tokio::test(flavor = "current_thread")] +async fn a_rejected_exchange_reports_diagnostics_but_not_the_response_text() { + let ca = TestCa::new(); + let directory = tempfile::tempdir().unwrap(); + let token_file = write_token(directory.path(), ASSERTION); + let body = format!( + r#"{{"error":"invalid_client","error_description":"bad assertion {ASSERTION}", + "error_codes":[70021],"trace_id":"11111111-2222","correlation_id":"33333333-4444", + "access_token":"secret-echoed-token"}}"# + ); + let login = Server::start(ca.acceptor("localhost"), failing(401, Vec::new(), &body)).await; + let blobs = Server::start(ca.acceptor("localhost"), storage()).await; + let store = store(&login.url(), &blobs.url(), &token_file, ca.client_options()); + + let error = store.put(&location(), vec![1].into()).await.unwrap_err(); + + let TokenAcquisitionError::Exchange { detail } = token_error(&error) else { + panic!("expected the exchange stage: {error:?}") + }; + for expected in [ + "401", + "error=invalid_client", + "AADSTS70021", + "trace_id=11111111-2222", + "correlation_id=33333333-4444", + ] { + assert!(detail.contains(expected), "missing {expected}: {detail}"); + } + assert_private(&full_chain(&error)); + assert_eq!(login.requests().len(), 1); + assert!(blobs.requests().is_empty()); +} + +#[tokio::test(flavor = "current_thread")] +async fn an_unusable_token_response_does_not_repeat_its_body() { + let ca = TestCa::new(); + let directory = tempfile::tempdir().unwrap(); + let token_file = write_token(directory.path(), ASSERTION); + let body = r#"{"access_token":"secret-token","expires_in":"secret-expiry"}"#; + let login = Server::start(ca.acceptor("localhost"), failing(200, Vec::new(), body)).await; + let blobs = Server::start(ca.acceptor("localhost"), storage()).await; + let store = store(&login.url(), &blobs.url(), &token_file, ca.client_options()); + + let error = store.put(&location(), vec![1].into()).await.unwrap_err(); + + assert!(matches!( + token_error(&error), + TokenAcquisitionError::InvalidResponse + )); + assert_private(&full_chain(&error)); +} + +#[tokio::test(flavor = "current_thread")] +async fn an_unreadable_token_file_is_reported_without_contacting_entra() { + let ca = TestCa::new(); + let directory = tempfile::tempdir().unwrap(); + let token_file = directory.path().join("missing"); + let login = Server::start(ca.acceptor("localhost"), entra(3600)).await; + let blobs = Server::start(ca.acceptor("localhost"), storage()).await; + let store = store(&login.url(), &blobs.url(), &token_file, ca.client_options()); + + let error = store.put(&location(), vec![1].into()).await.unwrap_err(); + + let TokenAcquisitionError::TokenFile { path, reason } = token_error(&error) else { + panic!("expected the token file stage: {error:?}") + }; + assert_eq!(path, token_file.to_str().unwrap()); + assert_eq!(*reason, Some(io::ErrorKind::NotFound)); + assert!(error.to_string().contains("AZURE_FEDERATED_TOKEN_FILE")); + assert!(login.requests().is_empty()); + assert!(blobs.requests().is_empty()); +} + +#[tokio::test(flavor = "current_thread")] +async fn blob_authorization_failures_stay_typed_and_private() { + let ca = TestCa::new(); + let directory = tempfile::tempdir().unwrap(); + let token_file = write_token(directory.path(), ASSERTION); + let login = Server::start(ca.acceptor("localhost"), entra(3600)).await; + let body = "AuthorizationPermissionMismatch\ + echoed Bearer secret-access-token-1"; + + for (status, code) in [ + (403, "AuthorizationPermissionMismatch"), + (401, "InvalidAuth"), + ] { + let headers = vec![ + ("x-ms-error-code", code.to_owned()), + ("x-ms-request-id", "request-1234".to_owned()), + ]; + let blobs = Server::start(ca.acceptor("localhost"), failing(status, headers, body)).await; + let store = store(&login.url(), &blobs.url(), &token_file, ca.client_options()); + + let error = store.put(&location(), vec![1].into()).await.unwrap_err(); + + match status { + 403 => assert!(matches!( + error, + object_store::Error::PermissionDenied { .. } + )), + _ => assert!(matches!(error, object_store::Error::Unauthenticated { .. })), + } + let text = full_chain(&error); + assert!(text.contains(&format!("storage_error={code}")), "{text}"); + assert!(text.contains("request_id=request-1234"), "{text}"); + assert_private(&text); + } +} + +#[tokio::test(flavor = "current_thread")] +async fn store_debug_output_hides_the_cached_access_token() { + let ca = TestCa::new(); + let directory = tempfile::tempdir().unwrap(); + let token_file = write_token(directory.path(), ASSERTION); + let login = Server::start(ca.acceptor("localhost"), entra(3600)).await; + let blobs = Server::start(ca.acceptor("localhost"), storage()).await; + let store = store(&login.url(), &blobs.url(), &token_file, ca.client_options()); + store.put(&location(), vec![1].into()).await.unwrap(); + + // The cache is populated here, so a leak would show in this output. + assert_eq!(login.requests().len(), 1); + let debug = format!("{store:?} {store}"); + assert!(debug.contains("FederatedCredentialProvider"), "{debug}"); + assert_private(&debug); +} + +#[tokio::test(flavor = "current_thread")] +async fn building_the_store_opens_no_connection() { + let ca = TestCa::new(); + let login = Server::start(ca.acceptor("localhost"), entra(3600)).await; + let blobs = Server::start(ca.acceptor("localhost"), storage()).await; + + let _store = store( + &login.url(), + &blobs.url(), + Path::new("/nonexistent/projected/token"), + ca.client_options(), + ); + tokio::task::yield_now().await; + + assert!(login.requests().is_empty()); + assert!(blobs.requests().is_empty()); +} + +#[tokio::test(flavor = "current_thread")] +async fn plaintext_authorities_and_endpoints_are_never_used() { + let ca = TestCa::new(); + let directory = tempfile::tempdir().unwrap(); + let token_file = write_token(directory.path(), ASSERTION); + + let plaintext_authority = build( + AzureObjectStoreConfig { + account: "account".into(), + container: "container".into(), + endpoint: None, + credential: AzureObjectStoreCredential::WorkloadIdentity(AzureWorkloadIdentity { + client_id: CLIENT_ID.into(), + tenant_id: TENANT_ID.into(), + federated_token_file: token_file.to_str().unwrap().into(), + authority_host: Some("http://localhost:9".into()), + }), + }, + ca.client_options(), + no_retries(), + ); + assert!(matches!( + plaintext_authority, + Err(object_store::Error::Generic { .. }) + )); + + // Configuration rejects this endpoint first; the client must still refuse it + // for a direct caller, after a real token has been obtained. + let login = Server::start(ca.acceptor("localhost"), entra(3600)).await; + let listener = TcpListener::bind(("127.0.0.1", 0)).await.unwrap(); + let endpoint = format!("http://127.0.0.1:{}", listener.local_addr().unwrap().port()); + let store = store(&login.url(), &endpoint, &token_file, ca.client_options()); + + let error = store.put(&location(), vec![1].into()).await.unwrap_err(); + + assert_eq!(login.requests().len(), 1); + assert!(full_chain(&error).contains("scheme"), "{error:?}"); + assert_private(&full_chain(&error)); + let accepted = tokio::time::timeout(Duration::from_millis(50), listener.accept()).await; + assert!(accepted.is_err(), "the plaintext endpoint was contacted"); +} + +#[test] +fn authority_hosts_normalize_to_one_form() { + let identity = |authority: Option<&str>| AzureWorkloadIdentity { + client_id: CLIENT_ID.into(), + tenant_id: TENANT_ID.into(), + federated_token_file: "/token".into(), + authority_host: authority.map(Into::into), + }; + assert_eq!(identity(None).normalized_authority_host(), Ok(None)); + for authority in [ + "https://login.microsoftonline.com", + "https://login.microsoftonline.com/", + "HTTPS://LOGIN.microsoftonline.com/", + "https://login.microsoftonline.com:443/", + ] { + assert_eq!( + identity(Some(authority)).normalized_authority_host(), + Ok(Some("https://login.microsoftonline.com".to_owned())), + "{authority}" + ); + } + assert_eq!( + identity(Some("https://login.example:8443/")).normalized_authority_host(), + Ok(Some("https://login.example:8443".to_owned())) + ); +} + +#[test] +fn diagnostic_summaries_keep_only_identifiers() { + let empty = HeaderMap::new(); + assert_eq!(diagnostic_summary(&empty, b""), "details withheld"); + assert_eq!( + diagnostic_summary(&empty, b"plain secret-text"), + "details withheld" + ); + // Dotted values such as JWTs are not identifiers, even in an allowlisted field. + assert_eq!( + diagnostic_summary(&empty, br#"{"error":"header.payload.signature"}"#), + "details withheld" + ); + let oversized = format!(r#"{{"error":"{}"}}"#, "a".repeat(MAX_DIAGNOSTIC_LEN + 1)); + assert_eq!( + diagnostic_summary(&empty, oversized.as_bytes()), + "details withheld" + ); + assert_eq!( + diagnostic_summary(&empty, b"BlobNotFound"), + "storage_error=BlobNotFound" + ); +} diff --git a/crates/store-pg/src/lib.rs b/crates/store-pg/src/lib.rs index 0e4977f9..0b1daec1 100644 --- a/crates/store-pg/src/lib.rs +++ b/crates/store-pg/src/lib.rs @@ -39,7 +39,10 @@ use uuid::Uuid; pub use access::AccessFilter; pub use access::{AccessStoreError, PgAccessStore, ResourceAccess}; -pub use azure::{AzureObjectStoreConfig, AzureObjectStoreCredential, build_azure_object_store}; +pub use azure::{ + AzureObjectStoreConfig, AzureObjectStoreCredential, AzureWorkloadIdentity, + AzureWorkloadIdentityError, build_azure_object_store, +}; pub use model_defaults::ModelDefaultsStoreError; pub use s3::{S3ObjectStoreConfig, build_s3_object_store}; diff --git a/crates/store-pg/tests/azure_blob_live.rs b/crates/store-pg/tests/azure_blob_live.rs index 6fdade59..40907c49 100644 --- a/crates/store-pg/tests/azure_blob_live.rs +++ b/crates/store-pg/tests/azure_blob_live.rs @@ -1,4 +1,8 @@ //! Explicit Azure/Azurite and PostgreSQL integration. Never reads .env. +//! +//! With `LIGHTSPEED_TEST_AZURE_AUTH_MODE=workload_identity`, run this from a +//! workload holding a federated identity, against an account with Shared Key +//! access disabled, so that only the projected token can authorize the requests. use std::{env, panic::AssertUnwindSafe, str::FromStr, sync::Arc}; use futures_util::{FutureExt as _, TryStreamExt as _}; @@ -12,21 +16,37 @@ use sqlx::{ postgres::{PgConnectOptions, PgPoolOptions}, }; use store_pg::{ - AzureObjectStoreConfig, AzureObjectStoreCredential, PgStore, PgStoreConfig, - build_azure_object_store, delete_objects_under_prefix, universe_cas_object_prefix, + AzureObjectStoreConfig, AzureObjectStoreCredential, AzureWorkloadIdentity, PgStore, + PgStoreConfig, build_azure_object_store, delete_objects_under_prefix, + universe_cas_object_prefix, }; use uuid::Uuid; fn azure_config() -> AzureObjectStoreConfig { - let credential = match ( - env::var("LIGHTSPEED_TEST_AZURE_ACCESS_KEY").ok(), - env::var("LIGHTSPEED_TEST_AZURE_SAS_TOKEN").ok(), - ) { - (Some(key), None) => AzureObjectStoreCredential::AccessKey(key), - (None, Some(token)) => AzureObjectStoreCredential::SasToken(token), - _ => panic!( - "set exactly one of LIGHTSPEED_TEST_AZURE_ACCESS_KEY or LIGHTSPEED_TEST_AZURE_SAS_TOKEN" - ), + let access_key = env::var("LIGHTSPEED_TEST_AZURE_ACCESS_KEY").ok(); + let sas_token = env::var("LIGHTSPEED_TEST_AZURE_SAS_TOKEN").ok(); + let credential = match env::var("LIGHTSPEED_TEST_AZURE_AUTH_MODE").ok().as_deref() { + Some("workload_identity") => { + assert!( + access_key.is_none() && sas_token.is_none(), + "workload identity runs must not configure an access key or SAS token" + ); + AzureObjectStoreCredential::WorkloadIdentity(AzureWorkloadIdentity { + client_id: env::var("AZURE_CLIENT_ID").expect("AZURE_CLIENT_ID required"), + tenant_id: env::var("AZURE_TENANT_ID").expect("AZURE_TENANT_ID required"), + federated_token_file: env::var("AZURE_FEDERATED_TOKEN_FILE") + .expect("AZURE_FEDERATED_TOKEN_FILE required"), + authority_host: env::var("AZURE_AUTHORITY_HOST").ok(), + }) + } + None => match (access_key, sas_token) { + (Some(key), None) => AzureObjectStoreCredential::AccessKey(key), + (None, Some(token)) => AzureObjectStoreCredential::SasToken(token), + _ => panic!( + "set exactly one of LIGHTSPEED_TEST_AZURE_ACCESS_KEY or LIGHTSPEED_TEST_AZURE_SAS_TOKEN" + ), + }, + Some(mode) => panic!("unsupported LIGHTSPEED_TEST_AZURE_AUTH_MODE: {mode}"), }; AzureObjectStoreConfig { account: env::var("LIGHTSPEED_TEST_AZURE_ACCOUNT") @@ -39,7 +59,7 @@ fn azure_config() -> AzureObjectStoreConfig { } #[tokio::test(flavor = "current_thread")] -#[ignore = "requires explicit LIGHTSPEED_TEST_POSTGRES_URL and LIGHTSPEED_TEST_AZURE_*; uses a unique schema and object prefix"] +#[ignore = "requires explicit LIGHTSPEED_TEST_POSTGRES_URL and LIGHTSPEED_TEST_AZURE_* (plus AZURE_* for workload identity); uses a unique schema and object prefix"] async fn azure_blobs_survive_reconnection_and_remain_isolated_between_universes() { let url = env::var("LIGHTSPEED_TEST_POSTGRES_URL").expect("explicit test PostgreSQL URL required"); diff --git a/crates/temporal-runtime/src/config/object_storage.rs b/crates/temporal-runtime/src/config/object_storage.rs index abad2f73..c76f294b 100644 --- a/crates/temporal-runtime/src/config/object_storage.rs +++ b/crates/temporal-runtime/src/config/object_storage.rs @@ -2,7 +2,7 @@ use std::{env, sync::Arc}; use object_store::ObjectStore; use store_pg::{ - AzureObjectStoreConfig, AzureObjectStoreCredential, S3ObjectStoreConfig, + AzureObjectStoreConfig, AzureObjectStoreCredential, AzureWorkloadIdentity, S3ObjectStoreConfig, build_azure_object_store, build_s3_object_store, }; @@ -16,7 +16,14 @@ const ACCOUNT: &str = "LIGHTSPEED_OBJECT_STORE_AZURE_ACCOUNT"; const CONTAINER: &str = "LIGHTSPEED_OBJECT_STORE_AZURE_CONTAINER"; const ACCESS_KEY: &str = "LIGHTSPEED_OBJECT_STORE_AZURE_ACCESS_KEY"; const SAS_TOKEN: &str = "LIGHTSPEED_OBJECT_STORE_AZURE_SAS_TOKEN"; -const AZURE_KEYS: [&str; 4] = [ACCOUNT, CONTAINER, ACCESS_KEY, SAS_TOKEN]; +const AUTH_MODE: &str = "LIGHTSPEED_OBJECT_STORE_AZURE_AUTH_MODE"; +// The standard identity variables are read only in workload identity mode. They +// are not in AZURE_KEYS because other software in the pod may set them. +const CLIENT_ID: &str = "AZURE_CLIENT_ID"; +const TENANT_ID: &str = "AZURE_TENANT_ID"; +const TOKEN_FILE: &str = "AZURE_FEDERATED_TOKEN_FILE"; +const AUTHORITY_HOST: &str = "AZURE_AUTHORITY_HOST"; +const AZURE_KEYS: [&str; 5] = [ACCOUNT, CONTAINER, ACCESS_KEY, SAS_TOKEN, AUTH_MODE]; const S3_KEYS: [&str; 3] = [BUCKET, REGION, PATH_STYLE]; #[derive(Debug)] @@ -33,7 +40,7 @@ impl ObjectStorageConfig { // Builder errors can include malformed SAS values. Report setting names only. anyhow::anyhow!( "invalid Azure object-store configuration; check {ACCOUNT}, {CONTAINER}, \ - {ENDPOINT}, and the credential in {ACCESS_KEY} or {SAS_TOKEN}" + {ENDPOINT}, and the credential selected by {AUTH_MODE}" ) }), } @@ -83,16 +90,15 @@ fn parse(get: impl Fn(&str) -> Option) -> anyhow::Result AzureObjectStoreCredential::AccessKey(key), - (None, Some(token)) => AzureObjectStoreCredential::SasToken(token), - _ => anyhow::bail!( - "set exactly one of {ACCESS_KEY} or {SAS_TOKEN} for Azure storage" - ), - }; + let credential = azure_credential(&get)?; let endpoint = value(ENDPOINT); if let Some(endpoint) = &endpoint { validate_azure_endpoint(endpoint)?; + anyhow::ensure!( + !matches!(credential, AzureObjectStoreCredential::WorkloadIdentity(_)) + || endpoint.starts_with("https://"), + "{ENDPOINT} must use HTTPS with {AUTH_MODE}=workload_identity" + ); } ObjectStorageConfig::Azure(AzureObjectStoreConfig { account: required(ACCOUNT)?, @@ -106,6 +112,55 @@ fn parse(get: impl Fn(&str) -> Option) -> anyhow::Result Option, +) -> anyhow::Result { + let value = |key: &str| get(key).filter(|value| !value.is_empty()); + let (access_key, sas_token) = (value(ACCESS_KEY), value(SAS_TOKEN)); + match value(AUTH_MODE).as_deref() { + Some("workload_identity") => { + for key in [ACCESS_KEY, SAS_TOKEN] { + anyhow::ensure!( + get(key).is_none(), + "{key} conflicts with {AUTH_MODE}=workload_identity" + ); + } + let required = |key: &str| { + value(key).ok_or_else(|| { + anyhow::anyhow!("{key} must be set when {AUTH_MODE}=workload_identity") + }) + }; + let identity = AzureWorkloadIdentity { + client_id: required(CLIENT_ID)?, + tenant_id: required(TENANT_ID)?, + federated_token_file: required(TOKEN_FILE)?, + authority_host: value(AUTHORITY_HOST), + }; + identity.validate()?; + Ok(AzureObjectStoreCredential::WorkloadIdentity(identity)) + } + Some("access_key") => match (access_key, sas_token) { + (Some(key), None) => Ok(AzureObjectStoreCredential::AccessKey(key)), + _ => anyhow::bail!("{AUTH_MODE}=access_key requires {ACCESS_KEY} and no {SAS_TOKEN}"), + }, + Some("sas_token") => match (access_key, sas_token) { + (None, Some(token)) => Ok(AzureObjectStoreCredential::SasToken(token)), + _ => anyhow::bail!("{AUTH_MODE}=sas_token requires {SAS_TOKEN} and no {ACCESS_KEY}"), + }, + None => match (access_key, sas_token) { + (Some(key), None) => Ok(AzureObjectStoreCredential::AccessKey(key)), + (None, Some(token)) => Ok(AzureObjectStoreCredential::SasToken(token)), + _ => anyhow::bail!( + "set exactly one of {ACCESS_KEY} or {SAS_TOKEN} for Azure storage, \ + or select another credential with {AUTH_MODE}" + ), + }, + Some(_) => { + anyhow::bail!("{AUTH_MODE} must be access_key, sas_token, or workload_identity") + } + } +} + fn validate_azure_endpoint(endpoint: &str) -> anyhow::Result<()> { let url = url::Url::parse(endpoint) .map_err(|_| anyhow::anyhow!("{ENDPOINT} must be a valid account endpoint URL"))?; @@ -153,6 +208,199 @@ mod tests { ] } + fn workload_identity() -> Vec<(&'static str, &'static str)> { + vec![ + (BACKEND, "azure"), + (AUTH_MODE, "workload_identity"), + (ACCOUNT, "account"), + (CONTAINER, "container"), + (PREFIX, "deployment"), + (CLIENT_ID, "client-id"), + (TENANT_ID, "tenant-id"), + ( + TOKEN_FILE, + "/var/run/secrets/azure/tokens/azure-identity-token", + ), + ] + } + + fn selected_identity(entries: &[(&str, &str)]) -> AzureWorkloadIdentity { + let Some(ObjectStorageConfig::Azure(config)) = config(entries).unwrap() else { + panic!("expected Azure") + }; + let AzureObjectStoreCredential::WorkloadIdentity(identity) = config.credential else { + panic!("expected workload identity") + }; + identity + } + + #[test] + fn workload_identity_reads_the_standard_azure_settings() { + let identity = selected_identity(&workload_identity()); + assert_eq!(identity.client_id, "client-id"); + assert_eq!(identity.tenant_id, "tenant-id"); + assert_eq!( + identity.federated_token_file, + "/var/run/secrets/azure/tokens/azure-identity-token" + ); + assert_eq!(identity.authority_host, None); + } + + #[test] + fn workload_identity_requires_each_identity_setting() { + for key in [CLIENT_ID, TENANT_ID, TOKEN_FILE] { + let mut entries = workload_identity(); + entries.retain(|(name, _)| *name != key); + let error = config(&entries).unwrap_err().to_string(); + assert!(error.contains(key), "{error}"); + + let mut entries = workload_identity(); + entries.retain(|(name, _)| *name != key); + entries.push((key, "")); + let error = config(&entries).unwrap_err().to_string(); + assert!(error.contains(key), "{error}"); + } + } + + #[test] + fn workload_identity_rejects_conflicting_credentials_without_values() { + for key in [ACCESS_KEY, SAS_TOKEN] { + let mut entries = workload_identity(); + entries.push((key, "secret-value")); + let error = config(&entries).unwrap_err().to_string(); + assert!(error.contains(key), "{error}"); + assert!(!error.contains("secret-value")); + } + } + + #[test] + fn workload_identity_validates_token_path_tenant_and_authority() { + for (key, invalid) in [ + (TOKEN_FILE, "relative/token"), + (TENANT_ID, "../other-tenant"), + (TENANT_ID, "tenant/path"), + (AUTHORITY_HOST, "http://login.microsoftonline.com"), + (AUTHORITY_HOST, "login.microsoftonline.com"), + (AUTHORITY_HOST, "https://login.microsoftonline.com/tenant"), + (AUTHORITY_HOST, "https://login.microsoftonline.com/?x=1"), + ( + AUTHORITY_HOST, + "https://user:secret@login.microsoftonline.com/", + ), + ] { + let mut entries = workload_identity(); + entries.retain(|(name, _)| *name != key); + entries.push((key, invalid)); + let error = config(&entries).unwrap_err().to_string(); + assert!(error.contains(key), "{invalid}: {error}"); + assert!(!error.contains("secret")); + } + for authority in [ + "https://login.microsoftonline.com", + "https://login.microsoftonline.com/", + "https://login.microsoftonline.us:8443/", + ] { + let mut entries = workload_identity(); + entries.push((AUTHORITY_HOST, authority)); + assert_eq!( + selected_identity(&entries).authority_host.as_deref(), + Some(authority) + ); + } + } + + #[test] + fn workload_identity_refuses_plaintext_endpoints() { + let mut entries = workload_identity(); + entries.push((ENDPOINT, "http://127.0.0.1:10000/account")); + assert!(config(&entries).is_err()); + + let mut entries = workload_identity(); + entries.push((ENDPOINT, "https://account.blob.core.windows.net")); + selected_identity(&entries); + } + + #[test] + fn building_workload_identity_needs_neither_token_file_nor_network() { + let entries = workload_identity(); + config(&entries).unwrap().unwrap().build().unwrap(); + } + + #[test] + fn unknown_auth_modes_are_rejected_without_echoing_the_value() { + let mut entries = azure(); + entries.push((AUTH_MODE, "managed-identity-secret")); + let error = config(&entries).unwrap_err().to_string(); + assert!(error.contains(AUTH_MODE), "{error}"); + assert!(!error.contains("managed-identity-secret")); + } + + #[test] + fn explicit_static_modes_select_only_their_credential() { + let select = |mode, extra: &[(&'static str, &'static str)]| { + let mut entries = vec![ + (BACKEND, "azure"), + (ACCOUNT, "account"), + (CONTAINER, "container"), + (AUTH_MODE, mode), + ]; + entries.extend_from_slice(extra); + config(&entries) + }; + let key = (ACCESS_KEY, "secret-key"); + let sas = (SAS_TOKEN, "sig=secret-sas"); + assert!(select("access_key", &[key]).is_ok()); + assert!(select("sas_token", &[sas]).is_ok()); + for result in [ + select("access_key", &[sas]), + select("access_key", &[key, sas]), + select("sas_token", &[key]), + select("sas_token", &[key, sas]), + ] { + assert!(result.is_err()); + } + } + + #[test] + fn standard_azure_variables_do_not_activate_storage_or_change_the_selection() { + let identity_variables = [ + (CLIENT_ID, "client-id"), + (TENANT_ID, "tenant-id"), + (TOKEN_FILE, "/token"), + (AUTHORITY_HOST, "not-a-url"), + ("AZURE_CLIENT_SECRET", "secret"), + ]; + assert!(config(&identity_variables).unwrap().is_none()); + + let mut s3 = vec![(BUCKET, "bucket")]; + s3.extend_from_slice(&identity_variables); + assert!(matches!( + config(&s3).unwrap().unwrap(), + ObjectStorageConfig::S3(_) + )); + + let mut access_key = azure(); + access_key.extend_from_slice(&identity_variables); + let Some(ObjectStorageConfig::Azure(selected)) = config(&access_key).unwrap() else { + panic!("expected Azure") + }; + assert!(matches!( + selected.credential, + AzureObjectStoreCredential::AccessKey(_) + )); + } + + #[test] + fn auth_mode_requires_the_azure_backend() { + for entries in [ + vec![(AUTH_MODE, "workload_identity")], + vec![(BUCKET, "bucket"), (AUTH_MODE, "workload_identity")], + ] { + let error = config(&entries).unwrap_err().to_string(); + assert!(error.contains(BACKEND) || error.contains(BUCKET), "{error}"); + } + } + #[test] fn storage_stays_optional_and_aws_credentials_do_not_enable_it() { assert!(config(&[]).unwrap().is_none()); diff --git a/docs/documentation/deployment/configuration.md b/docs/documentation/deployment/configuration.md index ca8202cc..c7ba2e37 100644 --- a/docs/documentation/deployment/configuration.md +++ b/docs/documentation/deployment/configuration.md @@ -161,9 +161,11 @@ LIGHTSPEED_OBJECT_STORE_AZURE_CONTAINER=lightspeed-content LIGHTSPEED_OBJECT_STORE_PREFIX=production ``` -Supply exactly one of `LIGHTSPEED_OBJECT_STORE_AZURE_ACCESS_KEY` or +Choose how the Runtime authenticates. With a static credential, supply exactly +one of `LIGHTSPEED_OBJECT_STORE_AZURE_ACCESS_KEY` or `LIGHTSPEED_OBJECT_STORE_AZURE_SAS_TOKEN` through the deployment's secret -mechanism. Create the container first. Do not carry S3 bucket, region, or +mechanism. Where account keys and SAS are prohibited, use +[workload identity](#azure-workload-identity). Create the container first. Do not carry S3 bucket, region, or path-style settings into this configuration. The client uses the account's standard HTTPS Blob endpoint unless `LIGHTSPEED_OBJECT_STORE_ENDPOINT` is set. For an emulator, the endpoint can be a loopback HTTP account URL, such as @@ -195,6 +197,45 @@ well as newly written content. replicas also increases the aggregate cache budget. Blob collection is a separate policy, described in [operations](operations.md#manage-retention-and-blob-collection). +### Azure workload identity + +Workload identity authenticates to Blob storage with a projected Kubernetes +service account token, so the deployment holds no account key, SAS token, or +client secret. Select it explicitly: + +```dotenv +LIGHTSPEED_OBJECT_STORE_BACKEND=azure +LIGHTSPEED_OBJECT_STORE_AZURE_AUTH_MODE=workload_identity +LIGHTSPEED_OBJECT_STORE_AZURE_ACCOUNT=yourstorageaccount +LIGHTSPEED_OBJECT_STORE_AZURE_CONTAINER=lightspeed-content +LIGHTSPEED_OBJECT_STORE_PREFIX=production + +AZURE_CLIENT_ID= +AZURE_TENANT_ID= +AZURE_FEDERATED_TOKEN_FILE= +``` + +The Azure Workload Identity webhook normally injects the `AZURE_*` values and +mounts the token. Set `AZURE_AUTHORITY_HOST` only for a cloud other than the +public one. On the Azure side, add a federated credential to the identity for +the Runtime's service account, and grant the identity a data-plane role that +can read, write, list, and delete blobs in the container, such as Storage Blob +Data Contributor. + +The Runtime exchanges the projected token for an Azure Storage access token when +it first needs one, then renews it as it nears expiry. Each exchange reads the +token file again, so rotation does not require a restart. Commands that use only +the database, such as migrations, do not read the file or contact Microsoft +Entra. Consequently, a wrong token path or a missing federated credential +appears at the first Blob operation rather than at startup. The error names the +stage: the token file, the Entra exchange with its `AADSTS` code, or the Blob +request with its storage error code and request ID. + +This mode rejects an access key or SAS token rather than choosing between them, +and it never falls back to another credential. Existing objects, keys, and +prefixes stay valid when an account moves to this mode; no data migration is +needed. + ## Add optional services ### Execution environments diff --git a/docs/documentation/deployment/troubleshooting.md b/docs/documentation/deployment/troubleshooting.md index 58380b3f..ef0e7ada 100644 --- a/docs/documentation/deployment/troubleshooting.md +++ b/docs/documentation/deployment/troubleshooting.md @@ -46,7 +46,7 @@ If the runtime exits before serving, check the error against these dependencies: | PostgreSQL connection | The runtime database URL, credentials, network path, and required connection options. Container-local `localhost` is not the database host. | | Schema verification | Use the deployed release's `schema-version` diagnostic. The runtime never migrates implicitly; follow the explicit migration procedure. | | Temporal connection | The frontend address and an existing namespace, reachable from the process. | -| Object-store configuration | S3 requires a bucket; `LIGHTSPEED_OBJECT_STORE_BACKEND=azure` requires account, container, and exactly one access key or SAS token. Check for settings mixed between backends and use the intended endpoint. | +| Object-store configuration | S3 requires a bucket; `LIGHTSPEED_OBJECT_STORE_BACKEND=azure` requires account, container, and exactly one access key or SAS token, or `LIGHTSPEED_OBJECT_STORE_AZURE_AUTH_MODE=workload_identity` with `AZURE_CLIENT_ID`, `AZURE_TENANT_ID`, and an absolute `AZURE_FEDERATED_TOKEN_FILE`. Check for settings mixed between backends and use the intended endpoint. | | Role or environment routing | Valid roles and the internal environment gateway URL/token on every process without that role. | | Secret configuration | A base64 master key decoding to 32 bytes, matching the stored encrypted state. | | Retired model variables | Remove `LIGHTSPEED_CHAT_PROVIDER` and `LIGHTSPEED_CHAT_MODEL`; configure each universe's model defaults through Models or `lightspeed model defaults`. | diff --git a/docs/documentation/reference/environment-variables.md b/docs/documentation/reference/environment-variables.md index ecdf4cd9..e7a8f915 100644 --- a/docs/documentation/reference/environment-variables.md +++ b/docs/documentation/reference/environment-variables.md @@ -130,12 +130,20 @@ Existing configurations default to S3; Azure must be selected explicitly. | `AWS_SECRET_ACCESS_KEY` | Credential-chain dependent | S3 secret key. | | `LIGHTSPEED_OBJECT_STORE_AZURE_ACCOUNT` | Required for Azure | Azure storage account name. | | `LIGHTSPEED_OBJECT_STORE_AZURE_CONTAINER` | Required for Azure | Existing Blob container name. | -| `LIGHTSPEED_OBJECT_STORE_AZURE_ACCESS_KEY` | Exactly one Azure credential | Storage account access key. Supply through a Secret. | -| `LIGHTSPEED_OBJECT_STORE_AZURE_SAS_TOKEN` | Exactly one Azure credential | SAS token, with or without a leading `?`. Supply through a Secret; permit the read/write/list/delete operations the blob store uses. | - -Azure requires exactly one of access key or SAS token; it does not fall back to -managed identity. S3-only bucket, region, and path-style settings are rejected -with Azure. Both backends use normal HTTPS certificate trust by default; no +| `LIGHTSPEED_OBJECT_STORE_AZURE_AUTH_MODE` | Inferred from the static credential | `access_key`, `sas_token`, or `workload_identity`. Unset selects whichever single static credential is present. | +| `LIGHTSPEED_OBJECT_STORE_AZURE_ACCESS_KEY` | Required for `access_key` | Storage account access key. Supply through a Secret. Rejected with `workload_identity`. | +| `LIGHTSPEED_OBJECT_STORE_AZURE_SAS_TOKEN` | Required for `sas_token` | SAS token, with or without a leading `?`. Supply through a Secret; permit the read/write/list/delete operations the blob store uses. Rejected with `workload_identity`. | +| `AZURE_CLIENT_ID` | Required for `workload_identity` | Client ID of the federated Microsoft Entra identity. | +| `AZURE_TENANT_ID` | Required for `workload_identity` | Tenant ID or domain name. | +| `AZURE_FEDERATED_TOKEN_FILE` | Required for `workload_identity` | Absolute path of the projected service account token. Read again for each token exchange, so rotation needs no restart. | +| `AZURE_AUTHORITY_HOST` | `https://login.microsoftonline.com` | Microsoft Entra authority for `workload_identity`. Must be HTTPS with no path, query, or credentials; a trailing slash is optional. | + +Azure uses only the selected credential and never falls back to another one, +such as the Azure CLI, a node or VM identity, or a client secret. S3-only bucket, +region, and path-style settings are rejected with Azure. The `AZURE_*` variables +are read only with `workload_identity`; they do not activate object storage or +change the backend. That mode requires an HTTPS endpoint, even for loopback +addresses. Both backends use normal HTTPS certificate trust by default; no certificate file is required for a server trusted by the container's CA roots. Keep the backend, account/container, and existing objects consistent across restarts: PostgreSQL records physical keys, not enough information to migrate diff --git a/docs/roadmap/azure-storage-and-postgres-tls.md b/docs/roadmap/azure-storage-and-postgres-tls.md index 9b34b931..ca0e0c42 100644 --- a/docs/roadmap/azure-storage-and-postgres-tls.md +++ b/docs/roadmap/azure-storage-and-postgres-tls.md @@ -52,3 +52,39 @@ denied, formatting, and documentation checks passed. Actual AKS, Azure PostgreSQL, and Azure Blob validation remains part of deployment acceptance; emulator tests do not establish bank connectivity. + +## Workload identity + +**Status:** Implemented and verified against a local TLS fake of Entra and Azure +Storage, 2026-11-05. Not yet run against a real storage account. + +`LIGHTSPEED_OBJECT_STORE_AZURE_AUTH_MODE=workload_identity` selects `object_store`'s +workload identity provider explicitly, using `AZURE_CLIENT_ID`, `AZURE_TENANT_ID`, +`AZURE_FEDERATED_TOKEN_FILE`, and an optional `AZURE_AUTHORITY_HOST`. The library +performs the exchange, caches access tokens, and re-reads the token file for each +exchange. Its token client and Blob client share one set of client options, so +they share certificate trust. + +Two wrappers in `store-pg` keep credentials private, because the library alone +does not: + +- Its provider's derived `Debug` prints the cached access token, so the store holds + it behind a provider whose `Debug` shows only the file path. That provider also + labels token failures by stage. +- Its request errors embed raw response bodies, so a connector reduces every error + body to allowlisted identifiers: Entra's error, `AADSTS` code, trace and correlation + IDs, and the storage error code and request ID. Free text such as + `error_description` and `AuthenticationErrorDetail` is dropped, so diagnosis beyond + the code uses those IDs in the service's own logs. + +`cargo test -p store-pg --lib azure` runs the real exchange and Blob client against +the fake over TLS, with certificates generated per test. It covers the wire content, +rotation through a symlink swap and a file replacement, rejected certificates and +hostnames on both endpoints, failed authentication, and redaction. Those tests trust +only their generated CA, so they exercise `webpki` rather than the platform verifier +that production uses with its default options. + +`azure_blob_live` also runs in this mode with +`LIGHTSPEED_TEST_AZURE_AUTH_MODE=workload_identity` and the `AZURE_*` settings, and +refuses to start if a test access key or SAS token is set. Run it from a workload with +a federated identity, against an account with Shared Key access disabled. From 36fee9f5a35e36216a950d1a408172ad242f20c7 Mon Sep 17 00:00:00 2001 From: lb <542828+lukebuehler@users.noreply.github.com> Date: Thu, 5 Nov 2026 14:44:02 +0100 Subject: [PATCH 2/3] Add Entra workload identity login for PostgreSQL The Runtime and Platform can now log in to Azure Database for PostgreSQL with a Microsoft Entra token instead of a stored password. Each selects it with its own setting, LIGHTSPEED_POSTGRES_AUTH_MODE or LIGHTSPEED_PLATFORM_DATABASE_AUTH_MODE, and password remains the default. The URL must name the identity, hold no password, and set sslmode=verify-full, because the token is sent as a password inside TLS. The Runtime exchanges the projected token itself and updates the pool's connect options at half of each token's lifetime. The Platform uses @azure/identity through pg's password function. Temporal stays on password authentication and is planned separately. Pool settings now limit reconnects in both modes. The Platform pool also gains an error listener, because losing an idle connection previously ended the process, and TCP keepalive, because pg does not check connections before handing them out. Verified with unit tests, a token rotation test against a real PostgreSQL role, and the Platform migration and release packaging checks. Not yet run against a real Azure server; azure_postgres_live covers that from a federated workload. Co-Authored-By: Claude Opus 5.5 Signed-off-by: lb <542828+lukebuehler@users.noreply.github.com> --- Cargo.lock | 1 + crates/store-pg/Cargo.toml | 3 +- crates/store-pg/src/azure.rs | 8 +- crates/store-pg/src/azure/postgres.rs | 228 +++++++++++ crates/store-pg/src/azure/postgres/tests.rs | 333 ++++++++++++++++ crates/store-pg/src/azure/test_support.rs | 224 +++++++++++ .../store-pg/src/azure/workload_identity.rs | 10 +- .../src/azure/workload_identity/tests.rs | 219 +---------- crates/store-pg/src/lib.rs | 15 +- crates/store-pg/tests/azure_postgres_live.rs | 38 ++ crates/temporal-runtime/src/config.rs | 16 +- .../src/config/object_storage.rs | 34 +- .../temporal-runtime/src/config/postgres.rs | 102 +++++ .../documentation/deployment/configuration.md | 41 ++ .../deployment/troubleshooting.md | 2 +- .../reference/environment-variables.md | 8 +- .../roadmap/azure-storage-and-postgres-tls.md | 46 +++ package-lock.json | 362 +++++++++++++++++- platform/backend/src/env.test.ts | 13 + platform/backend/src/env.ts | 5 + platform/backend/src/main.ts | 2 +- platform/db/package.json | 9 +- platform/db/src/index.ts | 32 +- platform/db/src/workload-identity.test.ts | 326 ++++++++++++++++ platform/db/src/workload-identity.ts | 160 ++++++++ 25 files changed, 1983 insertions(+), 254 deletions(-) create mode 100644 crates/store-pg/src/azure/postgres.rs create mode 100644 crates/store-pg/src/azure/postgres/tests.rs create mode 100644 crates/store-pg/src/azure/test_support.rs create mode 100644 crates/store-pg/tests/azure_postgres_live.rs create mode 100644 crates/temporal-runtime/src/config/postgres.rs create mode 100644 platform/db/src/workload-identity.test.ts create mode 100644 platform/db/src/workload-identity.ts diff --git a/Cargo.lock b/Cargo.lock index 1a98b4ea..a8135ac9 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5161,6 +5161,7 @@ dependencies = [ "profiles", "rand 0.10.3", "rcgen", + "reqwest", "serde", "serde_json", "sha2 0.11.0", diff --git a/crates/store-pg/Cargo.toml b/crates/store-pg/Cargo.toml index 5bcbae78..ad6f452b 100644 --- a/crates/store-pg/Cargo.toml +++ b/crates/store-pg/Cargo.toml @@ -21,13 +21,14 @@ mcp = { path = "../mcp" } object_store = { version = "0.14", features = ["aws", "azure"] } profiles = { path = "../profiles" } rand = "0.10" +reqwest = { version = "0.13", default-features = false, features = ["form", "rustls"] } sha2 = "0.11" serde = { version = "1", features = ["derive"] } serde_json = "1" sqlx = { version = "0.9", default-features = false, features = ["runtime-tokio", "tls-rustls-ring-native-roots", "postgres", "uuid", "json"] } thiserror = "2" tracing = "0.1" -tokio = { version = "1", features = ["rt"] } +tokio = { version = "1", features = ["fs", "macros", "rt", "time"] } url = "2" uuid = { version = "1", features = ["v4"] } vfs = { path = "../vfs" } diff --git a/crates/store-pg/src/azure.rs b/crates/store-pg/src/azure.rs index 31e33ee8..a9fdc0b6 100644 --- a/crates/store-pg/src/azure.rs +++ b/crates/store-pg/src/azure.rs @@ -5,9 +5,15 @@ use object_store::{ azure::{AzureConfigKey, MicrosoftAzureBuilder}, }; +mod postgres; +#[cfg(test)] +mod test_support; mod workload_identity; -pub use workload_identity::{AzureWorkloadIdentity, AzureWorkloadIdentityError}; +pub use postgres::{AzurePostgresError, connect_postgres_with_workload_identity}; +pub use workload_identity::{ + AzureWorkloadIdentity, AzureWorkloadIdentityError, TokenAcquisitionError, +}; /// Azure credentials supplied by the deployment. Debug output is redacted. #[derive(Clone)] diff --git a/crates/store-pg/src/azure/postgres.rs b/crates/store-pg/src/azure/postgres.rs new file mode 100644 index 00000000..1a64e4e1 --- /dev/null +++ b/crates/store-pg/src/azure/postgres.rs @@ -0,0 +1,228 @@ +//! Microsoft Entra workload identity authentication for PostgreSQL. +//! +//! The server accepts an Entra access token as the login password. It is checked +//! when a connection opens, so only new connections need a current token. sqlx has +//! no per-connection credential callback, so a background task keeps the pool's +//! connect options current. Refreshing at half of each token's lifetime leaves the +//! other half to ride out an Entra outage before new connections start failing. +use std::{str::FromStr, sync::Arc, time::Duration}; + +use object_store::azure::authority_hosts::AZURE_PUBLIC_CLOUD; +use serde::Deserialize; +use sqlx::{ + PgPool, + postgres::{PgConnectOptions, PgPoolOptions, PgSslMode}, +}; +use thiserror::Error; +use url::Url; + +use super::workload_identity::{ + AzureWorkloadIdentity, AzureWorkloadIdentityError, TokenAcquisitionError, diagnostic_summary, + render_chain, +}; + +const POSTGRES_SCOPE: &str = "https://ossrdbms-aad.database.windows.net/.default"; +const CONNECT_TIMEOUT: Duration = Duration::from_secs(10); +const REQUEST_TIMEOUT: Duration = Duration::from_secs(30); +/// The shortest wait between exchanges, and the wait after a failed one. +const MIN_REFRESH_DELAY: Duration = Duration::from_secs(30); + +#[derive(Debug, Error)] +pub enum AzurePostgresError { + #[error(transparent)] + Identity(#[from] AzureWorkloadIdentityError), + #[error("the PostgreSQL URL could not be parsed")] + InvalidUrl, + #[error("the PostgreSQL URL must name the managed identity as its user")] + MissingUser, + #[error("the PostgreSQL URL must not contain a password with workload identity")] + PasswordInUrl, + /// The token travels as a cleartext password inside TLS, so the server's + /// identity must be verified before it is sent. + #[error("the PostgreSQL URL must set sslmode=verify-full with workload identity")] + TlsNotVerified, + #[error(transparent)] + Token(#[from] TokenAcquisitionError), + #[error("could not connect to PostgreSQL with the Entra token: {0}")] + Connect(#[source] sqlx::Error), +} + +/// Open a pool that authenticates as the workload's federated identity and renews +/// its token until the pool is closed. Holding a clone of the pool keeps that task +/// and the connections alive, so a pool that should end must be closed explicitly. +pub async fn connect_postgres_with_workload_identity( + database_url: &str, + identity: AzureWorkloadIdentity, +) -> Result { + let options = connect_options(database_url)?; + let client = reqwest::Client::builder() + .https_only(true) + // A redirect could carry the assertion to another host. + .redirect(reqwest::redirect::Policy::none()) + .connect_timeout(CONNECT_TIMEOUT) + .timeout(REQUEST_TIMEOUT) + .build() + .map_err(|error| TokenAcquisitionError::Exchange { + detail: render_chain(&error), + })?; + let source = TokenSource::new(client, identity)?; + connect( + options, + crate::postgres_pool_options(), + source, + MIN_REFRESH_DELAY, + ) + .await +} + +fn connect_options(database_url: &str) -> Result { + let url = Url::parse(database_url).map_err(|_| AzurePostgresError::InvalidUrl)?; + if url.username().is_empty() { + return Err(AzurePostgresError::MissingUser); + } + if url.password().is_some() || url.query_pairs().any(|(key, _)| key == "password") { + return Err(AzurePostgresError::PasswordInUrl); + } + let options = + PgConnectOptions::from_str(database_url).map_err(|_| AzurePostgresError::InvalidUrl)?; + if !matches!(options.get_ssl_mode(), PgSslMode::VerifyFull) { + return Err(AzurePostgresError::TlsNotVerified); + } + Ok(options) +} + +async fn connect( + options: PgConnectOptions, + pool_options: PgPoolOptions, + source: TokenSource, + min_refresh_delay: Duration, +) -> Result { + let token = source.fetch().await?; + let pool = pool_options + .connect_with(options.clone().password(&token.value)) + .await + .map_err(AzurePostgresError::Connect)?; + tokio::spawn(keep_token_current( + pool.clone(), + options, + source, + refresh_delay(token.expires_in, min_refresh_delay), + min_refresh_delay, + )); + Ok(pool) +} + +async fn keep_token_current( + pool: PgPool, + options: PgConnectOptions, + source: TokenSource, + mut delay: Duration, + min_delay: Duration, +) { + loop { + tokio::select! { + () = pool.close_event() => return, + () = tokio::time::sleep(delay) => {} + } + delay = match source.fetch().await { + Ok(token) => { + pool.set_connect_options(options.clone().password(&token.value)); + refresh_delay(token.expires_in, min_delay) + } + Err(error) => { + tracing::warn!(%error, "could not renew the PostgreSQL access token; open connections are unaffected"); + min_delay + } + }; + } +} + +fn refresh_delay(expires_in: Duration, min_delay: Duration) -> Duration { + (expires_in / 2).max(min_delay) +} + +/// Exchanges the projected token for PostgreSQL access tokens. The library's +/// provider used for Blob storage has its scope fixed to Azure Storage. +struct TokenSource { + client: reqwest::Client, + token_url: Arc, + client_id: String, + token_file: String, +} + +struct AccessToken { + value: String, + expires_in: Duration, +} + +#[derive(Deserialize)] +struct TokenResponse { + access_token: String, + expires_in: u64, +} + +impl TokenSource { + fn new( + client: reqwest::Client, + identity: AzureWorkloadIdentity, + ) -> Result { + identity.validate()?; + let authority = identity + .normalized_authority_host()? + .unwrap_or_else(|| AZURE_PUBLIC_CLOUD.to_owned()); + Ok(Self { + client, + token_url: format!("{authority}/{}/oauth2/v2.0/token", identity.tenant_id).into(), + client_id: identity.client_id, + token_file: identity.federated_token_file, + }) + } + + async fn fetch(&self) -> Result { + // Read for every exchange: the kubelet replaces this file as it rotates. + let assertion = tokio::fs::read_to_string(&self.token_file) + .await + .map_err(|error| TokenAcquisitionError::TokenFile { + path: self.token_file.clone(), + reason: Some(error.kind()), + })?; + let response = self + .client + .post(&*self.token_url) + .form(&[ + ("client_id", self.client_id.as_str()), + ( + "client_assertion_type", + "urn:ietf:params:oauth:client-assertion-type:jwt-bearer", + ), + ("client_assertion", assertion.as_str()), + ("scope", POSTGRES_SCOPE), + ("grant_type", "client_credentials"), + ]) + .send() + .await + .map_err(|error| TokenAcquisitionError::Exchange { + detail: render_chain(&error), + })?; + let status = response.status(); + let headers = response.headers().clone(); + let body = response + .bytes() + .await + .map_err(|_| TokenAcquisitionError::InvalidResponse)?; + if !status.is_success() { + return Err(TokenAcquisitionError::Exchange { + detail: format!("HTTP {status}: {}", diagnostic_summary(&headers, &body)), + }); + } + let token: TokenResponse = + serde_json::from_slice(&body).map_err(|_| TokenAcquisitionError::InvalidResponse)?; + Ok(AccessToken { + value: token.access_token, + expires_in: Duration::from_secs(token.expires_in), + }) + } +} + +#[cfg(test)] +mod tests; diff --git a/crates/store-pg/src/azure/postgres/tests.rs b/crates/store-pg/src/azure/postgres/tests.rs new file mode 100644 index 00000000..7917a6e7 --- /dev/null +++ b/crates/store-pg/src/azure/postgres/tests.rs @@ -0,0 +1,333 @@ +use std::{ + path::Path, + sync::{Arc, Mutex}, +}; + +use uuid::Uuid; + +use super::{ + super::test_support::{Handler, Response, Server, TestCa, entra}, + *, +}; + +const ASSERTION: &str = "secret-assertion.jwt.value"; +const ROTATED_ASSERTION: &str = "secret-rotated-assertion.jwt.value"; + +fn source(ca: &TestCa, login: &Server, token_file: &Path) -> TokenSource { + TokenSource::new( + ca.reqwest_client(), + AzureWorkloadIdentity { + client_id: "client-id".into(), + tenant_id: "tenant-id".into(), + federated_token_file: token_file.to_str().unwrap().into(), + authority_host: Some(format!("{}/", login.url())), + }, + ) + .unwrap() +} + +fn write_token(directory: &Path, contents: &str) -> std::path::PathBuf { + let path = directory.join("token"); + std::fs::write(&path, contents).unwrap(); + path +} + +fn failing(status: u16, body: &str) -> Arc { + let body = body.to_owned(); + Arc::new(move |_| Response::new(status, body.clone())) +} + +#[test] +fn urls_must_name_the_identity_verify_tls_and_hold_no_password() { + let base = "postgres://identity@db.example/lightspeed"; + assert!(connect_options(&format!("{base}?sslmode=verify-full")).is_ok()); + for (url, expected) in [ + ("not a url".to_owned(), "InvalidUrl"), + ( + "postgres://db.example/lightspeed?sslmode=verify-full".to_owned(), + "MissingUser", + ), + ( + "postgres://identity:secret-password@db.example/lightspeed?sslmode=verify-full" + .to_owned(), + "PasswordInUrl", + ), + ( + format!("{base}?sslmode=verify-full&password=secret-password"), + "PasswordInUrl", + ), + (base.to_owned(), "TlsNotVerified"), + (format!("{base}?sslmode=require"), "TlsNotVerified"), + (format!("{base}?sslmode=verify-ca"), "TlsNotVerified"), + ] { + let error = connect_options(&url).unwrap_err(); + assert!( + format!("{error:?}").starts_with(expected), + "{url}: {error:?}" + ); + assert!(!error.to_string().contains("secret-"), "{error}"); + } +} + +#[test] +fn deployment_pools_limit_reconnects_and_still_check_connections_before_use() { + let options = crate::postgres_pool_options(); + assert_eq!(options.get_min_connections(), 2); + assert_eq!(options.get_max_lifetime(), Some(Duration::from_secs(3600))); + assert_eq!(options.get_idle_timeout(), Some(Duration::from_secs(600))); + assert!(options.get_test_before_acquire()); +} + +#[test] +fn renewal_waits_half_a_lifetime_but_never_less_than_the_minimum() { + let minimum = Duration::from_secs(30); + assert_eq!( + refresh_delay(Duration::from_secs(3600), minimum), + Duration::from_secs(1800) + ); + assert_eq!(refresh_delay(Duration::from_secs(10), minimum), minimum); + assert_eq!(refresh_delay(Duration::ZERO, minimum), minimum); +} + +#[tokio::test(flavor = "current_thread")] +async fn the_exchange_presents_the_projected_token_for_the_postgres_scope() { + let ca = TestCa::new(); + let directory = tempfile::tempdir().unwrap(); + let token_file = write_token(directory.path(), ASSERTION); + let login = Server::start(ca.acceptor("localhost"), entra(3600)).await; + + let token = source(&ca, &login, &token_file).fetch().await.unwrap(); + + assert_eq!(token.value, "secret-access-token-1"); + assert_eq!(token.expires_in, Duration::from_secs(3600)); + let requests = login.requests(); + assert_eq!(requests.len(), 1); + assert_eq!(requests[0].method, "POST"); + assert_eq!(requests[0].path, "/tenant-id/oauth2/v2.0/token"); + let form = requests[0].form(); + assert_eq!(form["grant_type"], "client_credentials"); + assert_eq!(form["client_id"], "client-id"); + assert_eq!(form["client_assertion"], ASSERTION); + assert_eq!( + form["client_assertion_type"], + "urn:ietf:params:oauth:client-assertion-type:jwt-bearer" + ); + assert_eq!( + form["scope"], + "https://ossrdbms-aad.database.windows.net/.default" + ); +} + +#[tokio::test(flavor = "current_thread")] +async fn each_exchange_reads_the_current_projected_token() { + let ca = TestCa::new(); + let directory = tempfile::tempdir().unwrap(); + let token_file = write_token(directory.path(), ASSERTION); + let login = Server::start(ca.acceptor("localhost"), entra(3600)).await; + let source = source(&ca, &login, &token_file); + + source.fetch().await.unwrap(); + let staged = directory.path().join("staged"); + std::fs::write(&staged, ROTATED_ASSERTION).unwrap(); + std::fs::rename(&staged, &token_file).unwrap(); + source.fetch().await.unwrap(); + + let requests = login.requests(); + assert_eq!(requests[0].form()["client_assertion"], ASSERTION); + assert_eq!(requests[1].form()["client_assertion"], ROTATED_ASSERTION); +} + +#[tokio::test(flavor = "current_thread")] +async fn a_rejected_exchange_reports_diagnostics_but_not_the_response_text() { + let ca = TestCa::new(); + let directory = tempfile::tempdir().unwrap(); + let token_file = write_token(directory.path(), ASSERTION); + let body = format!( + r#"{{"error":"invalid_client","error_description":"bad {ASSERTION}","error_codes":[70021], + "trace_id":"trace-1","access_token":"secret-echoed"}}"# + ); + let login = Server::start(ca.acceptor("localhost"), failing(401, &body)).await; + + let Err(error) = source(&ca, &login, &token_file).fetch().await else { + panic!("expected a rejected exchange") + }; + + let TokenAcquisitionError::Exchange { detail } = &error else { + panic!("expected the exchange stage: {error:?}") + }; + for expected in [ + "401", + "error=invalid_client", + "AADSTS70021", + "trace_id=trace-1", + ] { + assert!(detail.contains(expected), "missing {expected}: {detail}"); + } + let rendered = format!("{error} {error:?}"); + assert!(!rendered.contains("secret-"), "{rendered}"); +} + +#[tokio::test(flavor = "current_thread")] +async fn an_unusable_token_response_does_not_repeat_its_body() { + let ca = TestCa::new(); + let directory = tempfile::tempdir().unwrap(); + let token_file = write_token(directory.path(), ASSERTION); + let body = r#"{"access_token":"secret-token","expires_in":"secret-expiry"}"#; + let login = Server::start(ca.acceptor("localhost"), failing(200, body)).await; + + let Err(error) = source(&ca, &login, &token_file).fetch().await else { + panic!("expected an unusable response") + }; + + assert!(matches!(error, TokenAcquisitionError::InvalidResponse)); + assert!(!format!("{error} {error:?}").contains("secret-")); +} + +#[tokio::test(flavor = "current_thread")] +async fn an_unreadable_token_file_is_reported_without_contacting_entra() { + let ca = TestCa::new(); + let directory = tempfile::tempdir().unwrap(); + let missing = directory.path().join("missing"); + let login = Server::start(ca.acceptor("localhost"), entra(3600)).await; + + let Err(error) = source(&ca, &login, &missing).fetch().await else { + panic!("expected an unreadable file") + }; + + let TokenAcquisitionError::TokenFile { path, reason } = &error else { + panic!("expected the token file stage: {error:?}") + }; + assert_eq!(path, missing.to_str().unwrap()); + assert_eq!(*reason, Some(std::io::ErrorKind::NotFound)); + assert!(login.requests().is_empty()); +} + +#[tokio::test(flavor = "current_thread")] +async fn untrusted_or_misnamed_token_endpoints_receive_no_assertion() { + let trusted = TestCa::new(); + let untrusted = TestCa::new(); + let directory = tempfile::tempdir().unwrap(); + let token_file = write_token(directory.path(), ASSERTION); + + for acceptor in [ + untrusted.acceptor("localhost"), + trusted.acceptor("other.example"), + ] { + let login = Server::start(acceptor, entra(3600)).await; + let Err(error) = source(&trusted, &login, &token_file).fetch().await else { + panic!("expected a certificate rejection") + }; + assert!(error.to_string().contains("certificate"), "{error}"); + assert!(login.requests().is_empty()); + } +} + +#[test] +fn plaintext_authorities_are_rejected_before_any_request() { + let identity = AzureWorkloadIdentity { + client_id: "client-id".into(), + tenant_id: "tenant-id".into(), + federated_token_file: "/token".into(), + authority_host: Some("http://login.example".into()), + }; + assert!(matches!( + TokenSource::new(reqwest::Client::new(), identity), + Err(AzureWorkloadIdentityError::InvalidAuthorityHost) + )); +} + +/// Fake Entra issues the role's real password as the token, so the database +/// authenticates each new connection against whatever was exchanged last. +#[tokio::test(flavor = "current_thread")] +#[ignore = "requires LIGHTSPEED_TEST_POSTGRES_URL with password authentication and rights to create roles"] +async fn new_connections_use_the_renewed_token_and_open_ones_keep_working() { + let admin_url = + std::env::var("LIGHTSPEED_TEST_POSTGRES_URL").expect("explicit test PostgreSQL URL"); + let admin = PgPoolOptions::new() + .max_connections(1) + .connect(&admin_url) + .await + .unwrap(); + let role = format!("entra_rotation_{}", Uuid::new_v4().simple()); + let alter_password = |password: &str| format!("ALTER ROLE {role} PASSWORD '{password}'"); + sqlx::raw_sql(sqlx::AssertSqlSafe(format!( + "CREATE ROLE {role} LOGIN PASSWORD 'first-password'" + ))) + .execute(&admin) + .await + .unwrap(); + + let current = Arc::new(Mutex::new("first-password".to_owned())); + let issued = current.clone(); + let handler: Arc = Arc::new(move |_| { + Response::new( + 200, + format!( + r#"{{"access_token":"{}","expires_in":1}}"#, + issued.lock().unwrap() + ), + ) + }); + let ca = TestCa::new(); + let directory = tempfile::tempdir().unwrap(); + let token_file = write_token(directory.path(), ASSERTION); + let login = Server::start(ca.acceptor("localhost"), handler).await; + let options = PgConnectOptions::from_str(&admin_url) + .unwrap() + .username(&role); + + let outcome = async { + let pool = connect( + options.clone(), + PgPoolOptions::new().max_connections(2), + source(&ca, &login, &token_file), + Duration::from_millis(50), + ) + .await + .unwrap(); + let mut held = pool.acquire().await.unwrap(); + + sqlx::raw_sql(sqlx::AssertSqlSafe(alter_password("second-password"))) + .execute(&admin) + .await + .unwrap(); + *current.lock().unwrap() = "second-password".to_owned(); + let exchanges = login.requests().len(); + while login.requests().len() < exchanges + 2 { + tokio::time::sleep(Duration::from_millis(20)).await; + } + + // The old password really is dead, so the pool can only be using the new one. + assert!( + ::connect_with( + &options.clone().password("first-password") + ) + .await + .is_err() + ); + let mut opened = pool.acquire().await.unwrap(); + let user: String = sqlx::query_scalar("SELECT current_user::text") + .fetch_one(&mut *opened) + .await + .unwrap(); + assert_eq!(user, role); + let survived: i32 = sqlx::query_scalar("SELECT 1") + .fetch_one(&mut *held) + .await + .unwrap(); + assert_eq!(survived, 1); + + drop((held, opened)); + pool.close().await; + }; + let outcome = + futures_util::FutureExt::catch_unwind(std::panic::AssertUnwindSafe(outcome)).await; + sqlx::raw_sql(sqlx::AssertSqlSafe(format!("DROP ROLE {role}"))) + .execute(&admin) + .await + .unwrap(); + admin.close().await; + if let Err(panic) = outcome { + std::panic::resume_unwind(panic); + } +} diff --git a/crates/store-pg/src/azure/test_support.rs b/crates/store-pg/src/azure/test_support.rs new file mode 100644 index 00000000..a51a186f --- /dev/null +++ b/crates/store-pg/src/azure/test_support.rs @@ -0,0 +1,224 @@ +//! A local TLS fake of Entra and Azure Storage. Certificates are generated per test. +use std::{ + collections::HashMap, + sync::{Arc, Mutex}, +}; + +use object_store::{Certificate, ClientOptions}; +use rcgen::{BasicConstraints, CertificateParams, IsCa, Issuer, KeyPair}; +use tokio::{ + io::{AsyncReadExt as _, AsyncWriteExt as _}, + net::TcpListener, + task::JoinHandle, +}; +use tokio_rustls::{ + TlsAcceptor, + rustls::{ + ServerConfig, + crypto::aws_lc_rs, + pki_types::{CertificateDer, PrivateKeyDer}, + }, +}; + +pub(super) struct TestCa { + pub(super) certificate: Certificate, + issuer: Issuer<'static, KeyPair>, + der: CertificateDer<'static>, +} + +impl TestCa { + pub(super) fn new() -> Self { + let mut params = CertificateParams::new(Vec::new()).unwrap(); + params.is_ca = IsCa::Ca(BasicConstraints::Unconstrained); + let key = KeyPair::generate().unwrap(); + let certificate = params.self_signed(&key).unwrap(); + Self { + certificate: Certificate::from_der(certificate.der()).unwrap(), + der: certificate.der().clone(), + issuer: Issuer::new(params, key), + } + } + + pub(super) fn acceptor(&self, host: &str) -> TlsAcceptor { + let params = CertificateParams::new(vec![host.to_owned()]).unwrap(); + let key = KeyPair::generate().unwrap(); + let certificate = params.signed_by(&key, &self.issuer).unwrap(); + let config = ServerConfig::builder_with_provider(Arc::new(aws_lc_rs::default_provider())) + .with_safe_default_protocol_versions() + .unwrap() + .with_no_client_auth() + .with_single_cert( + vec![certificate.der().clone(), self.der.clone()], + PrivateKeyDer::Pkcs8(key.serialize_der().into()), + ) + .unwrap(); + TlsAcceptor::from(Arc::new(config)) + } + + /// Only this CA is trusted, so results do not depend on the machine's store. + pub(super) fn reqwest_client(&self) -> reqwest::Client { + reqwest::Client::builder() + .tls_certs_only([reqwest::Certificate::from_der(&self.der).unwrap()]) + .build() + .unwrap() + } + + pub(super) fn client_options(&self) -> ClientOptions { + ClientOptions::new() + .with_root_certificate(self.certificate.clone()) + .with_no_system_certificates(true) + } +} + +#[derive(Clone, Debug)] +pub(super) struct Request { + pub(super) method: String, + pub(super) path: String, + pub(super) headers: HashMap, + pub(super) body: Vec, +} + +impl Request { + pub(super) fn form(&self) -> HashMap { + url::form_urlencoded::parse(&self.body) + .into_owned() + .collect() + } +} + +pub(super) struct Response { + pub(super) status: u16, + pub(super) headers: Vec<(&'static str, String)>, + pub(super) body: Vec, +} + +impl Response { + pub(super) fn new(status: u16, body: impl Into>) -> Self { + Self { + status, + headers: Vec::new(), + body: body.into(), + } + } +} + +pub(super) type Handler = dyn Fn(&Request) -> Response + Send + Sync; + +/// One request per connection. A rejected TLS handshake records nothing, which +/// is how the rejection tests prove that no credential reached the server. +pub(super) struct Server { + port: u16, + requests: Arc>>, + task: JoinHandle<()>, +} + +impl Drop for Server { + fn drop(&mut self) { + self.task.abort(); + } +} + +impl Server { + pub(super) async fn start(acceptor: TlsAcceptor, handler: Arc) -> Self { + let listener = TcpListener::bind(("127.0.0.1", 0)).await.unwrap(); + let port = listener.local_addr().unwrap().port(); + let requests = Arc::new(Mutex::new(Vec::new())); + let recorded = requests.clone(); + let task = tokio::spawn(async move { + loop { + let (stream, _) = listener.accept().await.unwrap(); + let (acceptor, handler, recorded) = + (acceptor.clone(), handler.clone(), recorded.clone()); + tokio::spawn(async move { + let Ok(mut stream) = acceptor.accept(stream).await else { + return; + }; + let Some(request) = read_request(&mut stream).await else { + return; + }; + let response = handler(&request); + recorded.lock().unwrap().push(request); + let mut head = format!( + "HTTP/1.1 {} X\r\nConnection: close\r\nContent-Length: {}\r\n", + response.status, + response.body.len() + ); + for (name, value) in response.headers { + head.push_str(&format!("{name}: {value}\r\n")); + } + head.push_str("\r\n"); + let _ = stream.write_all(head.as_bytes()).await; + let _ = stream.write_all(&response.body).await; + let _ = stream.shutdown().await; + }); + } + }); + Self { + port, + requests, + task, + } + } + + pub(super) fn url(&self) -> String { + format!("https://localhost:{}", self.port) + } + + pub(super) fn requests(&self) -> Vec { + self.requests.lock().unwrap().clone() + } +} + +async fn read_request(stream: &mut (impl tokio::io::AsyncRead + Unpin)) -> Option { + let mut buffer = Vec::new(); + let head_end = loop { + if let Some(end) = buffer.windows(4).position(|w| w == b"\r\n\r\n") { + break end; + } + let mut chunk = [0; 4096]; + match stream.read(&mut chunk).await.ok()? { + 0 => return None, + read => buffer.extend_from_slice(&chunk[..read]), + } + }; + let head = String::from_utf8(buffer[..head_end].to_vec()).ok()?; + let mut lines = head.split("\r\n"); + let mut request_line = lines.next()?.split(' '); + let (method, path) = (request_line.next()?, request_line.next()?); + let headers: HashMap = lines + .filter_map(|line| line.split_once(':')) + .map(|(name, value)| (name.to_ascii_lowercase(), value.trim().to_owned())) + .collect(); + let length: usize = headers + .get("content-length") + .map_or(Some(0), |value| value.parse().ok())?; + let mut body = buffer.split_off(head_end + 4); + while body.len() < length { + let mut chunk = [0; 4096]; + match stream.read(&mut chunk).await.ok()? { + 0 => return None, + read => body.extend_from_slice(&chunk[..read]), + } + } + Some(Request { + method: method.into(), + path: path.into(), + headers, + body, + }) +} + +/// Issues `secret-access-token-` valid for `expires_in` seconds. +pub(super) fn entra(expires_in: u64) -> Arc { + let issued = Mutex::new(0); + Arc::new(move |_| { + let mut issued = issued.lock().unwrap(); + *issued += 1; + Response::new( + 200, + format!( + r#"{{"access_token":"secret-access-token-{issued}","token_type":"Bearer","expires_in":{expires_in}}}"# + ), + ) + }) +} diff --git a/crates/store-pg/src/azure/workload_identity.rs b/crates/store-pg/src/azure/workload_identity.rs index dceebe35..ce900ba8 100644 --- a/crates/store-pg/src/azure/workload_identity.rs +++ b/crates/store-pg/src/azure/workload_identity.rs @@ -76,7 +76,9 @@ impl AzureWorkloadIdentity { /// The library joins the authority and tenant with a slash, so a configured /// trailing slash must not reach it. - fn normalized_authority_host(&self) -> Result, AzureWorkloadIdentityError> { + pub(super) fn normalized_authority_host( + &self, + ) -> Result, AzureWorkloadIdentityError> { let Some(authority) = &self.authority_host else { return Ok(None); }; @@ -132,7 +134,7 @@ fn generic(error: impl StdError + Send + Sync + 'static) -> object_store::Error /// The stage at which an access token could not be obtained. Holds rendered text /// rather than the library's error so that nothing else is reachable as a source. #[derive(Debug, Error)] -pub(super) enum TokenAcquisitionError { +pub enum TokenAcquisitionError { #[error( "could not read the projected token file '{path}' named by AZURE_FEDERATED_TOKEN_FILE{}", reason.map(|kind| format!(": {kind}")).unwrap_or_default() @@ -195,7 +197,7 @@ fn classify_token_error(error: &object_store::Error, token_file: &str) -> TokenA /// Keep transport causes such as certificate rejections. Response bodies in this /// text have already passed through [`ScrubbingConnector`]. -fn render_chain(error: &(dyn StdError + 'static)) -> String { +pub(super) fn render_chain(error: &(dyn StdError + 'static)) -> String { let mut rendered = error.to_string(); let mut current = error.source(); while let Some(source) = current { @@ -240,7 +242,7 @@ impl HttpService for ScrubbingService { /// Reduce an Entra or Azure Storage error to fields that identify the failure /// and locate it in the service's logs. Free text is dropped because it can /// repeat request content; Entra's `AADSTS` code identifies the same failure. -fn diagnostic_summary(headers: &HeaderMap, body: &[u8]) -> String { +pub(super) fn diagnostic_summary(headers: &HeaderMap, body: &[u8]) -> String { let mut fields = Vec::new(); let mut push = |name: &str, value: Option<&str>| { if let Some(value) = value.filter(is_diagnostic_value) { diff --git a/crates/store-pg/src/azure/workload_identity/tests.rs b/crates/store-pg/src/azure/workload_identity/tests.rs index f71b0950..3bb25244 100644 --- a/crates/store-pg/src/azure/workload_identity/tests.rs +++ b/crates/store-pg/src/azure/workload_identity/tests.rs @@ -9,226 +9,23 @@ use std::{ time::Duration, }; -use object_store::{Certificate, ClientOptions, ObjectStore, ObjectStoreExt as _, RetryConfig}; -use rcgen::{BasicConstraints, CertificateParams, IsCa, Issuer, KeyPair}; -use tokio::{ - io::{AsyncReadExt as _, AsyncWriteExt as _}, - net::TcpListener, - task::JoinHandle, -}; -use tokio_rustls::{ - TlsAcceptor, - rustls::{ - ServerConfig, - crypto::aws_lc_rs, - pki_types::{CertificateDer, PrivateKeyDer}, +use object_store::{ClientOptions, ObjectStore, ObjectStoreExt as _, RetryConfig}; +use tokio::net::TcpListener; + +use super::{ + super::{ + test_support::{Handler, Response, Server, TestCa, entra}, + *, }, + *, }; -use super::{super::*, *}; - const CLIENT_ID: &str = "client-id"; const TENANT_ID: &str = "tenant-id"; const ASSERTION: &str = "secret-assertion.jwt.value"; const ROTATED_ASSERTION: &str = "secret-rotated-assertion.jwt.value"; const LAST_MODIFIED: &str = "Wed, 21 Oct 2015 07:28:00 GMT"; -struct TestCa { - certificate: Certificate, - issuer: Issuer<'static, KeyPair>, - der: CertificateDer<'static>, -} - -impl TestCa { - fn new() -> Self { - let mut params = CertificateParams::new(Vec::new()).unwrap(); - params.is_ca = IsCa::Ca(BasicConstraints::Unconstrained); - let key = KeyPair::generate().unwrap(); - let certificate = params.self_signed(&key).unwrap(); - Self { - certificate: Certificate::from_der(certificate.der()).unwrap(), - der: certificate.der().clone(), - issuer: Issuer::new(params, key), - } - } - - fn acceptor(&self, host: &str) -> TlsAcceptor { - let params = CertificateParams::new(vec![host.to_owned()]).unwrap(); - let key = KeyPair::generate().unwrap(); - let certificate = params.signed_by(&key, &self.issuer).unwrap(); - let config = ServerConfig::builder_with_provider(Arc::new(aws_lc_rs::default_provider())) - .with_safe_default_protocol_versions() - .unwrap() - .with_no_client_auth() - .with_single_cert( - vec![certificate.der().clone(), self.der.clone()], - PrivateKeyDer::Pkcs8(key.serialize_der().into()), - ) - .unwrap(); - TlsAcceptor::from(Arc::new(config)) - } - - /// Only this CA is trusted, so results do not depend on the machine's store. - fn client_options(&self) -> ClientOptions { - ClientOptions::new() - .with_root_certificate(self.certificate.clone()) - .with_no_system_certificates(true) - } -} - -#[derive(Clone, Debug)] -struct Request { - method: String, - path: String, - headers: HashMap, - body: Vec, -} - -impl Request { - fn form(&self) -> HashMap { - url::form_urlencoded::parse(&self.body) - .into_owned() - .collect() - } -} - -struct Response { - status: u16, - headers: Vec<(&'static str, String)>, - body: Vec, -} - -impl Response { - fn new(status: u16, body: impl Into>) -> Self { - Self { - status, - headers: Vec::new(), - body: body.into(), - } - } -} - -type Handler = dyn Fn(&Request) -> Response + Send + Sync; - -/// One request per connection. A rejected TLS handshake records nothing, which -/// is how the rejection tests prove that no credential reached the server. -struct Server { - port: u16, - requests: Arc>>, - task: JoinHandle<()>, -} - -impl Drop for Server { - fn drop(&mut self) { - self.task.abort(); - } -} - -impl Server { - async fn start(acceptor: TlsAcceptor, handler: Arc) -> Self { - let listener = TcpListener::bind(("127.0.0.1", 0)).await.unwrap(); - let port = listener.local_addr().unwrap().port(); - let requests = Arc::new(Mutex::new(Vec::new())); - let recorded = requests.clone(); - let task = tokio::spawn(async move { - loop { - let (stream, _) = listener.accept().await.unwrap(); - let (acceptor, handler, recorded) = - (acceptor.clone(), handler.clone(), recorded.clone()); - tokio::spawn(async move { - let Ok(mut stream) = acceptor.accept(stream).await else { - return; - }; - let Some(request) = read_request(&mut stream).await else { - return; - }; - let response = handler(&request); - recorded.lock().unwrap().push(request); - let mut head = format!( - "HTTP/1.1 {} X\r\nConnection: close\r\nContent-Length: {}\r\n", - response.status, - response.body.len() - ); - for (name, value) in response.headers { - head.push_str(&format!("{name}: {value}\r\n")); - } - head.push_str("\r\n"); - let _ = stream.write_all(head.as_bytes()).await; - let _ = stream.write_all(&response.body).await; - let _ = stream.shutdown().await; - }); - } - }); - Self { - port, - requests, - task, - } - } - - fn url(&self) -> String { - format!("https://localhost:{}", self.port) - } - - fn requests(&self) -> Vec { - self.requests.lock().unwrap().clone() - } -} - -async fn read_request(stream: &mut (impl tokio::io::AsyncRead + Unpin)) -> Option { - let mut buffer = Vec::new(); - let head_end = loop { - if let Some(end) = buffer.windows(4).position(|w| w == b"\r\n\r\n") { - break end; - } - let mut chunk = [0; 4096]; - match stream.read(&mut chunk).await.ok()? { - 0 => return None, - read => buffer.extend_from_slice(&chunk[..read]), - } - }; - let head = String::from_utf8(buffer[..head_end].to_vec()).ok()?; - let mut lines = head.split("\r\n"); - let mut request_line = lines.next()?.split(' '); - let (method, path) = (request_line.next()?, request_line.next()?); - let headers: HashMap = lines - .filter_map(|line| line.split_once(':')) - .map(|(name, value)| (name.to_ascii_lowercase(), value.trim().to_owned())) - .collect(); - let length: usize = headers - .get("content-length") - .map_or(Some(0), |value| value.parse().ok())?; - let mut body = buffer.split_off(head_end + 4); - while body.len() < length { - let mut chunk = [0; 4096]; - match stream.read(&mut chunk).await.ok()? { - 0 => return None, - read => body.extend_from_slice(&chunk[..read]), - } - } - Some(Request { - method: method.into(), - path: path.into(), - headers, - body, - }) -} - -/// Issues `secret-access-token-` valid for `expires_in` seconds. -fn entra(expires_in: u64) -> Arc { - let issued = Mutex::new(0); - Arc::new(move |_| { - let mut issued = issued.lock().unwrap(); - *issued += 1; - Response::new( - 200, - format!( - r#"{{"access_token":"secret-access-token-{issued}","token_type":"Bearer","expires_in":{expires_in}}}"# - ), - ) - }) -} - fn storage() -> Arc { let blobs = Mutex::new(HashMap::>::new()); Arc::new(move |request| { diff --git a/crates/store-pg/src/lib.rs b/crates/store-pg/src/lib.rs index 0b1daec1..550bca36 100644 --- a/crates/store-pg/src/lib.rs +++ b/crates/store-pg/src/lib.rs @@ -40,8 +40,9 @@ use uuid::Uuid; pub use access::AccessFilter; pub use access::{AccessStoreError, PgAccessStore, ResourceAccess}; pub use azure::{ - AzureObjectStoreConfig, AzureObjectStoreCredential, AzureWorkloadIdentity, - AzureWorkloadIdentityError, build_azure_object_store, + AzureObjectStoreConfig, AzureObjectStoreCredential, AzurePostgresError, AzureWorkloadIdentity, + AzureWorkloadIdentityError, TokenAcquisitionError, build_azure_object_store, + connect_postgres_with_workload_identity, }; pub use model_defaults::ModelDefaultsStoreError; pub use s3::{S3ObjectStoreConfig, build_s3_object_store}; @@ -145,6 +146,16 @@ impl PgStoreConfig { } } +/// Pool settings for deployment processes. They limit how often connections open, +/// because each one costs a TLS handshake and server-side authentication. Connections +/// are still checked before use, so a longer lifetime does not hand out dead ones. +pub fn postgres_pool_options() -> PgPoolOptions { + PgPoolOptions::new() + // Stay warm through quiet periods instead of reconnecting for the next query. + .min_connections(2) + .max_lifetime(std::time::Duration::from_secs(60 * 60)) +} + #[derive(Clone)] pub struct PgStore { pub(crate) pool: PgPool, diff --git a/crates/store-pg/tests/azure_postgres_live.rs b/crates/store-pg/tests/azure_postgres_live.rs new file mode 100644 index 00000000..a6a27d5e --- /dev/null +++ b/crates/store-pg/tests/azure_postgres_live.rs @@ -0,0 +1,38 @@ +//! Workload identity against a real Azure Database for PostgreSQL server. Run it +//! from a workload holding a federated identity that exists as a database role. +//! Never reads .env. +use std::env; + +use store_pg::{AzureWorkloadIdentity, connect_postgres_with_workload_identity}; + +#[tokio::test(flavor = "current_thread")] +#[ignore = "requires LIGHTSPEED_TEST_AZURE_POSTGRES_URL (user is the identity name, sslmode=verify-full) and AZURE_*"] +async fn workload_identity_opens_a_verified_session_as_the_managed_identity() { + let url = env::var("LIGHTSPEED_TEST_AZURE_POSTGRES_URL") + .expect("explicit Azure PostgreSQL test URL required"); + let identity = AzureWorkloadIdentity { + client_id: env::var("AZURE_CLIENT_ID").expect("AZURE_CLIENT_ID required"), + tenant_id: env::var("AZURE_TENANT_ID").expect("AZURE_TENANT_ID required"), + federated_token_file: env::var("AZURE_FEDERATED_TOKEN_FILE") + .expect("AZURE_FEDERATED_TOKEN_FILE required"), + authority_host: env::var("AZURE_AUTHORITY_HOST").ok(), + }; + + let pool = connect_postgres_with_workload_identity(&url, identity) + .await + .unwrap(); + + let expected = url::Url::parse(&url).unwrap(); + let user: String = sqlx::query_scalar("SELECT current_user::text") + .fetch_one(&pool) + .await + .unwrap(); + assert_eq!(user, expected.username()); + let encrypted: bool = + sqlx::query_scalar("SELECT ssl FROM pg_stat_ssl WHERE pid = pg_backend_pid()") + .fetch_one(&pool) + .await + .unwrap(); + assert!(encrypted); + pool.close().await; +} diff --git a/crates/temporal-runtime/src/config.rs b/crates/temporal-runtime/src/config.rs index 3a4744ff..352b9ad2 100644 --- a/crates/temporal-runtime/src/config.rs +++ b/crates/temporal-runtime/src/config.rs @@ -1,12 +1,15 @@ use std::{env, sync::Arc, time::Duration}; use object_store::ObjectStore; -use sqlx::{PgPool, postgres::PgPoolOptions}; +use sqlx::PgPool; use store_pg::{BlobCache, PgStore, PgStoreConfig, PgStoreError, SchemaStatus, SecretsMasterKey}; use temporal_workflow::{DEFAULT_TASK_QUEUE, bots::DEFAULT_BOTS_TASK_QUEUE}; use uuid::Uuid; mod object_storage; +mod postgres; + +pub use postgres::postgres_pool_from_env; /// Retired deployment model policy must be migrated to universe defaults. /// Credentials and provider transport still use their existing configuration. @@ -366,17 +369,6 @@ fn evaluate_schema_verification( } } -/// Connect to the deployment database without inspecting or changing its -/// schema. Migration and diagnostic commands use this lower-level boundary. -pub async fn postgres_pool_from_env() -> anyhow::Result { - let database_url = env::var("LIGHTSPEED_POSTGRES_URL") - .or_else(|_| env::var("LIGHTSPEED_TEST_POSTGRES_URL")) - .map_err(|_| { - anyhow::anyhow!("LIGHTSPEED_POSTGRES_URL or LIGHTSPEED_TEST_POSTGRES_URL must be set") - })?; - Ok(PgPoolOptions::new().connect(&database_url).await?) -} - /// Single-universe store bound to `LIGHTSPEED_PG_UNIVERSE_ID`. Used by /// `single`-mode deployments, tests, and tools that operate on one universe. pub async fn pg_store_from_env() -> anyhow::Result> { diff --git a/crates/temporal-runtime/src/config/object_storage.rs b/crates/temporal-runtime/src/config/object_storage.rs index c76f294b..73cae609 100644 --- a/crates/temporal-runtime/src/config/object_storage.rs +++ b/crates/temporal-runtime/src/config/object_storage.rs @@ -112,6 +112,27 @@ fn parse(get: impl Fn(&str) -> Option) -> anyhow::Result Option, + selector: &str, +) -> anyhow::Result { + let value = |key: &str| get(key).filter(|value| !value.is_empty()); + let required = |key: &str| { + value(key) + .ok_or_else(|| anyhow::anyhow!("{key} must be set when {selector}=workload_identity")) + }; + let identity = AzureWorkloadIdentity { + client_id: required(CLIENT_ID)?, + tenant_id: required(TENANT_ID)?, + federated_token_file: required(TOKEN_FILE)?, + authority_host: value(AUTHORITY_HOST), + }; + identity.validate()?; + Ok(identity) +} + fn azure_credential( get: &impl Fn(&str) -> Option, ) -> anyhow::Result { @@ -125,18 +146,7 @@ fn azure_credential( "{key} conflicts with {AUTH_MODE}=workload_identity" ); } - let required = |key: &str| { - value(key).ok_or_else(|| { - anyhow::anyhow!("{key} must be set when {AUTH_MODE}=workload_identity") - }) - }; - let identity = AzureWorkloadIdentity { - client_id: required(CLIENT_ID)?, - tenant_id: required(TENANT_ID)?, - federated_token_file: required(TOKEN_FILE)?, - authority_host: value(AUTHORITY_HOST), - }; - identity.validate()?; + let identity = workload_identity_settings(get, AUTH_MODE)?; Ok(AzureObjectStoreCredential::WorkloadIdentity(identity)) } Some("access_key") => match (access_key, sas_token) { diff --git a/crates/temporal-runtime/src/config/postgres.rs b/crates/temporal-runtime/src/config/postgres.rs new file mode 100644 index 00000000..6a25ab8e --- /dev/null +++ b/crates/temporal-runtime/src/config/postgres.rs @@ -0,0 +1,102 @@ +use std::env; + +use sqlx::PgPool; +use store_pg::{ + AzureWorkloadIdentity, connect_postgres_with_workload_identity, postgres_pool_options, +}; + +use super::object_storage::workload_identity_settings; + +const AUTH_MODE: &str = "LIGHTSPEED_POSTGRES_AUTH_MODE"; + +/// Connect to the deployment database without inspecting or changing its +/// schema. Migration and diagnostic commands use this lower-level boundary. +pub async fn postgres_pool_from_env() -> anyhow::Result { + let database_url = env::var("LIGHTSPEED_POSTGRES_URL") + .or_else(|_| env::var("LIGHTSPEED_TEST_POSTGRES_URL")) + .map_err(|_| { + anyhow::anyhow!("LIGHTSPEED_POSTGRES_URL or LIGHTSPEED_TEST_POSTGRES_URL must be set") + })?; + match workload_identity(|key| env::var(key).ok())? { + Some(identity) => { + Ok(connect_postgres_with_workload_identity(&database_url, identity).await?) + } + None => Ok(postgres_pool_options().connect(&database_url).await?), + } +} + +fn workload_identity( + get: impl Fn(&str) -> Option, +) -> anyhow::Result> { + match get(AUTH_MODE).filter(|mode| !mode.is_empty()).as_deref() { + None | Some("password") => Ok(None), + Some("workload_identity") => workload_identity_settings(&get, AUTH_MODE).map(Some), + Some(_) => anyhow::bail!("{AUTH_MODE} must be password or workload_identity"), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + const IDENTITY: [(&str, &str); 3] = [ + ("AZURE_CLIENT_ID", "client-id"), + ("AZURE_TENANT_ID", "tenant-id"), + ("AZURE_FEDERATED_TOKEN_FILE", "/var/run/token"), + ]; + + fn selected(entries: &[(&str, &str)]) -> anyhow::Result> { + workload_identity(|key| { + entries + .iter() + .find(|(name, _)| *name == key) + .map(|(_, value)| (*value).to_owned()) + }) + } + + #[test] + fn password_stays_the_default_and_azure_variables_do_not_change_it() { + assert!(selected(&[]).unwrap().is_none()); + assert!(selected(&IDENTITY).unwrap().is_none()); + assert!(selected(&[(AUTH_MODE, "")]).unwrap().is_none()); + let mut explicit = IDENTITY.to_vec(); + explicit.push((AUTH_MODE, "password")); + assert!(selected(&explicit).unwrap().is_none()); + } + + #[test] + fn workload_identity_reads_the_standard_settings() { + let mut entries = IDENTITY.to_vec(); + entries.push((AUTH_MODE, "workload_identity")); + let identity = selected(&entries).unwrap().unwrap(); + assert_eq!(identity.client_id, "client-id"); + assert_eq!(identity.tenant_id, "tenant-id"); + assert_eq!(identity.federated_token_file, "/var/run/token"); + } + + #[test] + fn workload_identity_requires_each_setting() { + for (key, _) in IDENTITY { + let mut entries = IDENTITY.to_vec(); + entries.retain(|(name, _)| *name != key); + entries.push((AUTH_MODE, "workload_identity")); + let error = selected(&entries).unwrap_err().to_string(); + assert!(error.contains(key), "{error}"); + assert!(error.contains(AUTH_MODE), "{error}"); + } + } + + #[test] + fn invalid_settings_and_unknown_modes_are_rejected_without_echoing_values() { + let mut entries = IDENTITY.to_vec(); + entries.push((AUTH_MODE, "workload_identity")); + entries.push(("AZURE_AUTHORITY_HOST", "http://login.example")); + assert!(selected(&entries).is_err()); + + let error = selected(&[(AUTH_MODE, "secret-mode")]) + .unwrap_err() + .to_string(); + assert!(error.contains(AUTH_MODE), "{error}"); + assert!(!error.contains("secret-mode")); + } +} diff --git a/docs/documentation/deployment/configuration.md b/docs/documentation/deployment/configuration.md index c7ba2e37..829f8d92 100644 --- a/docs/documentation/deployment/configuration.md +++ b/docs/documentation/deployment/configuration.md @@ -236,6 +236,47 @@ and it never falls back to another credential. Existing objects, keys, and prefixes stay valid when an account moves to this mode; no data migration is needed. +### PostgreSQL workload identity + +The Runtime and Platform can log in to Azure Database for PostgreSQL with a +Microsoft Entra token instead of a stored password. Each selects this separately, +and the URL names the managed identity as its user and holds no password: + +```dotenv +LIGHTSPEED_POSTGRES_AUTH_MODE=workload_identity +LIGHTSPEED_POSTGRES_URL=postgres://@.postgres.database.azure.com:5432/lightspeed?sslmode=verify-full + +LIGHTSPEED_PLATFORM_DATABASE_AUTH_MODE=workload_identity +LIGHTSPEED_PLATFORM_DATABASE_URL=postgres://@.postgres.database.azure.com:5432/platform?sslmode=verify-full +``` + +The user is the identity's name, not its client ID. Both processes also need +`AZURE_CLIENT_ID`, `AZURE_TENANT_ID`, and `AZURE_FEDERATED_TOKEN_FILE`, which +Blob storage shares in the Runtime. Set `AZURE_AUTHORITY_HOST` only for a cloud +other than the public one. `sslmode=verify-full` is required because the token is +sent as a password, so the server's identity must be verified first. + +On the database side, an Entra administrator creates a role for each identity and +gives it the database: + +```sql +SELECT * FROM pgaadauth_create_principal('', false, false); +GRANT CONNECT ON DATABASE lightspeed TO ""; +ALTER DATABASE lightspeed OWNER TO ""; +``` + +In an existing database, also reassign the objects the previous role owns with +`REASSIGN OWNED`. Run the Runtime's migration job with the same identity as the +Runtime; the Platform migrates at startup. + +Only opening a connection needs a current token, and open connections are not +affected when one expires. The Runtime renews its token at half of its lifetime, +and the Platform lets the Azure SDK renew its token as it nears expiry. Both +depend on Microsoft Entra to open a connection at startup. During a later outage +they can still open connections until the current token expires, and they keep +using the ones already open. This mode applies only to the Runtime and Platform; +Temporal is configured separately. + ## Add optional services ### Execution environments diff --git a/docs/documentation/deployment/troubleshooting.md b/docs/documentation/deployment/troubleshooting.md index ef0e7ada..7737150d 100644 --- a/docs/documentation/deployment/troubleshooting.md +++ b/docs/documentation/deployment/troubleshooting.md @@ -43,7 +43,7 @@ If the runtime exits before serving, check the error against these dependencies: | Error area | What to inspect | | --- | --- | -| PostgreSQL connection | The runtime database URL, credentials, network path, and required connection options. Container-local `localhost` is not the database host. | +| PostgreSQL connection | The runtime database URL, credentials, network path, and required connection options. Container-local `localhost` is not the database host. With workload identity, the error names the stage: the token file, the Entra exchange with its `AADSTS` code, or the login, where the URL user must be the identity's name and that role must exist. | | Schema verification | Use the deployed release's `schema-version` diagnostic. The runtime never migrates implicitly; follow the explicit migration procedure. | | Temporal connection | The frontend address and an existing namespace, reachable from the process. | | Object-store configuration | S3 requires a bucket; `LIGHTSPEED_OBJECT_STORE_BACKEND=azure` requires account, container, and exactly one access key or SAS token, or `LIGHTSPEED_OBJECT_STORE_AZURE_AUTH_MODE=workload_identity` with `AZURE_CLIENT_ID`, `AZURE_TENANT_ID`, and an absolute `AZURE_FEDERATED_TOKEN_FILE`. Check for settings mixed between backends and use the intended endpoint. | diff --git a/docs/documentation/reference/environment-variables.md b/docs/documentation/reference/environment-variables.md index e7a8f915..10bf2374 100644 --- a/docs/documentation/reference/environment-variables.md +++ b/docs/documentation/reference/environment-variables.md @@ -39,6 +39,7 @@ They do not configure the TypeScript Platform backend. | Variable | Requirement/default | Purpose | | --- | --- | --- | | `LIGHTSPEED_POSTGRES_URL` | **Required**; falls back to `LIGHTSPEED_TEST_POSTGRES_URL` | PostgreSQL connection URL used by the runtime, migration commands, and schema diagnostics. Production must use this name. TLS is supported; use `sslmode=verify-full` for remote databases to verify the certificate and hostname. | +| `LIGHTSPEED_POSTGRES_AUTH_MODE` | `password` | `password` or `workload_identity`. With `workload_identity`, the Runtime logs in with a Microsoft Entra token using the [`AZURE_*` settings](#object-storage). The URL must name the identity as its user, hold no password, and set `sslmode=verify-full`. | | `LIGHTSPEED_PG_UNIVERSE_ID` | **Required in `single` auth mode** | UUID of the sole universe in a single-tenant deployment. Not used to auto-create universes in multi-tenant modes. | | `LIGHTSPEED_ROLES` | `gateway,environment-gateway,sessions,bots,channels` | Roles this process runs, a comma-separated subset of `gateway` (JSON-RPC, OAuth callbacks, webhook hooks), `environment-gateway` (worker environment routes, the public daemon registration routes, the lifecycle reconciler that creates and closes provisioned environments, and the registered-environment sweep), `sessions` (session workflows plus the promise-repair reaper, the session-retention reaper, both every five minutes, and the hourly CAS blob sweeper), `bots`, `channels` (also `--roles`). Each worker role polls its own task queue with that subsystem's workflows, activities, and background loops. Run exactly one `environment-gateway` process per deployment. | | `LIGHTSPEED_TASK_QUEUE` | `lightspeed-sessions` | Sessions task queue shared by the gateway and the `sessions` role. Deployments sharing a Temporal namespace must use distinct queues. | @@ -133,7 +134,7 @@ Existing configurations default to S3; Azure must be selected explicitly. | `LIGHTSPEED_OBJECT_STORE_AZURE_AUTH_MODE` | Inferred from the static credential | `access_key`, `sas_token`, or `workload_identity`. Unset selects whichever single static credential is present. | | `LIGHTSPEED_OBJECT_STORE_AZURE_ACCESS_KEY` | Required for `access_key` | Storage account access key. Supply through a Secret. Rejected with `workload_identity`. | | `LIGHTSPEED_OBJECT_STORE_AZURE_SAS_TOKEN` | Required for `sas_token` | SAS token, with or without a leading `?`. Supply through a Secret; permit the read/write/list/delete operations the blob store uses. Rejected with `workload_identity`. | -| `AZURE_CLIENT_ID` | Required for `workload_identity` | Client ID of the federated Microsoft Entra identity. | +| `AZURE_CLIENT_ID` | Required for `workload_identity` | Client ID of the federated Microsoft Entra identity. Blob storage and PostgreSQL in one process share these `AZURE_*` settings. | | `AZURE_TENANT_ID` | Required for `workload_identity` | Tenant ID or domain name. | | `AZURE_FEDERATED_TOKEN_FILE` | Required for `workload_identity` | Absolute path of the projected service account token. Read again for each token exchange, so rotation needs no restart. | | `AZURE_AUTHORITY_HOST` | `https://login.microsoftonline.com` | Microsoft Entra authority for `workload_identity`. Must be HTTPS with no path, query, or credentials; a trailing slash is optional. | @@ -141,8 +142,8 @@ Existing configurations default to S3; Azure must be selected explicitly. Azure uses only the selected credential and never falls back to another one, such as the Azure CLI, a node or VM identity, or a client secret. S3-only bucket, region, and path-style settings are rejected with Azure. The `AZURE_*` variables -are read only with `workload_identity`; they do not activate object storage or -change the backend. That mode requires an HTTPS endpoint, even for loopback +are read only where a setting selects `workload_identity`; they do not activate +object storage, change the backend, or change how a process logs in to PostgreSQL. That mode requires an HTTPS endpoint, even for loopback addresses. Both backends use normal HTTPS certificate trust by default; no certificate file is required for a server trusted by the container's CA roots. Keep the backend, account/container, and existing objects consistent across @@ -320,6 +321,7 @@ acceptance checks, follow [Company sign-in (SSO)](../access-and-security/single- | --- | --- | --- | | `LIGHTSPEED_PLATFORM_API_KEY` | Unset | The Platform's deployment key: every method group and allowed to assert actors. Create with `lightspeed-runtime api-key create --deployment --assert-actor`. Interactive calls name the universe and assert the signed-in user as the actor after Platform permission checks. Full development provisions a key when omitted. | | `LIGHTSPEED_PLATFORM_DATABASE_URL` | **Required** | Platform PostgreSQL connection URL. | +| `LIGHTSPEED_PLATFORM_DATABASE_AUTH_MODE` | `password` | `password` or `workload_identity`. With `workload_identity`, the Platform logs in with a Microsoft Entra token using the [`AZURE_*` settings](#object-storage). The URL must name the identity as its user, hold no password, and set `sslmode=verify-full`. | | `LIGHTSPEED_PLATFORM_AUTH_SECRET` | **Required** | Better Auth signing/encryption secret. Use a strong, stable deployment secret. | | `LIGHTSPEED_PLATFORM_BASE_URL` | `http://localhost:3000` | Public Platform origin used by authentication and trusted-origin checks. | | `LIGHTSPEED_PLATFORM_TRUSTED_ORIGINS` | Empty list | Comma-separated additional browser origins accepted by Better Auth. The development supervisor supplies both `http://127.0.0.1:5173` and `http://localhost:5173`. | diff --git a/docs/roadmap/azure-storage-and-postgres-tls.md b/docs/roadmap/azure-storage-and-postgres-tls.md index ca0e0c42..dd465790 100644 --- a/docs/roadmap/azure-storage-and-postgres-tls.md +++ b/docs/roadmap/azure-storage-and-postgres-tls.md @@ -88,3 +88,49 @@ that production uses with its default options. `LIGHTSPEED_TEST_AZURE_AUTH_MODE=workload_identity` and the `AZURE_*` settings, and refuses to start if a test access key or SAS token is set. Run it from a workload with a federated identity, against an account with Shared Key access disabled. + +## PostgreSQL workload identity + +**Status:** Implemented for the Runtime and Platform, 2026-11-05. Not yet run against a +real Azure server. Temporal stays on password authentication and is planned separately. + +`LIGHTSPEED_POSTGRES_AUTH_MODE` and `LIGHTSPEED_PLATFORM_DATABASE_AUTH_MODE` select an +Entra token as the login password, for the scope +`https://ossrdbms-aad.database.windows.net/.default`. The URL must name the identity, hold +no password, and set `sslmode=verify-full`, because the token is sent as a cleartext +password inside TLS. + +- **Runtime:** sqlx has no per-connection credential callback, and the Blob provider's + scope is fixed, so `store-pg` performs the exchange. A background task re-reads the + projected token and updates the pool's connect options at half of each token's lifetime. + It logs and retries every 30 seconds on failure. +- **Platform:** `pg` accepts a password function, called for each new connection, and + `WorkloadIdentityCredential` caches and renews the token. `pg` discards that function when + combined with a `connectionString`, so the URL is expanded into fields. The SDK reuses the + token file's contents for five minutes, far less than the rotation interval. Failures are + rebuilt as new errors holding only identifiers. + +Pool settings apply in both authentication modes, so each new connection is less common: + +| | Runtime (`sqlx`) | Platform (`pg`) | +| --- | --- | --- | +| Connections kept when idle | 2, opened eagerly | up to 2, kept once opened | +| Idle timeout | 10 minutes (default) | 10 minutes, was 10 seconds | +| Maximum lifetime | 1 hour, was 30 minutes | 1 hour, was unlimited | +| Dead connection handling | checked before use (default) | TCP keepalive after 60 seconds, plus an `error` listener | + +The Platform's `error` listener is also a correction for both modes: without one, losing an +idle connection ended the process. + +Verification: + +- `cargo test -p store-pg --lib azure` covers the exchange against a local TLS fake: + wire content, rotation, certificate and hostname rejection, and redaction. +- An ignored test in the same module rotates a real role's password on the development + PostgreSQL, with the fake issuing it as the token. It shows that the old password is + refused, new connections use the renewed token, and an open connection keeps working. +- `npm run test --workspace @lightspeed-ai/platform-db` drives the real Azure credential + through a scripted transport. +- `cargo test -p store-pg --test azure_postgres_live -- --ignored` requires + `LIGHTSPEED_TEST_AZURE_POSTGRES_URL` and the `AZURE_*` settings, and must run from a workload + with a federated identity. diff --git a/package-lock.json b/package-lock.json index 40a5fb14..45c864a0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -669,6 +669,173 @@ "yaml": "^2.8.3" } }, + "node_modules/@azure/abort-controller": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/@azure/abort-controller/-/abort-controller-2.2.0.tgz", + "integrity": "sha512-fNAjWnA/nZ2jz31kxR/AqRaUT8ewHBw/WuBIosK0moMy1C9e5ValbDfFdIxJzVOOYaYkV/b2F1S4H/aHiqfVQg==", + "license": "MIT", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@azure/core-auth": { + "version": "1.11.0", + "resolved": "https://registry.npmjs.org/@azure/core-auth/-/core-auth-1.11.0.tgz", + "integrity": "sha512-IUZydyTUkDnYdstOW9pFOOUQlBjAepK5teihDE3x6yxsPJs/hsAaaYpeGxdxrgtOiJbBKSjKW7MDk7AEhb4LRg==", + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^2.1.2", + "@azure/core-util": "^1.13.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@azure/core-client": { + "version": "1.11.1", + "resolved": "https://registry.npmjs.org/@azure/core-client/-/core-client-1.11.1.tgz", + "integrity": "sha512-2QygG2F76ZpMP2eMztiJvAiFMu71M9rDeU7vO/QKg5Css7MgM4frUOslFjhVjRhbGaCNPtz/S8M6y46/fFKVuQ==", + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^2.1.2", + "@azure/core-auth": "^1.10.0", + "@azure/core-rest-pipeline": "^1.22.0", + "@azure/core-tracing": "^1.3.0", + "@azure/core-util": "^1.13.0", + "@azure/logger": "^1.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@azure/core-process": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/@azure/core-process/-/core-process-1.0.0.tgz", + "integrity": "sha512-/shnJ+ooO8WPxDhPEeI/2oRQuubn16gZ6CvlbpWbEswZfzwI9tI/sMAHmF3x1LuQ9yZYXfLW3TjzGMLEC5blKg==", + "license": "MIT", + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@azure/core-rest-pipeline": { + "version": "1.25.0", + "resolved": "https://registry.npmjs.org/@azure/core-rest-pipeline/-/core-rest-pipeline-1.25.0.tgz", + "integrity": "sha512-bMs8ekJLjX8wPV+9IPBges1SLPyuDtE9g5gLDWOpxzKcoOFQnpLGkbcT1tdw3FaAmDS1gnPmMmJ6y/T5B96kIA==", + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^2.1.2", + "@azure/core-auth": "^1.10.0", + "@azure/core-tracing": "^1.3.0", + "@azure/core-util": "^1.13.0", + "@azure/logger": "^1.3.0", + "@typespec/ts-http-runtime": "^0.3.4", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@azure/core-tracing": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/@azure/core-tracing/-/core-tracing-1.4.0.tgz", + "integrity": "sha512-eGwxD0AtncrxeBM4tG8R55Pc3rdX1hNW2WibJAgYpCVA6E93mvvVH+LcssoVjOBrSKWS55yEIHsk0X8ctHmfOQ==", + "license": "MIT", + "dependencies": { + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@azure/core-util": { + "version": "1.14.0", + "resolved": "https://registry.npmjs.org/@azure/core-util/-/core-util-1.14.0.tgz", + "integrity": "sha512-9n2pWK61veAuN0V20t9lOuoV4CFMdyAZ1ygZzvBGk/pBBJRib/PjL9PLXa/aI2CcPpyHfqVsxxqLCYl6uZlfDw==", + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^2.1.2", + "@typespec/ts-http-runtime": "^0.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@azure/identity": { + "version": "4.14.1", + "resolved": "https://registry.npmjs.org/@azure/identity/-/identity-4.14.1.tgz", + "integrity": "sha512-HHuuFrmN1uuF9D5A6aAo5eHzNFNX4Q+P4SksZqDKse5YaJFFy/zDLG3wXHEV2FVGgetZ3QM8jGURNBtnQYf1NQ==", + "license": "MIT", + "dependencies": { + "@azure/abort-controller": "^2.0.0", + "@azure/core-auth": "^1.9.0", + "@azure/core-client": "^1.9.2", + "@azure/core-process": "^1.0.0", + "@azure/core-rest-pipeline": "^1.17.0", + "@azure/core-tracing": "^1.0.0", + "@azure/core-util": "^1.11.0", + "@azure/logger": "^1.0.0", + "@azure/msal-browser": "^5.5.0", + "@azure/msal-node": "^7.1.0", + "open": "^11.0.0", + "tslib": "^2.2.0" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@azure/logger": { + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/@azure/logger/-/logger-1.4.0.tgz", + "integrity": "sha512-rbAE25KUfjU/s3XHUdJgceoCP5dEOpMx85J04kF+QMdta73XkuG9JGHHinch+XIoKpBdqljin+KqURpJriSzLA==", + "license": "MIT", + "dependencies": { + "@typespec/ts-http-runtime": "^0.3.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=22.0.0" + } + }, + "node_modules/@azure/msal-browser": { + "version": "5.28.0", + "resolved": "https://registry.npmjs.org/@azure/msal-browser/-/msal-browser-5.28.0.tgz", + "integrity": "sha512-f3yY518Z++x3ZogCbcLK6vnABgpxc8rAKGKtQibcUJKAVU9gamgh5mNAPJIRwduzXQvO2xtHx/gtmPz8iuZ8+w==", + "license": "MIT", + "dependencies": { + "@azure/msal-common": "16.16.0" + }, + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/@azure/msal-common": { + "version": "16.16.0", + "resolved": "https://registry.npmjs.org/@azure/msal-common/-/msal-common-16.16.0.tgz", + "integrity": "sha512-EZgd8h5nczsdRAQAgXA0A0pC5L/zj43n1jePXvUMgdwXMbH2zhb7b7Hp+Oe7ZOODK/sch2x+soD6FOrtgUbx/w==", + "license": "MIT", + "engines": { + "node": ">=0.8.0" + } + }, + "node_modules/@azure/msal-node": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/@azure/msal-node/-/msal-node-7.2.0.tgz", + "integrity": "sha512-Y87VJsMWF4jiqLqq8XYLb1o+rGc/HacwS3Zydi3bDtaVm5Sa8FlB7YrKrBFKRIYj27RJXAaDkErImMAqtM10Pg==", + "license": "MIT", + "dependencies": { + "@azure/msal-common": "16.16.0", + "jsonwebtoken": "^9.0.0" + }, + "engines": { + "node": ">=20" + } + }, "node_modules/@babel/code-frame": { "version": "7.29.7", "resolved": "https://registry.npmjs.org/@babel/code-frame/-/code-frame-7.29.7.tgz", @@ -6383,6 +6550,20 @@ "integrity": "sha512-lrpDziQipxCEeK5kWxvljWYhUvOiB2A9izZd9B2AFarYAkqZshb4lPbRs7zKEic6eGtH8V/2qJW+dPp9OtF6bw==", "license": "MIT" }, + "node_modules/@typespec/ts-http-runtime": { + "version": "0.3.9", + "resolved": "https://registry.npmjs.org/@typespec/ts-http-runtime/-/ts-http-runtime-0.3.9.tgz", + "integrity": "sha512-edSdeAqkdxBVzA1yL1LrLCml1YjyCVvPMtMqJpbF+6K609tHe8V6sQUzFQSGcYNhcuhOceZtjvN32+mpIth30A==", + "license": "MIT", + "dependencies": { + "http-proxy-agent": "^7.0.0", + "https-proxy-agent": "^7.0.0", + "tslib": "^2.6.2" + }, + "engines": { + "node": ">=22.0.0" + } + }, "node_modules/@ungap/structured-clone": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/@ungap/structured-clone/-/structured-clone-1.4.0.tgz", @@ -6776,6 +6957,15 @@ "acorn": "^6.0.0 || ^7.0.0 || ^8.0.0" } }, + "node_modules/agent-base": { + "version": "7.1.4", + "resolved": "https://registry.npmjs.org/agent-base/-/agent-base-7.1.4.tgz", + "integrity": "sha512-MnA+YT8fwfJPgBx3m60MNqakm30XOkyIoH1y6huTQvC0PwZG7ki8NacLBcrPbNoo8vEZy7Jpuk7+jMO+CUovTQ==", + "license": "MIT", + "engines": { + "node": ">= 14" + } + }, "node_modules/ajv": { "version": "8.20.0", "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.20.0.tgz", @@ -7486,6 +7676,12 @@ "node": "^6 || ^7 || ^8 || ^9 || ^10 || ^11 || ^12 || >=13.7" } }, + "node_modules/buffer-equal-constant-time": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/buffer-equal-constant-time/-/buffer-equal-constant-time-1.0.1.tgz", + "integrity": "sha512-zRpUiDwd/xk6ADqPMATG8vc9VPrkck7T07OIx0gnjmJAnHnTVXNQG3vfvWNuiZIkwu9KrKdA1iJKfsfTVxE6NA==", + "license": "BSD-3-Clause" + }, "node_modules/buffer-from": { "version": "1.1.2", "resolved": "https://registry.npmjs.org/buffer-from/-/buffer-from-1.1.2.tgz", @@ -9653,6 +9849,15 @@ "node": ">= 0.4" } }, + "node_modules/ecdsa-sig-formatter": { + "version": "1.0.11", + "resolved": "https://registry.npmjs.org/ecdsa-sig-formatter/-/ecdsa-sig-formatter-1.0.11.tgz", + "integrity": "sha512-nagl3RYrbNv6kQkeJIpt6NJZy8twLB/2vtz6yN9Z4vRKHN4/QZJIEbqohALSgwKdnksuY3k5Addp5lg8sVoVcQ==", + "license": "Apache-2.0", + "dependencies": { + "safe-buffer": "^5.0.1" + } + }, "node_modules/ee-first": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz", @@ -11249,6 +11454,32 @@ "url": "https://opencollective.com/express" } }, + "node_modules/http-proxy-agent": { + "version": "7.0.2", + "resolved": "https://registry.npmjs.org/http-proxy-agent/-/http-proxy-agent-7.0.2.tgz", + "integrity": "sha512-T1gkAiYYDWYx3V5Bmyu7HcfcvL7mUrTWiM6yOfa3PIphViJ/gFPbvidQ+veqSOHci/PxBcDabeUNCzpOODJZig==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.0", + "debug": "^4.3.4" + }, + "engines": { + "node": ">= 14" + } + }, + "node_modules/https-proxy-agent": { + "version": "7.0.6", + "resolved": "https://registry.npmjs.org/https-proxy-agent/-/https-proxy-agent-7.0.6.tgz", + "integrity": "sha512-vK9P5/iUfdl95AI+JVyUuIcVtd4ofvtrOr3HNtM2yxC9bnMbEdp3x01OhQNnjb8IJYi38VlTE3mBXwcfvywuSw==", + "license": "MIT", + "dependencies": { + "agent-base": "^7.1.2", + "debug": "4" + }, + "engines": { + "node": ">= 14" + } + }, "node_modules/human-signals": { "version": "8.0.1", "resolved": "https://registry.npmjs.org/human-signals/-/human-signals-8.0.1.tgz", @@ -11886,6 +12117,67 @@ "graceful-fs": "^4.1.6" } }, + "node_modules/jsonwebtoken": { + "version": "9.0.3", + "resolved": "https://registry.npmjs.org/jsonwebtoken/-/jsonwebtoken-9.0.3.tgz", + "integrity": "sha512-MT/xP0CrubFRNLNKvxJ2BYfy53Zkm++5bX9dtuPbqAeQpTVe0MQTFhao8+Cp//EmJp244xt6Drw/GVEGCUj40g==", + "license": "MIT", + "dependencies": { + "jws": "^4.0.1", + "lodash.includes": "^4.3.0", + "lodash.isboolean": "^3.0.3", + "lodash.isinteger": "^4.0.4", + "lodash.isnumber": "^3.0.3", + "lodash.isplainobject": "^4.0.6", + "lodash.isstring": "^4.0.1", + "lodash.once": "^4.0.0", + "ms": "^2.1.1", + "semver": "^7.5.4" + }, + "engines": { + "node": ">=12", + "npm": ">=6" + } + }, + "node_modules/jsonwebtoken/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "license": "MIT" + }, + "node_modules/jsonwebtoken/node_modules/semver": { + "version": "7.8.6", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.8.6.tgz", + "integrity": "sha512-9lpSSU4Mly3mrtruwLs1eLK6+gzbzqeBFwRNjIjE+9LuDpWdYctrQlQ3ej1Ilgi7gknpsTRnPoLP8DpCHqHIJw==", + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/jwa": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/jwa/-/jwa-2.0.1.tgz", + "integrity": "sha512-hRF04fqJIP8Abbkq5NKGN0Bbr3JxlQ+qhZufXVr0DvujKy93ZCbXZMHDL4EOtodSbCWxOqR8MS1tXA5hwqCXDg==", + "license": "MIT", + "dependencies": { + "buffer-equal-constant-time": "^1.0.1", + "ecdsa-sig-formatter": "1.0.11", + "safe-buffer": "^5.0.1" + } + }, + "node_modules/jws": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/jws/-/jws-4.0.1.tgz", + "integrity": "sha512-EKI/M/yqPncGUUh44xz0PxSidXFr/+r0pA70+gIYhjv+et7yxM+s29Y+VGDkovRofQem0fs7Uvf4+YmAdyRduA==", + "license": "MIT", + "dependencies": { + "jwa": "^2.0.1", + "safe-buffer": "^5.0.1" + } + }, "node_modules/keyv": { "version": "5.6.0", "resolved": "https://registry.npmjs.org/keyv/-/keyv-5.6.0.tgz", @@ -12224,6 +12516,48 @@ "integrity": "sha512-TwuEnCnxbc3rAvhf/LbG7tJUDzhqXyFnv3dtzLOPgCG/hODL7WFnsbwktkD7yUV0RrreP/l1PALq/YSg6VvjlA==", "license": "MIT" }, + "node_modules/lodash.includes": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/lodash.includes/-/lodash.includes-4.3.0.tgz", + "integrity": "sha512-W3Bx6mdkRTGtlJISOvVD/lbqjTlPPUDTMnlXZFnVwi9NKJ6tiAk6LVdlhZMm17VZisqhKcgzpO5Wz91PCt5b0w==", + "license": "MIT" + }, + "node_modules/lodash.isboolean": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/lodash.isboolean/-/lodash.isboolean-3.0.3.tgz", + "integrity": "sha512-Bz5mupy2SVbPHURB98VAcw+aHh4vRV5IPNhILUCsOzRmsTmSQ17jIuqopAentWoehktxGd9e/hbIXq980/1QJg==", + "license": "MIT" + }, + "node_modules/lodash.isinteger": { + "version": "4.0.4", + "resolved": "https://registry.npmjs.org/lodash.isinteger/-/lodash.isinteger-4.0.4.tgz", + "integrity": "sha512-DBwtEWN2caHQ9/imiNeEA5ys1JoRtRfY3d7V9wkqtbycnAmTvRRmbHKDV4a0EYc678/dia0jrte4tjYwVBaZUA==", + "license": "MIT" + }, + "node_modules/lodash.isnumber": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/lodash.isnumber/-/lodash.isnumber-3.0.3.tgz", + "integrity": "sha512-QYqzpfwO3/CWf3XP+Z+tkQsfaLL/EnUlXWVkIk5FUPc4sBdTehEqZONuyRt2P67PXAk+NXmTBcc97zw9t1FQrw==", + "license": "MIT" + }, + "node_modules/lodash.isplainobject": { + "version": "4.0.6", + "resolved": "https://registry.npmjs.org/lodash.isplainobject/-/lodash.isplainobject-4.0.6.tgz", + "integrity": "sha512-oSXzaWypCMHkPC3NvBEaPHf0KsA5mvPrOPgQWDsbg8n7orZ290M0BmC/jgRZ4vcJ6DTAhjrsSYgdsW/F+MFOBA==", + "license": "MIT" + }, + "node_modules/lodash.isstring": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/lodash.isstring/-/lodash.isstring-4.0.1.tgz", + "integrity": "sha512-0wJxfxH1wgO3GrbuP+dTTk7op+6L41QCXbGINEmD+ny/G/eCqGzxyCsh7159S+mgDDcoarnBw6PC1PS5+wUGgw==", + "license": "MIT" + }, + "node_modules/lodash.once": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/lodash.once/-/lodash.once-4.1.1.tgz", + "integrity": "sha512-Sb487aTOCr9drQVL8pIxOzVhafOjZN9UU54hiN8PU3uAiSV7lx1yYNpbNmex2PK6dSJoNTSJUUswT651yww3Mg==", + "license": "MIT" + }, "node_modules/log-symbols": { "version": "6.0.0", "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-6.0.0.tgz", @@ -15727,6 +16061,26 @@ "integrity": "sha512-ASEq9atUw7lualXB+knvgtvwkCEvGWV2gDD/8qnASzBkzEARZck9JAyxmY8OS6Nc1pCPEgDTKNcx+YqqYfzArw==", "license": "MIT" }, + "node_modules/safe-buffer": { + "version": "5.2.1", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", + "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/feross" + }, + { + "type": "patreon", + "url": "https://www.patreon.com/feross" + }, + { + "type": "consulting", + "url": "https://feross.org/support" + } + ], + "license": "MIT" + }, "node_modules/safe-stable-stringify": { "version": "2.5.0", "resolved": "https://registry.npmjs.org/safe-stable-stringify/-/safe-stable-stringify-2.5.0.tgz", @@ -18904,13 +19258,17 @@ "platform/db": { "name": "@lightspeed-ai/platform-db", "dependencies": { + "@azure/identity": "^4.14.1", "drizzle-orm": "^0.45.3", - "pg": "^8.23.1" + "pg": "^8.23.1", + "pg-connection-string": "^2.14.1" }, "devDependencies": { + "@azure/core-rest-pipeline": "^1.25.0", "@types/pg": "^8.23.1", "better-auth": "1.7.7", - "drizzle-kit": "^0.31.11" + "drizzle-kit": "^0.31.11", + "vitest": "^5.0.3" }, "engines": { "node": ">=24" diff --git a/platform/backend/src/env.test.ts b/platform/backend/src/env.test.ts index 73319903..744b45fe 100644 --- a/platform/backend/src/env.test.ts +++ b/platform/backend/src/env.test.ts @@ -7,6 +7,7 @@ beforeEach(() => { } for (const name of [ "LIGHTSPEED_PLATFORM_DATABASE_URL", + "LIGHTSPEED_PLATFORM_DATABASE_AUTH_MODE", "LIGHTSPEED_PLATFORM_AUTH_SECRET", "LIGHTSPEED_PLATFORM_DEV_SEED", "LIGHTSPEED_PLATFORM_BASE_URL", @@ -26,6 +27,18 @@ afterEach(() => { }); describe("platform environment", () => { + test("selects database workload identity only when asked", () => { + vi.stubEnv("LIGHTSPEED_PLATFORM_DATABASE_URL", "postgres://platform"); + vi.stubEnv("LIGHTSPEED_PLATFORM_AUTH_SECRET", "platform-secret"); + vi.stubEnv("AZURE_CLIENT_ID", "client-id"); + vi.stubEnv("AZURE_TENANT_ID", "tenant-id"); + vi.stubEnv("AZURE_FEDERATED_TOKEN_FILE", "/var/run/token"); + expect(loadEnv().databaseWorkloadIdentity).toBeNull(); + + vi.stubEnv("LIGHTSPEED_PLATFORM_DATABASE_AUTH_MODE", "workload_identity"); + expect(loadEnv().databaseWorkloadIdentity).toMatchObject({ clientId: "client-id" }); + }); + test("loads the Lightspeed platform names", () => { vi.stubEnv("LIGHTSPEED_PLATFORM_DATABASE_URL", "postgres://platform"); vi.stubEnv("LIGHTSPEED_PLATFORM_AUTH_SECRET", "platform-secret"); diff --git a/platform/backend/src/env.ts b/platform/backend/src/env.ts index c4364e07..bb477935 100644 --- a/platform/backend/src/env.ts +++ b/platform/backend/src/env.ts @@ -1,3 +1,5 @@ +import { databaseWorkloadIdentityFromEnv, type DatabaseWorkloadIdentity } from "@lightspeed-ai/platform-db"; + export interface OidcConfig { issuer: string; discoveryUrl: string; @@ -16,6 +18,8 @@ export interface OidcConfig { export interface ServerEnv { databaseUrl: string; + /// Set when the database login uses a Microsoft Entra token instead of a password. + databaseWorkloadIdentity?: DatabaseWorkloadIdentity | null; authSecret: string; /// Public origin the platform is served from. baseUrl: string; @@ -81,6 +85,7 @@ export function loadEnv(): ServerEnv { } return { databaseUrl: required("LIGHTSPEED_PLATFORM_DATABASE_URL"), + databaseWorkloadIdentity: databaseWorkloadIdentityFromEnv(process.env), authSecret: required("LIGHTSPEED_PLATFORM_AUTH_SECRET"), baseUrl: process.env.LIGHTSPEED_PLATFORM_BASE_URL ?? "http://localhost:3000", trustedOrigins: csv(process.env.LIGHTSPEED_PLATFORM_TRUSTED_ORIGINS), diff --git a/platform/backend/src/main.ts b/platform/backend/src/main.ts index 3d6a5de9..354cc7ca 100644 --- a/platform/backend/src/main.ts +++ b/platform/backend/src/main.ts @@ -7,7 +7,7 @@ import { seedDevelopment } from "./development-seed.js"; import { loadEnv } from "./env.js"; const env = loadEnv(); -const { db, pool } = createDb(env.databaseUrl); +const { db, pool } = createDb(env.databaseUrl, env.databaseWorkloadIdentity); console.log("[main] applying migrations…"); await migrateDb({ db, pool }); diff --git a/platform/db/package.json b/platform/db/package.json index a04bc93c..d34f0411 100644 --- a/platform/db/package.json +++ b/platform/db/package.json @@ -8,16 +8,21 @@ }, "scripts": { "generate": "drizzle-kit generate", + "test": "vitest run", "test:migrations": "tsx scripts/check-migrations.ts" }, "dependencies": { + "@azure/identity": "^4.14.1", "drizzle-orm": "^0.45.3", - "pg": "^8.23.1" + "pg": "^8.23.1", + "pg-connection-string": "^2.14.1" }, "devDependencies": { + "@azure/core-rest-pipeline": "^1.25.0", "@types/pg": "^8.23.1", "better-auth": "1.7.7", - "drizzle-kit": "^0.31.11" + "drizzle-kit": "^0.31.11", + "vitest": "^5.0.3" }, "engines": { "node": ">=24" diff --git a/platform/db/src/index.ts b/platform/db/src/index.ts index c7d00191..2beb8609 100644 --- a/platform/db/src/index.ts +++ b/platform/db/src/index.ts @@ -4,8 +4,10 @@ import { fileURLToPath } from "node:url"; import path from "node:path"; import pg from "pg"; import * as schema from "./schema/index.js"; +import { workloadIdentityPoolConfig, type DatabaseWorkloadIdentity } from "./workload-identity.js"; export { schema }; +export { databaseWorkloadIdentityFromEnv, type DatabaseWorkloadIdentity } from "./workload-identity.js"; export { identityUserFields, identitySessionFields } from "./auth-fields.js"; export type Db = NodePgDatabase; @@ -14,8 +16,34 @@ export interface DbHandle { pool: pg.Pool; } -export function createDb(connectionString: string): DbHandle { - const pool = new pg.Pool({ connectionString }); +/// Limits how often connections open, because each one costs a TLS handshake and +/// server-side authentication. `pg` does not check a connection before handing it +/// out, so keepalive is what stops long idle periods from leaving dead sockets. +export const POOL_TUNING = { + min: 2, + idleTimeoutMillis: 10 * 60_000, + maxLifetimeSeconds: 60 * 60, + keepAlive: true, + keepAliveInitialDelayMillis: 60_000, + // Idle connections must not keep a finished script or test process alive. + allowExitOnIdle: true, +} satisfies pg.PoolConfig; + +export function createDb( + connectionString: string, + workloadIdentity?: DatabaseWorkloadIdentity | null, +): DbHandle { + const pool = new pg.Pool({ + ...POOL_TUNING, + ...(workloadIdentity + ? workloadIdentityPoolConfig(connectionString, workloadIdentity) + : { connectionString }), + }); + // Without a listener, losing an idle connection raises an uncaught error and ends + // the process. The pool replaces the connection on demand. + pool.on("error", (error) => { + console.error(`[db] idle connection lost: ${error.message}`); + }); const db = drizzle(pool, { schema, casing: "snake_case" }); return { db, pool }; } diff --git a/platform/db/src/workload-identity.test.ts b/platform/db/src/workload-identity.test.ts new file mode 100644 index 00000000..3eac65b9 --- /dev/null +++ b/platform/db/src/workload-identity.test.ts @@ -0,0 +1,326 @@ +import { mkdtempSync, renameSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import type { AccessToken, TokenCredential } from "@azure/identity"; +import { WorkloadIdentityCredential } from "@azure/identity"; +import { createHttpHeaders, type HttpClient, type PipelineRequest } from "@azure/core-rest-pipeline"; +import pg from "pg"; +import { afterEach, describe, expect, test, vi } from "vitest"; +import { createDb, POOL_TUNING } from "./index.js"; +import { + databaseWorkloadIdentityFromEnv, + workloadIdentityPoolConfig, + type DatabaseWorkloadIdentity, +} from "./workload-identity.js"; + +const URL_OK = "postgres://identity-name@db.example:5432/platform?sslmode=verify-full"; +const ASSERTION = "secret-assertion.jwt.value"; +const ENV = { + LIGHTSPEED_PLATFORM_DATABASE_AUTH_MODE: "workload_identity", + AZURE_CLIENT_ID: "client-id", + AZURE_TENANT_ID: "tenant-id", + AZURE_FEDERATED_TOKEN_FILE: "/var/run/token", +}; + +const directories: string[] = []; +function tempDirectory() { + const directory = mkdtempSync(join(tmpdir(), "platform-db-")); + directories.push(directory); + return directory; +} + +afterEach(() => { + for (const directory of directories.splice(0)) rmSync(directory, { recursive: true, force: true }); + vi.restoreAllMocks(); +}); + +function identity(tokenFile = "/var/run/token"): DatabaseWorkloadIdentity { + return { + clientId: "client-id", + tenantId: "tenant-id", + tokenFile, + authorityHost: "https://login.example", + }; +} + +function fixedCredential(getToken: TokenCredential["getToken"]): TokenCredential { + return { getToken }; +} + +async function passwordOf(config: pg.PoolConfig): Promise { + const password = config.password; + if (typeof password !== "function") throw new Error("expected a password function"); + return password(); +} + +describe("databaseWorkloadIdentityFromEnv", () => { + test("password stays the default and Azure variables do not change it", () => { + expect(databaseWorkloadIdentityFromEnv({})).toBeNull(); + expect( + databaseWorkloadIdentityFromEnv({ ...ENV, LIGHTSPEED_PLATFORM_DATABASE_AUTH_MODE: undefined }), + ).toBeNull(); + expect( + databaseWorkloadIdentityFromEnv({ ...ENV, LIGHTSPEED_PLATFORM_DATABASE_AUTH_MODE: "password" }), + ).toBeNull(); + }); + + test("requires each identity setting and names it", () => { + for (const name of ["AZURE_CLIENT_ID", "AZURE_TENANT_ID", "AZURE_FEDERATED_TOKEN_FILE"]) { + expect(() => databaseWorkloadIdentityFromEnv({ ...ENV, [name]: "" })).toThrow(name); + } + }); + + test("rejects unknown modes without echoing them", () => { + const load = () => + databaseWorkloadIdentityFromEnv({ LIGHTSPEED_PLATFORM_DATABASE_AUTH_MODE: "secret-mode" }); + expect(load).toThrow("LIGHTSPEED_PLATFORM_DATABASE_AUTH_MODE"); + expect(load).not.toThrow("secret-mode"); + }); + + test("validates the token path and tenant", () => { + expect(() => + databaseWorkloadIdentityFromEnv({ ...ENV, AZURE_FEDERATED_TOKEN_FILE: "relative/token" }), + ).toThrow("absolute"); + for (const tenant of ["../other", "tenant/path"]) { + expect(() => databaseWorkloadIdentityFromEnv({ ...ENV, AZURE_TENANT_ID: tenant })).toThrow( + "AZURE_TENANT_ID", + ); + } + }); + + test("normalizes authorities to one form and defaults to the public cloud", () => { + expect(databaseWorkloadIdentityFromEnv(ENV)?.authorityHost).toBe( + "https://login.microsoftonline.com", + ); + for (const authority of [ + "https://login.microsoftonline.us", + "https://login.microsoftonline.us/", + "HTTPS://LOGIN.microsoftonline.us:443/", + ]) { + expect( + databaseWorkloadIdentityFromEnv({ ...ENV, AZURE_AUTHORITY_HOST: authority })?.authorityHost, + ).toBe("https://login.microsoftonline.us"); + } + for (const invalid of [ + "http://login.example", + "login.example", + "https://login.example/tenant", + "https://login.example/?x=1", + "https://user:secret@login.example/", + ]) { + expect(() => + databaseWorkloadIdentityFromEnv({ ...ENV, AZURE_AUTHORITY_HOST: invalid }), + ).toThrow("AZURE_AUTHORITY_HOST"); + } + }); +}); + +describe("workloadIdentityPoolConfig", () => { + const credential = fixedCredential(async () => ({ token: "t", expiresOnTimestamp: 0 })); + + test("requires a named user, verified TLS, and no password", () => { + for (const [url, expected] of [ + ["not a url", "could not be parsed"], + ["postgres://db.example/platform?sslmode=verify-full", "managed identity"], + ["postgres://user:secret-password@db.example/platform?sslmode=verify-full", "password"], + ["postgres://user@db.example/platform?sslmode=verify-full&password=secret-password", "password"], + ["postgres://user@db.example/platform", "verify-full"], + ["postgres://user@db.example/platform?sslmode=require", "verify-full"], + ] as const) { + const build = () => workloadIdentityPoolConfig(url, identity(), credential); + expect(build).toThrow(expected); + expect(build).not.toThrow("secret-password"); + } + }); + + test("pg keeps the password function, which a connectionString would discard", () => { + const config = workloadIdentityPoolConfig(URL_OK, identity(), credential); + const client = new pg.Client(config); + expect(typeof client.password).toBe("function"); + expect(client.user).toBe("identity-name"); + expect(client.database).toBe("platform"); + expect(client.host).toBe("db.example"); + expect(client.ssl).toBeTruthy(); + expect(client.ssl).not.toHaveProperty("rejectUnauthorized", false); + expect(config).not.toHaveProperty("connectionString"); + }); + + test("asks for the PostgreSQL scope on each new connection", async () => { + const getToken = vi.fn(async () => ({ token: "access-token", expiresOnTimestamp: 0 })); + const config = workloadIdentityPoolConfig(URL_OK, identity(), fixedCredential(getToken)); + expect(await passwordOf(config)).toBe("access-token"); + await passwordOf(config); + expect(getToken).toHaveBeenCalledTimes(2); + expect(getToken).toHaveBeenCalledWith("https://ossrdbms-aad.database.windows.net/.default"); + }); + + test("an unreadable token file names the stage and path", async () => { + const tokenFile = join(tempDirectory(), "missing"); + const failing = fixedCredential(async () => { + throw new Error(`failed with ${ASSERTION}`); + }); + const config = workloadIdentityPoolConfig(URL_OK, identity(tokenFile), failing); + const error: Error = await passwordOf(config).catch((caught) => caught); + expect(error.message).toContain(tokenFile); + expect(error.message).toContain("AZURE_FEDERATED_TOKEN_FILE"); + expect(error.message).toContain("ENOENT"); + expect(error.message).not.toContain("secret-"); + expect(error.cause).toBeUndefined(); + }); + + test("a failed exchange keeps identifiers and drops the original error", async () => { + const tokenFile = join(tempDirectory(), "token"); + writeFileSync(tokenFile, ASSERTION); + const failing = fixedCredential(async () => { + throw Object.assign(new Error(`rejected ${ASSERTION}`), { + statusCode: 401, + errorResponse: { + error: "invalid_client", + errorDescription: `bad assertion ${ASSERTION}`, + errorCodes: [70021], + traceId: "trace-1", + correlationId: "header.payload.signature", + }, + }); + }); + const config = workloadIdentityPoolConfig(URL_OK, identity(tokenFile), failing); + const error: Error = await passwordOf(config).catch((caught) => caught); + expect(error.message).toContain("Microsoft Entra token exchange failed"); + expect(error.message).toContain("HTTP 401"); + expect(error.message).toContain("error=invalid_client"); + expect(error.message).toContain("AADSTS70021"); + expect(error.message).toContain("trace_id=trace-1"); + // Dotted values such as JWTs are not identifiers, even in an allowlisted field. + expect(error.message).not.toContain("header.payload.signature"); + expect(error.message).not.toContain("secret-"); + expect(error.cause).toBeUndefined(); + }); +}); + +/// Runs the real Azure credential against a scripted transport, so the wire content +/// and caching are the SDK's own. +describe("WorkloadIdentityCredential wiring", () => { + function entra() { + const exchanges: Record[] = []; + let issued = 0; + const httpClient: HttpClient = { + async sendRequest(request: PipelineRequest) { + const respond = (status: number, body: unknown) => ({ + request, + status, + headers: createHttpHeaders({ "content-type": "application/json" }), + bodyAsText: JSON.stringify(body), + }); + if (new URL(request.url).pathname.endsWith("/oauth2/v2.0/token")) { + exchanges.push(Object.fromEntries(new URLSearchParams(String(request.body)))); + issued += 1; + return respond(200, { + token_type: "Bearer", + expires_in: 3600, + access_token: `access-token-${issued}`, + }); + } + return respond(200, { + token_endpoint: "https://login.example/tenant-id/oauth2/v2.0/token", + authorization_endpoint: "https://login.example/tenant-id/oauth2/v2.0/authorize", + issuer: "https://login.example/tenant-id/v2.0", + jwks_uri: "https://login.example/tenant-id/discovery/v2.0/keys", + end_session_endpoint: "https://login.example/tenant-id/oauth2/v2.0/logout", + }); + }, + }; + return { exchanges, httpClient }; + } + + function credentialFor(tokenFile: string, httpClient: HttpClient) { + return new WorkloadIdentityCredential({ + clientId: "client-id", + tenantId: "tenant-id", + tokenFilePath: tokenFile, + authorityHost: "https://login.example", + disableInstanceDiscovery: true, + httpClient, + }); + } + + test("presents the projected token and reuses the access token until it nears expiry", async () => { + const tokenFile = join(tempDirectory(), "token"); + writeFileSync(tokenFile, ASSERTION); + const { exchanges, httpClient } = entra(); + const config = workloadIdentityPoolConfig( + URL_OK, + identity(tokenFile), + credentialFor(tokenFile, httpClient), + ); + + expect(await passwordOf(config)).toBe("access-token-1"); + expect(await passwordOf(config)).toBe("access-token-1"); + + expect(exchanges).toHaveLength(1); + expect(exchanges[0]).toMatchObject({ + grant_type: "client_credentials", + client_id: "client-id", + client_assertion: ASSERTION, + client_assertion_type: "urn:ietf:params:oauth:client-assertion-type:jwt-bearer", + }); + expect(exchanges[0]?.scope).toContain("https://ossrdbms-aad.database.windows.net/.default"); + }); + + test("a new exchange reads a rotated token file", async () => { + const directory = tempDirectory(); + const tokenFile = join(directory, "token"); + writeFileSync(tokenFile, ASSERTION); + const { exchanges, httpClient } = entra(); + const credential = credentialFor(tokenFile, httpClient); + const token = (value: AccessToken | null) => value?.token; + + expect(token(await credential.getToken("https://first.example/.default"))).toBeDefined(); + const staged = join(directory, "staged"); + writeFileSync(staged, "secret-rotated-assertion"); + renameSync(staged, tokenFile); + const realNow = Date.now(); + // The SDK reuses the file's contents for five minutes. + vi.spyOn(Date, "now").mockReturnValue(realNow + 6 * 60_000); + expect(token(await credential.getToken("https://second.example/.default"))).toBeDefined(); + + expect(exchanges.map((exchange) => exchange.client_assertion)).toEqual([ + ASSERTION, + "secret-rotated-assertion", + ]); + }); +}); + +describe("token failure reasons", () => { + test("keep a leading SDK reason code and nothing after it", async () => { + const tokenFile = join(tempDirectory(), "token"); + writeFileSync(tokenFile, ASSERTION); + const failing = fixedCredential(async () => { + throw new Error(`endpoints_resolution_error: details ${ASSERTION}`); + }); + const config = workloadIdentityPoolConfig(URL_OK, identity(tokenFile), failing); + const error: Error = await passwordOf(config).catch((caught) => caught); + expect(error.message).toContain("reason=endpoints_resolution_error"); + expect(error.message).not.toContain("secret-"); + }); +}); + +describe("createDb", () => { + test("applies one tuned pool in password mode and survives losing an idle connection", async () => { + const consoleError = vi.spyOn(console, "error").mockImplementation(() => {}); + const { pool } = createDb("postgres://user:password@db.example/platform"); + expect(pool.options).toMatchObject(POOL_TUNING); + expect(pool.options.connectionString).toBe("postgres://user:password@db.example/platform"); + + // Without a listener, EventEmitter throws this error at the process. + expect(() => pool.emit("error", new Error("connection reset"), {} as pg.PoolClient)).not.toThrow(); + expect(consoleError).toHaveBeenCalledWith(expect.stringContaining("connection reset")); + await pool.end(); + }); + + test("applies the same tuning with workload identity", async () => { + const { pool } = createDb(URL_OK, identity()); + expect(pool.options).toMatchObject(POOL_TUNING); + expect(typeof pool.options.password).toBe("function"); + await pool.end(); + }); +}); diff --git a/platform/db/src/workload-identity.ts b/platform/db/src/workload-identity.ts new file mode 100644 index 00000000..dc171acd --- /dev/null +++ b/platform/db/src/workload-identity.ts @@ -0,0 +1,160 @@ +import { access } from "node:fs/promises"; +import { isAbsolute } from "node:path"; +import { WorkloadIdentityCredential, type TokenCredential } from "@azure/identity"; +import { parse, toClientConfig } from "pg-connection-string"; +import type pg from "pg"; + +const DATABASE_URL = "LIGHTSPEED_PLATFORM_DATABASE_URL"; +const AUTH_MODE = "LIGHTSPEED_PLATFORM_DATABASE_AUTH_MODE"; +const POSTGRES_SCOPE = "https://ossrdbms-aad.database.windows.net/.default"; +const PUBLIC_AUTHORITY = "https://login.microsoftonline.com"; +/// Anything longer or with other characters is not a code or identifier, so it is withheld. +const DIAGNOSTIC_VALUE = /^[A-Za-z0-9_-]{1,128}$/; + +export interface DatabaseWorkloadIdentity { + clientId: string; + tenantId: string; + tokenFile: string; + authorityHost: string; +} + +/// Reads the auth mode and, when it selects workload identity, the standard Azure +/// settings. Those settings are ignored otherwise, because other software in the +/// pod may set them. +export function databaseWorkloadIdentityFromEnv( + env: Record, +): DatabaseWorkloadIdentity | null { + const mode = env[AUTH_MODE] || "password"; + if (mode === "password") return null; + if (mode !== "workload_identity") { + throw new Error(`${AUTH_MODE} must be password or workload_identity`); + } + const required = (name: string) => { + const value = env[name]; + if (!value?.trim()) throw new Error(`${name} must be set when ${AUTH_MODE}=workload_identity`); + return value; + }; + const tenantId = required("AZURE_TENANT_ID"); + // The tenant becomes a path segment of the token URL. + if (!/^[A-Za-z0-9][A-Za-z0-9.-]*$/.test(tenantId)) { + throw new Error("AZURE_TENANT_ID must be a tenant ID or domain name"); + } + const tokenFile = required("AZURE_FEDERATED_TOKEN_FILE"); + if (!isAbsolute(tokenFile)) throw new Error("AZURE_FEDERATED_TOKEN_FILE must be an absolute path"); + return { + clientId: required("AZURE_CLIENT_ID"), + tenantId, + tokenFile, + authorityHost: normalizedAuthorityHost(env.AZURE_AUTHORITY_HOST), + }; +} + +function normalizedAuthorityHost(value: string | undefined): string { + if (!value) return PUBLIC_AUTHORITY; + const invalid = new Error( + "AZURE_AUTHORITY_HOST must be an HTTPS authority with a host and no credentials, path, query, or fragment", + ); + let url: URL; + try { + url = new URL(value); + } catch { + throw invalid; + } + if ( + url.protocol !== "https:" || + url.username || + url.password || + url.pathname !== "/" || + url.search || + url.hash + ) { + throw invalid; + } + return url.origin; +} + +/// Connection settings that present an Entra token as the login password. `pg` +/// discards a password function when it is combined with a `connectionString`, so +/// the URL is expanded into separate fields. +export function workloadIdentityPoolConfig( + connectionString: string, + identity: DatabaseWorkloadIdentity, + credential: TokenCredential = new WorkloadIdentityCredential({ + clientId: identity.clientId, + tenantId: identity.tenantId, + tokenFilePath: identity.tokenFile, + authorityHost: identity.authorityHost, + // The authority is configured explicitly, so it needs no discovery request. + disableInstanceDiscovery: true, + }), +): pg.PoolConfig { + let url: URL; + try { + url = new URL(connectionString); + } catch { + throw new Error(`${DATABASE_URL} could not be parsed`); + } + if (!url.username) throw new Error(`${DATABASE_URL} must name the managed identity as its user`); + if (url.password || url.searchParams.has("password")) { + throw new Error(`${DATABASE_URL} must not contain a password with workload identity`); + } + // The token travels as a cleartext password inside TLS, so the server's identity + // must be verified before it is sent. + if (url.searchParams.get("sslmode") !== "verify-full") { + throw new Error(`${DATABASE_URL} must set sslmode=verify-full with workload identity`); + } + const config = toClientConfig(parse(connectionString)) as pg.PoolConfig; + return { ...config, password: entraTokenPassword(credential, identity.tokenFile) }; +} + +/// The credential caches and renews its token, so this only reaches Entra when a +/// token is close to expiry. `pg` calls it once for each new connection. +function entraTokenPassword(credential: TokenCredential, tokenFile: string): () => Promise { + return async () => { + let failure: unknown; + try { + const token = await credential.getToken(POSTGRES_SCOPE); + if (token) return token.token; + } catch (error) { + failure = error; + } + throw await tokenFailure(failure, tokenFile); + }; +} + +/// Builds a new error rather than wrapping the original, whose message and response +/// body would otherwise stay reachable through logs and `cause`. +async function tokenFailure(failure: unknown, tokenFile: string): Promise { + const unreadable = await access(tokenFile).then( + () => null, + (error: NodeJS.ErrnoException) => error.code ?? "unreadable", + ); + if (unreadable) { + return new Error( + `could not read the projected token file '${tokenFile}' named by AZURE_FEDERATED_TOKEN_FILE: ${unreadable}`, + ); + } + const { message, statusCode, code, errorResponse } = (failure ?? {}) as { + message?: unknown; + statusCode?: unknown; + code?: unknown; + errorResponse?: { error?: unknown; errorCodes?: unknown; traceId?: unknown; correlationId?: unknown }; + }; + const fields = [ + typeof statusCode === "number" ? `HTTP ${statusCode}` : null, + diagnostic("code", code), + // The SDK's other failures begin with a reason code, such as a resolution error. + diagnostic("reason", typeof message === "string" ? message.split(":", 1)[0] : undefined), + diagnostic("error", errorResponse?.error), + ...(Array.isArray(errorResponse?.errorCodes) ? errorResponse.errorCodes : []) + .filter((value) => typeof value === "number") + .map((value) => `AADSTS${value}`), + diagnostic("trace_id", errorResponse?.traceId), + diagnostic("correlation_id", errorResponse?.correlationId), + ].filter(Boolean); + return new Error(`Microsoft Entra token exchange failed: ${fields.join(" ") || "details withheld"}`); +} + +function diagnostic(name: string, value: unknown): string | null { + return typeof value === "string" && DIAGNOSTIC_VALUE.test(value) ? `${name}=${value}` : null; +} From fa2873f6a66da1267a29515ef287e8db3065b93b Mon Sep 17 00:00:00 2001 From: lb <542828+lukebuehler@users.noreply.github.com> Date: Thu, 5 Nov 2026 14:47:54 +0100 Subject: [PATCH 3/3] Prepare Lightspeed v0.6.1 release Bump the product version and add release notes for workload identity and the connection pool changes. workflow-compat.sh v0.6.0 reports restart-in-place. Co-Authored-By: Claude Opus 5.5 Signed-off-by: lb <542828+lukebuehler@users.noreply.github.com> --- Cargo.lock | 10 +++++----- Cargo.toml | 2 +- release/metadata.env | 2 +- release/notes/v0.6.1.md | 37 +++++++++++++++++++++++++++++++++++++ 4 files changed, 44 insertions(+), 7 deletions(-) create mode 100644 release/notes/v0.6.1.md diff --git a/Cargo.lock b/Cargo.lock index a8135ac9..bc5c8273 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -735,7 +735,7 @@ checksum = "1c133bc6a41be0d194c306b5506d15e6feeea7b1d6604bd3f8310dfb2ca96486" [[package]] name = "cli" -version = "0.6.0" +version = "0.6.1" dependencies = [ "anyhow", "api", @@ -1378,7 +1378,7 @@ dependencies = [ [[package]] name = "environment-daemon" -version = "0.6.0" +version = "0.6.1" dependencies = [ "anyhow", "clap", @@ -1433,7 +1433,7 @@ dependencies = [ [[package]] name = "environment-provider-incus" -version = "0.6.0" +version = "0.6.1" dependencies = [ "anyhow", "async-trait", @@ -4263,7 +4263,7 @@ dependencies = [ [[package]] name = "release-info" -version = "0.6.0" +version = "0.6.1" dependencies = [ "tempfile", ] @@ -5329,7 +5329,7 @@ dependencies = [ [[package]] name = "temporal-runtime" -version = "0.6.0" +version = "0.6.1" dependencies = [ "anyhow", "api", diff --git a/Cargo.toml b/Cargo.toml index 49ee0a00..86cb9347 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,4 +30,4 @@ members = [ resolver = "2" [workspace.package] -version = "0.6.0" +version = "0.6.1" diff --git a/release/metadata.env b/release/metadata.env index 6bc838d3..092bb31c 100644 --- a/release/metadata.env +++ b/release/metadata.env @@ -4,7 +4,7 @@ LIGHTSPEED_RELEASE_TARGET=x86_64-unknown-linux-gnu # The environment daemon alone is published as a static binary so it runs on # any Linux image regardless of its glibc. LIGHTSPEED_ENVD_TARGET=x86_64-unknown-linux-musl -LIGHTSPEED_PRODUCT_VERSION=0.6.0 +LIGHTSPEED_PRODUCT_VERSION=0.6.1 LIGHTSPEED_RELEASE_RUST_VERSION=1.99.0 LIGHTSPEED_RELEASE_BUILD_BASE_IMAGE=rust:1.99.0-bookworm@sha256:59037199c44290f2befcdd58dcc540164763fc296950255aaefeef096a1866b0 LIGHTSPEED_API_PROTOCOL_VERSION=lightspeed.agent.api.v1 diff --git a/release/notes/v0.6.1.md b/release/notes/v0.6.1.md new file mode 100644 index 00000000..e1fba3ef --- /dev/null +++ b/release/notes/v0.6.1.md @@ -0,0 +1,37 @@ +Lightspeed v0.6.1 adds Microsoft Entra workload identity for Azure Blob storage and for PostgreSQL logins from the Runtime and Platform, so Azure deployments can run without account keys, SAS tokens or stored database passwords. It also reduces how often the Runtime and Platform reconnect to PostgreSQL. + +**Compatibility and upgrade** + +This is a `restart-in-place` release: `scripts/release/workflow-compat.sh v0.6.0` reports `restart-in-place`, so existing sessions continue on their current Temporal namespace. + +Nothing is converted. Runtime schema revision remains **14**, Platform schema revision remains **4**, the public API protocol remains `lightspeed.agent.api.v1`, and envd and controller protocols remain **4**. Stored objects, keys and prefixes stay valid when a deployment moves to workload identity. + +Restart the Runtime and Platform with matching images from this release. Existing settings keep working without changes. + +**Azure Blob storage** + +- Set `LIGHTSPEED_OBJECT_STORE_AZURE_AUTH_MODE=workload_identity`, with `AZURE_CLIENT_ID`, `AZURE_TENANT_ID`, `AZURE_FEDERATED_TOKEN_FILE` and optionally `AZURE_AUTHORITY_HOST`. The identity needs a data-plane role that can read, write, list and delete blobs in the container. +- The mode rejects a configured access key or SAS token and an HTTP endpoint, including loopback, and never falls back to another credential. `access_key` and `sas_token` are also accepted as explicit modes. Leaving the setting unset selects whichever single static credential is present, as before. +- A wrong token path or a missing federated credential appears at the first Blob operation rather than at startup. In this mode, errors keep only identifiers such as `AADSTS` codes, storage error codes and request IDs, so look up further detail in the service's logs using those IDs. + +**PostgreSQL** + +- Set `LIGHTSPEED_POSTGRES_AUTH_MODE=workload_identity` for the Runtime and `LIGHTSPEED_PLATFORM_DATABASE_AUTH_MODE=workload_identity` for the Platform. Both default to `password` and use the same `AZURE_*` settings. +- The database URL must name the identity as its user, hold no password and set `sslmode=verify-full`. The user is the identity's name, not its client ID, and an Entra administrator must first create a role for it. See [deployment configuration](https://ls.bot/docs/deployment/configuration/) for the SQL. +- Both processes need Microsoft Entra to open connections at startup. During a later outage they can still open connections until the current token expires, and they keep using the ones already open. +- Temporal is configured separately and still uses its own password. + +**Connection pools, in every authentication mode** + +- The Runtime keeps 2 connections open when idle and recycles connections after 1 hour instead of 30 minutes. +- The Platform keeps idle connections for 10 minutes instead of 10 seconds, retains up to 2, recycles connections after 1 hour and enables TCP keepalive. +- Losing an idle connection no longer ends the Platform process. +- Each process now holds connections for longer, so check that the total stays under the server's `max_connections`. + +**Validation limits** + +Workload identity has not been run against real Azure services, so treat the first deployment as acceptance testing. The real Entra exchange, Azure's token validation and connection cost on a real server are all unproven. `azure_blob_live` and `azure_postgres_live` cover those from a federated workload. + +Local validation ran the real Blob and token clients against a TLS fake of Entra and Azure Storage, covering wire content, token rotation, certificate and hostname rejection, failed authentication and redaction. The Platform's Azure credential ran through a scripted transport. A rotation test against a real PostgreSQL role showed the old password refused, new connections using the renewed token and an open connection still working. The Platform's workload identity path has not run over TLS. + +[Full changelog: v0.6.0...v0.6.1](https://github.com/smartcomputer-ai/lightspeed/compare/v0.6.0...v0.6.1)