From 59044f1956c0ecc8365bd0475eb1391fd15f4971 Mon Sep 17 00:00:00 2001 From: ELA CITY Date: Fri, 16 Oct 2026 17:12:10 +0000 Subject: [PATCH 01/11] perf(runtime): make request admission atomic and cleanup linear Review package extracted from preserved candidate b0fa431d. The original eighteen commits remain on fix/runtime-performance-080. Validation of this recomposed head is recorded separately from historical candidate evidence. --- .../elastos-runtime/src/capability/pending.rs | 577 +++++++++++++----- justfile | 11 +- 2 files changed, 427 insertions(+), 161 deletions(-) diff --git a/elastos/crates/elastos-runtime/src/capability/pending.rs b/elastos/crates/elastos-runtime/src/capability/pending.rs index 75594750f..79d496fe3 100644 --- a/elastos/crates/elastos-runtime/src/capability/pending.rs +++ b/elastos/crates/elastos-runtime/src/capability/pending.rs @@ -192,13 +192,61 @@ impl PendingCapabilityRequest { } } +#[derive(Default)] +struct PendingRequestState { + /// Requests and their index change together under the store's lock. + requests: HashMap, + /// Index from session ID to request IDs (for listing) + session_requests: HashMap>, +} + +impl PendingRequestState { + fn cleanup_expired(&mut self, now: &SecureTimestamp) -> usize { + let mut expired = 0; + for request in self.requests.values_mut() { + if matches!(request.status, RequestStatus::Pending) && request.expires_at.is_before(now) + { + request.status = RequestStatus::Expired; + expired += 1; + } + } + expired + } + + fn cleanup_old(&mut self, now: &SecureTimestamp, retention_secs: u64) -> usize { + let previous_len = self.requests.len(); + self.requests.retain(|_, request| { + // Request expiry alone says nothing about token validity. Only an + // explicit token expiry or revocation makes a grant reclaimable. + let resolved = match &request.status { + RequestStatus::Pending => false, + RequestStatus::Granted { token, .. } => { + token.expiry().is_some_and(|expiry| expiry.is_before(now)) + } + RequestStatus::Denied { .. } | RequestStatus::Expired => true, + }; + if !resolved { + return true; + } + let age_secs = now.unix_secs.saturating_sub(request.requested_at.unix_secs); + retention_secs != 0 && age_secs <= retention_secs + }); + + let removed = previous_len - self.requests.len(); + if removed != 0 { + let requests = &self.requests; + self.session_requests.retain(|_, ids| { + ids.retain(|id| requests.contains_key(id)); + !ids.is_empty() + }); + } + removed + } +} + /// Store for pending capability requests pub struct PendingRequestStore { - /// Pending requests indexed by request ID - requests: RwLock>, - - /// Index from session ID to request IDs (for listing) - session_requests: RwLock>>, + state: RwLock, /// Audit log audit_log: Arc, @@ -211,8 +259,7 @@ impl PendingRequestStore { /// Create a new pending request store pub fn new(audit_log: Arc) -> Self { Self { - requests: RwLock::new(HashMap::new()), - session_requests: RwLock::new(HashMap::new()), + state: RwLock::new(PendingRequestState::default()), audit_log, timeout_secs: DEFAULT_REQUEST_TIMEOUT_SECS, } @@ -226,8 +273,7 @@ impl PendingRequestStore { /// Create with custom timeout pub fn with_timeout(audit_log: Arc, timeout_secs: u64) -> Self { Self { - requests: RwLock::new(HashMap::new()), - session_requests: RwLock::new(HashMap::new()), + state: RwLock::new(PendingRequestState::default()), audit_log, timeout_secs, } @@ -255,83 +301,52 @@ impl PendingRequestStore { action: Action, reason: String, ) -> PendingCapabilityRequest { - // Capacity guard: evict expired if at limit - { - let count = self.requests.read().await.len(); - if count >= MAX_PENDING_REQUESTS { - self.cleanup_expired().await; - self.cleanup_old(0).await; - } - } - // Re-check after cleanup — reject if still at capacity - { - let count = self.requests.read().await.len(); - if count >= MAX_PENDING_REQUESTS { - let mut request = PendingCapabilityRequest::new_with_reason( - session_id.clone(), - resource.clone(), - action, - reason.clone(), - self.timeout_secs, - ); - request.status = RequestStatus::Denied { - reason: "Too many pending requests".to_string(), - }; - return request; - } + // Admission, cleanup, and both index updates are one state transition. + let mut state = self.state.write().await; + let now = SecureTimestamp::now(); + if state.requests.len() >= MAX_PENDING_REQUESTS { + state.cleanup_expired(&now); + state.cleanup_old(&now, 0); } - // Per-session rate limit: prevent one session from starving others - { - let session_requests = self.session_requests.read().await; - if let Some(ids) = session_requests.get(&session_id.0) { - let requests = self.requests.read().await; - let pending_count = ids - .iter() + let denial = if state.requests.len() >= MAX_PENDING_REQUESTS { + Some("Too many pending requests") + } else { + let pending_count = state.session_requests.get(&session_id.0).map_or(0, |ids| { + ids.iter() .filter(|id| { - requests - .get(*id) - .is_some_and(|r| matches!(r.status, RequestStatus::Pending)) + state.requests.get(*id).is_some_and(|request| { + matches!(request.status, RequestStatus::Pending) + && !request.expires_at.is_before(&now) + }) }) - .count(); - if pending_count >= MAX_PENDING_PER_SESSION { - let mut request = PendingCapabilityRequest::new_with_reason( - session_id.clone(), - resource.clone(), - action, - reason.clone(), - self.timeout_secs, - ); - request.status = RequestStatus::Denied { - reason: "Too many pending requests for this session".to_string(), - }; - return request; - } - } - } + .count() + }); + (pending_count >= MAX_PENDING_PER_SESSION) + .then_some("Too many pending requests for this session") + }; - let request = PendingCapabilityRequest::new_with_reason( + let mut request = PendingCapabilityRequest::new_with_reason( session_id.clone(), resource.clone(), action, reason, self.timeout_secs, ); - - // Store request - { - let mut requests = self.requests.write().await; - requests.insert(request.id.0.clone(), request.clone()); + if let Some(reason) = denial { + request.status = RequestStatus::Denied { + reason: reason.to_string(), + }; + return request; } - // Add to session index - { - let mut session_requests = self.session_requests.write().await; - session_requests - .entry(session_id.0.clone()) - .or_default() - .push(request.id.0.clone()); - } + state.requests.insert(request.id.0.clone(), request.clone()); + state + .session_requests + .entry(session_id.0.clone()) + .or_default() + .push(request.id.0.clone()); + drop(state); // Audit self.audit_log @@ -348,8 +363,8 @@ impl PendingRequestStore { /// Get a request by ID pub async fn get_request(&self, request_id: &str) -> Option { - let requests = self.requests.read().await; - requests.get(request_id).cloned() + let state = self.state.read().await; + state.requests.get(request_id).cloned() } /// Grant a pending request @@ -359,9 +374,10 @@ impl PendingRequestStore { token: CapabilityToken, duration: GrantDuration, ) -> Result<(), String> { - let mut requests = self.requests.write().await; + let mut state = self.state.write().await; - let request = requests + let request = state + .requests .get_mut(request_id) .ok_or_else(|| format!("Request not found: {}", request_id))?; @@ -384,9 +400,10 @@ impl PendingRequestStore { /// Deny a pending request pub async fn deny_request(&self, request_id: &str, reason: &str) -> Result<(), String> { - let mut requests = self.requests.write().await; + let mut state = self.state.write().await; - let request = requests + let request = state + .requests .get_mut(request_id) .ok_or_else(|| format!("Request not found: {}", request_id))?; @@ -397,13 +414,15 @@ impl PendingRequestStore { request.status = RequestStatus::Denied { reason: reason.to_string(), }; + let session_id = request.session_id.to_string(); + drop(state); // Audit self.audit_log .emit(crate::primitives::audit::AuditEvent::CapabilityDenied { timestamp: SecureTimestamp::now(), request_id: request_id.to_string(), - session_id: request.session_id.to_string(), + session_id, reason: reason.to_string(), }); @@ -412,8 +431,9 @@ impl PendingRequestStore { /// List all pending requests (for shell to display) pub async fn list_pending(&self) -> Vec { - let requests = self.requests.read().await; - requests + let state = self.state.read().await; + state + .requests .values() .filter(|r| r.is_pending()) .cloned() @@ -422,14 +442,14 @@ impl PendingRequestStore { /// List pending requests for a specific session pub async fn list_session_pending(&self, session_id: &str) -> Vec { - let session_requests = self.session_requests.read().await; - let requests = self.requests.read().await; + let state = self.state.read().await; - session_requests + state + .session_requests .get(session_id) .map(|ids| { ids.iter() - .filter_map(|id| requests.get(id)) + .filter_map(|id| state.requests.get(id)) .filter(|r| r.is_pending()) .cloned() .collect() @@ -441,80 +461,35 @@ impl PendingRequestStore { /// /// Returns the number of requests cleaned up pub async fn cleanup_expired(&self) -> usize { - let mut expired_ids = Vec::new(); - - // Find expired requests - { - let requests = self.requests.read().await; - for (id, request) in requests.iter() { - if request.is_expired() && matches!(request.status, RequestStatus::Pending) { - expired_ids.push(id.clone()); - } - } - } - - // Mark them as expired - { - let mut requests = self.requests.write().await; - for id in &expired_ids { - if let Some(request) = requests.get_mut(id) { - request.status = RequestStatus::Expired; - } - } - } - - expired_ids.len() + let mut state = self.state.write().await; + state.cleanup_expired(&SecureTimestamp::now()) } - /// Remove old completed/expired requests + /// Remove old denied/expired requests and explicitly expired grants. /// - /// Keeps requests for `retention_secs` after they're resolved + /// Retention is measured from request creation. Zero removes resolved records + /// immediately; active grants stay available for listing and revocation. pub async fn cleanup_old(&self, retention_secs: u64) -> usize { - let now = SecureTimestamp::now(); - let mut removed = Vec::new(); - - { - let mut requests = self.requests.write().await; - requests.retain(|id, request| { - // Keep pending requests - if matches!(request.status, RequestStatus::Pending) { - return true; - } - - // Remove old resolved requests - let age_secs = now.unix_secs.saturating_sub(request.requested_at.unix_secs); - if age_secs > retention_secs { - removed.push(id.clone()); - false - } else { - true - } - }); - } - - // Clean up session index - if !removed.is_empty() { - let mut session_requests = self.session_requests.write().await; - for ids in session_requests.values_mut() { - ids.retain(|id| !removed.contains(id)); - } - } - - removed.len() + let mut state = self.state.write().await; + state.cleanup_old(&SecureTimestamp::now(), retention_secs) } /// Get the number of pending requests pub async fn pending_count(&self) -> usize { - let requests = self.requests.read().await; - requests.values().filter(|r| r.is_pending()).count() + let state = self.state.read().await; + state.requests.values().filter(|r| r.is_pending()).count() } /// List granted requests for a specific session pub async fn list_session_granted(&self, session_id: &str) -> Vec { - let requests = self.requests.read().await; - requests - .values() - .filter(|r| r.session_id.to_string() == session_id && r.is_granted()) + let state = self.state.read().await; + state + .session_requests + .get(session_id) + .into_iter() + .flatten() + .filter_map(|id| state.requests.get(id)) + .filter(|request| request.is_granted()) .cloned() .collect() } @@ -523,19 +498,21 @@ impl PendingRequestStore { /// /// This changes the status to Denied with reason "Revoked" pub async fn revoke_request(&self, request_id: &str) { - let mut requests = self.requests.write().await; - if let Some(request) = requests.get_mut(request_id) { + let mut state = self.state.write().await; + if let Some(request) = state.requests.get_mut(request_id) { if request.is_granted() { request.status = RequestStatus::Denied { reason: "Revoked by user".to_string(), }; + let session_id = request.session_id.to_string(); + drop(state); // Audit self.audit_log .emit(crate::primitives::audit::AuditEvent::CapabilityDenied { timestamp: SecureTimestamp::now(), request_id: request_id.to_string(), - session_id: request.session_id.to_string(), + session_id, reason: "Revoked by user".to_string(), }); } @@ -546,24 +523,28 @@ impl PendingRequestStore { /// /// Called when epoch is advanced to revoke all capabilities pub async fn revoke_all_granted(&self) { - let mut requests = self.requests.write().await; + let mut state = self.state.write().await; let now = SecureTimestamp::now(); + let mut events = Vec::new(); - for (request_id, request) in requests.iter_mut() { + for (request_id, request) in state.requests.iter_mut() { if request.is_granted() { request.status = RequestStatus::Denied { reason: "Epoch advanced - all capabilities revoked".to_string(), }; - self.audit_log - .emit(crate::primitives::audit::AuditEvent::CapabilityDenied { - timestamp: now, - request_id: request_id.clone(), - session_id: request.session_id.to_string(), - reason: "Epoch advanced - all capabilities revoked".to_string(), - }); + events.push(crate::primitives::audit::AuditEvent::CapabilityDenied { + timestamp: now, + request_id: request_id.clone(), + session_id: request.session_id.to_string(), + reason: "Epoch advanced - all capabilities revoked".to_string(), + }); } } + drop(state); + for event in events { + self.audit_log.emit(event); + } } } @@ -575,6 +556,282 @@ mod tests { PendingRequestStore::new(Arc::new(AuditLog::new())) } + async fn request_for(store: &PendingRequestStore, session: &str) -> PendingCapabilityRequest { + store + .create_request( + SessionId::from_string(session), + ResourceId::new("localhost://Users/self/Documents/test"), + Action::Read, + ) + .await + } + + fn token_for(request: &PendingCapabilityRequest) -> CapabilityToken { + CapabilityToken::new( + "test-capsule".to_string(), + [0u8; 32], + request.resource.clone(), + request.action, + Default::default(), + SecureTimestamp::now(), + None, + ) + } + + async fn concurrent_admissions(store: Arc, shared_session: bool) -> usize { + const CONTENDERS: usize = 64; + let barrier = Arc::new(tokio::sync::Barrier::new(CONTENDERS + 1)); + let mut tasks = tokio::task::JoinSet::new(); + for index in 0..CONTENDERS { + let store = Arc::clone(&store); + let barrier = Arc::clone(&barrier); + tasks.spawn(async move { + barrier.wait().await; + let session = if shared_session { + "shared-session".to_string() + } else { + format!("concurrent-{index}") + }; + request_for(&store, &session).await.is_pending() + }); + } + barrier.wait().await; + let mut admitted = 0; + while let Some(result) = tasks.join_next().await { + admitted += usize::from(result.unwrap()); + } + admitted + } + + #[tokio::test] + async fn test_expired_requests_release_session_capacity() { + let store = create_test_store(); + for _ in 0..MAX_PENDING_PER_SESSION { + assert!(request_for(&store, "shared-session").await.is_pending()); + } + { + let mut state = store.state.write().await; + for request in state.requests.values_mut() { + request.expires_at = SecureTimestamp::at(0); + } + } + assert_eq!(store.pending_count().await, 0); + + let recovered = request_for(&store, "shared-session").await; + assert!(recovered.is_pending()); + assert!(store.get_request(recovered.id.as_str()).await.is_some()); + assert_eq!(store.list_session_pending("shared-session").await.len(), 1); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn test_concurrent_admission_respects_session_limit() { + let store = Arc::new(create_test_store()); + for _ in 0..MAX_PENDING_PER_SESSION - 1 { + request_for(&store, "shared-session").await; + } + + assert_eq!(concurrent_admissions(Arc::clone(&store), true).await, 1); + assert_eq!(store.pending_count().await, MAX_PENDING_PER_SESSION); + assert_eq!( + store.list_session_pending("shared-session").await.len(), + MAX_PENDING_PER_SESSION + ); + assert!(request_for(&store, "other-session").await.is_pending()); + } + + #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + async fn test_concurrent_admission_respects_global_limit() { + let store = Arc::new(create_test_store()); + for index in 0..MAX_PENDING_REQUESTS - 1 { + request_for(&store, &format!("prefill-{index}")).await; + } + + assert_eq!(concurrent_admissions(Arc::clone(&store), false).await, 1); + assert_eq!(store.pending_count().await, MAX_PENDING_REQUESTS); + let state = store.state.read().await; + assert_eq!(state.requests.len(), MAX_PENDING_REQUESTS); + assert_eq!(state.session_requests.len(), MAX_PENDING_REQUESTS); + } + + #[tokio::test] + async fn test_cleanup_preserves_grants_and_removes_empty_session_indexes() { + let store = create_test_store(); + let pending = request_for(&store, "mixed-session").await; + let once = request_for(&store, "mixed-session").await; + let session = request_for(&store, "grant-session").await; + let denied = request_for(&store, "mixed-session").await; + let expired = request_for(&store, "expired-session").await; + store + .grant_request(once.id.as_str(), token_for(&once), GrantDuration::Once) + .await + .unwrap(); + store + .grant_request( + session.id.as_str(), + token_for(&session), + GrantDuration::Session, + ) + .await + .unwrap(); + store + .deny_request(denied.id.as_str(), "test denial") + .await + .unwrap(); + { + let mut state = store.state.write().await; + for request in state.requests.values_mut() { + request.requested_at = SecureTimestamp::at(0); + if request.id != pending.id { + request.expires_at = SecureTimestamp::at(0); + } + } + } + + // Request expiry never expires a granted token or changes its status. + assert_eq!(store.cleanup_expired().await, 1); + assert_eq!(store.cleanup_old(60).await, 2); + assert!(store.get_request(denied.id.as_str()).await.is_none()); + assert!(store.get_request(expired.id.as_str()).await.is_none()); + assert_eq!(store.list_session_pending("mixed-session").await.len(), 1); + assert_eq!( + store.list_session_granted("mixed-session").await[0].id, + once.id + ); + assert_eq!( + store.list_session_granted("grant-session").await[0].id, + session.id + ); + assert!(!store + .state + .read() + .await + .session_requests + .contains_key("expired-session")); + + // Individual and epoch revocation make grant records eligible for cleanup. + store.revoke_request(once.id.as_str()).await; + assert_eq!(store.cleanup_old(0).await, 1); + assert!(store.list_session_granted("mixed-session").await.is_empty()); + store.revoke_all_granted().await; + assert_eq!(store.cleanup_old(0).await, 1); + assert!(!store + .state + .read() + .await + .session_requests + .contains_key("grant-session")); + assert_eq!(store.cleanup_old(0).await, 0); + assert_eq!( + store.list_session_pending("mixed-session").await[0].id, + pending.id + ); + } + + #[tokio::test] + async fn test_pressure_cleanup_preserves_grants_and_reclaims_fresh_denials() { + let store = create_test_store(); + let granted = request_for(&store, "grant-session").await; + let token = token_for(&granted); + let token_id = token.id; + store + .grant_request(granted.id.as_str(), token, GrantDuration::Session) + .await + .unwrap(); + for index in 0..MAX_PENDING_REQUESTS - 1 { + let request = request_for(&store, &format!("denied-{index}")).await; + store + .deny_request(request.id.as_str(), "test denial") + .await + .unwrap(); + } + assert_eq!( + store.state.read().await.requests.len(), + MAX_PENDING_REQUESTS + ); + + assert!(request_for(&store, "new-session").await.is_pending()); + let retained = store.get_request(granted.id.as_str()).await.unwrap(); + assert_eq!(retained.granted_token().unwrap().id, token_id); + assert_eq!(store.list_session_granted("grant-session").await.len(), 1); + let state = store.state.read().await; + assert_eq!(state.requests.len(), 2); + assert_eq!(state.session_requests.len(), 2); + } + + #[tokio::test] + async fn test_live_grants_fill_capacity_until_explicit_revocation() { + let store = create_test_store(); + for _ in 0..MAX_PENDING_REQUESTS { + let request = request_for(&store, "grant-session").await; + assert!(request.is_pending()); + store + .grant_request( + request.id.as_str(), + token_for(&request), + GrantDuration::Session, + ) + .await + .unwrap(); + } + assert!(request_for(&store, "new-session").await.is_denied()); + assert_eq!( + store.list_session_granted("grant-session").await.len(), + MAX_PENDING_REQUESTS + ); + assert_eq!(store.cleanup_old(0).await, 0); + + store.revoke_all_granted().await; + assert!(request_for(&store, "new-session").await.is_pending()); + assert!(store.list_session_granted("grant-session").await.is_empty()); + assert_eq!(store.state.read().await.requests.len(), 1); + } + + #[tokio::test] + async fn test_cleanup_reclaims_only_explicitly_expired_grants() { + let store = create_test_store(); + let expired = request_for(&store, "expired-grant-session").await; + let live = request_for(&store, "live-grant-session").await; + let mut expired_token = token_for(&expired); + expired_token.expiry = Some(SecureTimestamp::at(0)); + let mut live_token = token_for(&live); + live_token.expiry = Some(SecureTimestamp::after_secs(300)); + store + .grant_request(expired.id.as_str(), expired_token, GrantDuration::Session) + .await + .unwrap(); + store + .grant_request(live.id.as_str(), live_token, GrantDuration::Once) + .await + .unwrap(); + + assert_eq!(store.cleanup_old(0).await, 1); + assert!(store.get_request(expired.id.as_str()).await.is_none()); + assert_eq!( + store.list_session_granted("live-grant-session").await[0].id, + live.id + ); + assert!(!store + .state + .read() + .await + .session_requests + .contains_key("expired-grant-session")); + } + + #[tokio::test] + async fn test_expired_capacity_recovers_without_waiting_for_next_second() { + let store = PendingRequestStore::with_timeout(Arc::new(AuditLog::new()), 0); + for index in 0..MAX_PENDING_REQUESTS { + request_for(&store, &format!("expired-{index}")).await; + } + let recovered = request_for(&store, "new-session").await; + assert!(!recovered.is_denied()); + assert!(store.get_request(recovered.id.as_str()).await.is_some()); + let state = store.state.read().await; + assert_eq!(state.requests.len(), 1); + assert_eq!(state.session_requests.len(), 1); + } + #[tokio::test] async fn test_create_request() { let store = create_test_store(); diff --git a/justfile b/justfile index 615e4f434..947c3b7b5 100644 --- a/justfile +++ b/justfile @@ -244,8 +244,17 @@ product-data: # Verify the checked-in ElastOS Bus contract and real Component fixture bus-conformance: + #!/usr/bin/env bash + set -euo pipefail node scripts/check-elastos-bus-wit.mjs - cd elastos && cargo test -p elastos-server component_runs_through_real_bus_authority_provider_and_audit_paths -- --nocapture + cd elastos + test_name=runtime::tests::component_conformance_exercises_bus_authorization_dispatch_and_audit + matches="$(cargo test --locked -p elastos-server --lib "$test_name" -- --exact --list | awk -v expected="$test_name: test" '$0 == expected { count++ } END { print count + 0 }')" + if [[ "$matches" != 1 ]]; then + echo "Bus conformance must select exactly one test; found $matches" >&2 + exit 1 + fi + cargo test --locked -p elastos-server --lib "$test_name" -- --exact --include-ignored --nocapture # Validate canonical capsule scaffolds against the current manifest and WIT contracts capsule-templates: From 79dd7b42f8a70e445ca1927dc6d97b051f3a7293 Mon Sep 17 00:00:00 2001 From: ELA CITY Date: Sat, 17 Oct 2026 06:56:09 +0000 Subject: [PATCH 02/11] fix(runtime): serialize persisted revocation epoch updates --- .../elastos-runtime/src/capability/store.rs | 240 ++++++++++++++++-- 1 file changed, 212 insertions(+), 28 deletions(-) diff --git a/elastos/crates/elastos-runtime/src/capability/store.rs b/elastos/crates/elastos-runtime/src/capability/store.rs index 8973db7bc..5fb2761a7 100644 --- a/elastos/crates/elastos-runtime/src/capability/store.rs +++ b/elastos/crates/elastos-runtime/src/capability/store.rs @@ -9,6 +9,7 @@ use std::collections::{HashMap, HashSet}; use std::fs; use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::Mutex; use tokio::sync::RwLock; use super::token::TokenId; @@ -22,6 +23,9 @@ pub struct CapabilityStore { /// Revocation epoch — all tokens with epoch < this value are rejected. epoch: AtomicU64, + /// Serializes every epoch snapshot, persistent write and memory update. + epoch_writer: Mutex<()>, + /// Per-token use counts use_counts: RwLock>, @@ -37,6 +41,7 @@ impl CapabilityStore { pub fn new() -> Self { Self { epoch: AtomicU64::new(0), + epoch_writer: Mutex::new(()), use_counts: RwLock::new(HashMap::new()), revoked_tokens: RwLock::new(HashSet::new()), storage_path: None, @@ -52,6 +57,7 @@ impl CapabilityStore { let store = Self { epoch: AtomicU64::new(0), + epoch_writer: Mutex::new(()), use_counts: RwLock::new(HashMap::new()), revoked_tokens: RwLock::new(HashSet::new()), storage_path: Some(path.clone()), @@ -76,35 +82,57 @@ impl CapabilityStore { /// the next startup loads the higher epoch and tokens stay revoked. /// Returns the new epoch value. pub fn advance_epoch(&self) -> u64 { - let new_epoch = self.epoch.load(Ordering::SeqCst) + 1; - - // Persist FIRST — crash-safe ordering - if let Some(path) = &self.storage_path { - let epoch_path = path.join("epoch"); - let tmp_path = path.join("epoch.tmp"); - if let Err(e) = fs::write(&tmp_path, new_epoch.to_string()) - .and_then(|_| fs::rename(&tmp_path, &epoch_path)) - { - tracing::error!( - "CRITICAL: Failed to persist epoch to {}: {}. Epoch advance aborted.", - epoch_path.display(), - e - ); - // Do NOT advance in-memory if persistence failed — - // better to fail the revocation than to have in-memory - // and on-disk diverge. - return self.epoch.load(Ordering::SeqCst); + match self.epoch_transaction( + |current| { + current + .checked_add(1) + .ok_or_else(|| std::io::Error::other("capability epoch exhausted")) + }, + |epoch| self.write_epoch(epoch), + ) { + Ok(epoch) => epoch, + Err(error) => { + tracing::error!("CRITICAL: Capability epoch advance aborted: {}", error); + self.current_epoch() } } - - // Only update in-memory after successful persistence - self.epoch.store(new_epoch, Ordering::SeqCst); - new_epoch } - /// Set the epoch to a specific value (for initialization/loading) + /// Load an already-persisted epoch during initialization. + /// + /// A lower value cannot restore previously revoked authority. The only + /// Runtime caller is construction-time loading; use `advance_epoch` for + /// revocation, which persists the new value before publishing it. pub fn set_epoch(&self, epoch: u64) { - self.epoch.store(epoch, Ordering::SeqCst); + self.epoch_transaction(|current| Ok(current.max(epoch)), |_| Ok(())) + .expect("initializing an epoch cannot fail"); + } + + /// All epoch writers share this synchronous transaction. The update and + /// writer run under one lock; memory is published only after writer success. + fn epoch_transaction( + &self, + update: impl FnOnce(u64) -> std::io::Result, + writer: impl FnOnce(u64) -> std::io::Result<()>, + ) -> std::io::Result { + let _epoch_writer = self + .epoch_writer + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + let next = update(self.current_epoch())?; + writer(next)?; + self.epoch.store(next, Ordering::SeqCst); + Ok(next) + } + + fn write_epoch(&self, epoch: u64) -> std::io::Result<()> { + if let Some(path) = &self.storage_path { + let epoch_path = path.join("epoch"); + let tmp_path = path.join("epoch.tmp"); + fs::write(&tmp_path, epoch.to_string())?; + fs::rename(&tmp_path, &epoch_path)?; + } + Ok(()) } /// Check if a token's epoch is valid @@ -179,10 +207,9 @@ impl CapabilityStore { /// Persist state to disk pub async fn persist(&self) -> std::io::Result<()> { - if let Some(path) = &self.storage_path { - // Persist epoch - let epoch_path = path.join("epoch"); - fs::write(epoch_path, self.current_epoch().to_string())?; + if self.storage_path.is_some() { + // The synchronous transaction releases its guard before the await. + self.epoch_transaction(Ok, |epoch| self.write_epoch(epoch))?; // Persist revoked tokens self.persist_revoked_tokens().await; @@ -257,6 +284,163 @@ impl Default for CapabilityStore { mod tests { use super::*; + fn assert_epoch_writer_held(store: &CapabilityStore) { + // This is a deterministic lock-coverage probe, not a timing assertion. + let blocked = std::thread::scope(|scope| { + scope + .spawn(|| { + matches!( + store.epoch_writer.try_lock(), + Err(std::sync::TryLockError::WouldBlock) + ) + }) + .join() + .unwrap() + }); + assert!(blocked, "another epoch writer could enter the transaction"); + } + + #[tokio::test] + async fn test_epoch_transaction_serializes_snapshot_write_and_publication() { + let directory = tempfile::tempdir().unwrap(); + let store = CapabilityStore::with_persistence(directory.path()) + .await + .unwrap(); + for advance in [true, false] { + let before = store.current_epoch(); + let result = store + .epoch_transaction( + |current| { + assert_epoch_writer_held(&store); + assert_eq!(current, before); + Ok(if advance { current + 1 } else { current }) + }, + |next| { + assert_epoch_writer_held(&store); + assert_eq!(store.current_epoch(), before); + store.write_epoch(next)?; + assert_eq!( + fs::read_to_string(directory.path().join("epoch"))?, + next.to_string() + ); + assert_eq!( + store.current_epoch(), + before, + "publish only after writer success" + ); + Ok(()) + }, + ) + .unwrap(); + assert_eq!(store.current_epoch(), result); + assert!(store.epoch_writer.try_lock().is_ok()); + } + } + + #[tokio::test] + async fn test_concurrent_epoch_advance_and_snapshot_preserve_persisted_epoch() { + use std::sync::{Arc, Barrier}; + const WRITERS: usize = 8; + const ADVANCES: usize = 8; + let directory = tempfile::tempdir().unwrap(); + let store = Arc::new( + CapabilityStore::with_persistence(directory.path()) + .await + .unwrap(), + ); + let barrier = Arc::new(Barrier::new(WRITERS + 2)); + let mut handles = Vec::new(); + for _ in 0..WRITERS { + let store = Arc::clone(&store); + let barrier = Arc::clone(&barrier); + handles.push(std::thread::spawn(move || { + barrier.wait(); + (0..ADVANCES) + .map(|_| store.advance_epoch()) + .collect::>() + })); + } + let snapshot_store = Arc::clone(&store); + let snapshot_barrier = Arc::clone(&barrier); + let snapshots = std::thread::spawn(move || { + let runtime = tokio::runtime::Builder::new_current_thread() + .build() + .unwrap(); + snapshot_barrier.wait(); + for _ in 0..WRITERS * ADVANCES { + runtime.block_on(snapshot_store.persist()).unwrap(); + } + }); + barrier.wait(); + let mut epochs = handles + .into_iter() + .flat_map(|handle| handle.join().unwrap()) + .collect::>(); + snapshots.join().unwrap(); + epochs.sort_unstable(); + let final_epoch = (WRITERS * ADVANCES) as u64; + assert_eq!(epochs, (1..=final_epoch).collect::>()); + assert_eq!(store.current_epoch(), final_epoch); + assert_eq!( + fs::read_to_string(directory.path().join("epoch")).unwrap(), + final_epoch.to_string() + ); + let reloaded = CapabilityStore::with_persistence(directory.path()) + .await + .unwrap(); + assert_eq!(reloaded.current_epoch(), final_epoch); + } + + #[tokio::test] + async fn test_epoch_write_and_rename_failures_do_not_publish_memory() { + fn assert_send(_: T) {} + for blocked_path in ["epoch.tmp", "epoch"] { + let directory = tempfile::tempdir().unwrap(); + let store = CapabilityStore::with_persistence(directory.path()) + .await + .unwrap(); + fs::create_dir(directory.path().join(blocked_path)).unwrap(); + assert_eq!(store.advance_epoch(), 0); + assert_eq!(store.current_epoch(), 0); + // A synchronous writer guard must not make this async API !Send. + assert_send(store.persist()); + assert!(store.persist().await.is_err()); + assert_eq!(store.current_epoch(), 0); + fs::remove_dir(directory.path().join(blocked_path)).unwrap(); + assert_eq!(store.advance_epoch(), 1); + let reloaded = CapabilityStore::with_persistence(directory.path()) + .await + .unwrap(); + assert_eq!(reloaded.current_epoch(), 1); + } + } + + #[tokio::test] + async fn test_epoch_initialization_is_monotonic_and_overflow_stays_unchanged() { + let memory = CapabilityStore::new(); + memory.set_epoch(7); + memory.set_epoch(3); + assert_eq!(memory.current_epoch(), 7); + assert!(!memory.is_epoch_valid(6)); + + let directory = tempfile::tempdir().unwrap(); + fs::write(directory.path().join("epoch"), u64::MAX.to_string()).unwrap(); + let store = CapabilityStore::with_persistence(directory.path()) + .await + .unwrap(); + assert_eq!(store.advance_epoch(), u64::MAX); + assert_eq!(store.current_epoch(), u64::MAX); + assert_eq!( + fs::read_to_string(directory.path().join("epoch")).unwrap(), + u64::MAX.to_string() + ); + store.persist().await.unwrap(); + let reloaded = CapabilityStore::with_persistence(directory.path()) + .await + .unwrap(); + assert_eq!(reloaded.current_epoch(), u64::MAX); + } + #[test] fn test_epoch() { let store = CapabilityStore::new(); From a66409aa5fb7a279bdd7a6802c88997e68ebcb5b Mon Sep 17 00:00:00 2001 From: ELA CITY Date: Sat, 17 Oct 2026 06:56:09 +0000 Subject: [PATCH 03/11] fix(runtime): reclaim spent and revoked grant records at capacity --- .../elastos-runtime/src/capability/manager.rs | 17 + .../elastos-runtime/src/capability/pending.rs | 252 ++++++++++++- .../src/api/gateway_browser_operator.rs | 4 + .../src/api/handlers/capability.rs | 330 ++++++++++++++++++ .../elastos-server/src/resource_bridge.rs | 3 + 5 files changed, 599 insertions(+), 7 deletions(-) diff --git a/elastos/crates/elastos-runtime/src/capability/manager.rs b/elastos/crates/elastos-runtime/src/capability/manager.rs index c221c02d1..0bc26a9ef 100644 --- a/elastos/crates/elastos-runtime/src/capability/manager.rs +++ b/elastos/crates/elastos-runtime/src/capability/manager.rs @@ -345,6 +345,23 @@ impl CapabilityManager { token } + /// Whether an issued token is revoked or has no uses remaining. + /// + /// This is only a bookkeeping query for already-granted tokens: it neither + /// validates authority nor consumes a use. Callers must still use `validate` + /// before exercising a capability. + pub(crate) async fn is_revoked_or_exhausted(&self, token: &CapabilityToken) -> bool { + if !self.store.is_epoch_valid(token.constraints.epoch) + || self.store.is_token_revoked(&token.id).await + { + return true; + } + match token.constraints.max_uses { + Some(max_uses) => self.store.has_exceeded_uses(&token.id, max_uses).await, + None => false, + } + } + /// Validate a token for use /// /// This performs all 12 validation checks as specified in PHASE3.md. diff --git a/elastos/crates/elastos-runtime/src/capability/pending.rs b/elastos/crates/elastos-runtime/src/capability/pending.rs index 79d496fe3..7394f88bc 100644 --- a/elastos/crates/elastos-runtime/src/capability/pending.rs +++ b/elastos/crates/elastos-runtime/src/capability/pending.rs @@ -8,6 +8,7 @@ use tokio::sync::RwLock; use serde::{Deserialize, Serialize}; +use super::manager::CapabilityManager; use super::token::{Action, CapabilityToken, ResourceId}; use crate::primitives::audit::AuditLog; use crate::primitives::time::SecureTimestamp; @@ -216,8 +217,8 @@ impl PendingRequestState { fn cleanup_old(&mut self, now: &SecureTimestamp, retention_secs: u64) -> usize { let previous_len = self.requests.len(); self.requests.retain(|_, request| { - // Request expiry alone says nothing about token validity. Only an - // explicit token expiry or revocation makes a grant reclaimable. + // Request expiry alone says nothing about token validity. This pass + // checks token expiry; authoritative retirement needs the manager below. let resolved = match &request.status { RequestStatus::Pending => false, RequestStatus::Granted { token, .. } => { @@ -234,14 +235,18 @@ impl PendingRequestState { let removed = previous_len - self.requests.len(); if removed != 0 { - let requests = &self.requests; - self.session_requests.retain(|_, ids| { - ids.retain(|id| requests.contains_key(id)); - !ids.is_empty() - }); + self.prune_session_index(); } removed } + + fn prune_session_index(&mut self) { + let requests = &self.requests; + self.session_requests.retain(|_, ids| { + ids.retain(|id| requests.contains_key(id)); + !ids.is_empty() + }); + } } /// Store for pending capability requests @@ -474,6 +479,44 @@ impl PendingRequestStore { state.cleanup_old(&SecureTimestamp::now(), retention_secs) } + /// Reclaim authoritatively revoked or exhausted grants when the store is full. + /// + /// The Runtime owner must supply the manager that issued and validates this + /// store's grants. Revocation and remaining uses come from that manager's + /// authoritative state, never the requested duration or session visibility. + /// This does not consume, revoke or authorize a use. + /// Admission still rechecks both limits under the same state lock afterward. + pub async fn cleanup_retired_grants_at_capacity( + &self, + capability_manager: &CapabilityManager, + ) -> usize { + if self.state.read().await.requests.len() < MAX_PENDING_REQUESTS { + return 0; + } + let mut state = self.state.write().await; + if state.requests.len() < MAX_PENDING_REQUESTS { + return 0; + } + + let mut retired = Vec::new(); + for (id, request) in &state.requests { + if let RequestStatus::Granted { token, .. } = &request.status { + // The query reads only the manager's epoch/revocation/use state. + // Those guards are released before callers access pending state. + if capability_manager.is_revoked_or_exhausted(token).await { + retired.push(id.clone()); + } + } + } + for id in &retired { + state.requests.remove(id); + } + if !retired.is_empty() { + state.prune_session_index(); + } + retired.len() + } + /// Get the number of pending requests pub async fn pending_count(&self) -> usize { let state = self.state.read().await; @@ -786,6 +829,201 @@ mod tests { assert_eq!(store.state.read().await.requests.len(), 1); } + #[tokio::test] + async fn test_pressure_reclaims_only_authoritatively_exhausted_grants() { + use crate::capability::{CapabilityStore, TokenConstraints}; + use crate::primitives::metrics::MetricsManager; + + let audit = Arc::new(AuditLog::new()); + let counts = Arc::new(CapabilityStore::new()); + let manager = CapabilityManager::new( + counts.clone(), + audit.clone(), + Arc::new(MetricsManager::new()), + ); + let store = PendingRequestStore::new(audit); + let mut grants = Vec::new(); + for (session, max_uses, used) in [ + ("exhausted-only", Some(1), 1), + ("mixed", Some(1), 1), + ("mixed", Some(1), 0), + ("mixed", None, 0), + ("mixed", Some(2), 1), + ] { + let request = request_for(&store, session).await; + let token = manager.grant( + session, + request.resource.clone(), + request.action, + TokenConstraints::new(0, false, None, max_uses), + None, + ); + store + .grant_request(request.id.as_str(), token.clone(), GrantDuration::Once) + .await + .unwrap(); + for _ in 0..used { + manager + .validate(&token, session, request.action, &request.resource, None) + .await + .unwrap(); + } + grants.push((request, token, used)); + } + assert_eq!(store.cleanup_retired_grants_at_capacity(&manager).await, 0); + for index in 0..MAX_PENDING_REQUESTS - grants.len() { + assert!(request_for(&store, &format!("prefill-{index}")) + .await + .is_pending()); + } + + assert_eq!(store.cleanup_retired_grants_at_capacity(&manager).await, 2); + for (index, (request, token, used)) in grants.iter().enumerate() { + assert_eq!( + store.get_request(request.id.as_str()).await.is_some(), + index >= 2 + ); + assert_eq!(counts.get_use_count(token.id()).await, *used); + } + let state = store.state.read().await; + assert!(!state.session_requests.contains_key("exhausted-only")); + assert_eq!(state.session_requests["mixed"].len(), 3); + assert_eq!(state.requests.len(), MAX_PENDING_REQUESTS - 2); + assert_eq!( + state.session_requests.values().map(Vec::len).sum::(), + state.requests.len() + ); + drop(state); + + assert!(request_for(&store, "recovered").await.is_pending()); + assert!(request_for(&store, "recovered").await.is_pending()); + assert_eq!(store.cleanup_retired_grants_at_capacity(&manager).await, 0); + assert!(request_for(&store, "overflow").await.is_denied()); + } + + #[tokio::test] + async fn test_pressure_reclaims_revoked_grants_without_changing_authority_state() { + use crate::capability::{manager::ValidationError, CapabilityStore, TokenConstraints}; + use crate::primitives::metrics::MetricsManager; + + let audit = Arc::new(AuditLog::new()); + let authority = Arc::new(CapabilityStore::new()); + let manager = CapabilityManager::new( + authority.clone(), + audit.clone(), + Arc::new(MetricsManager::new()), + ); + let store = PendingRequestStore::new(audit); + let stale = request_for(&store, "stale-epoch").await; + let stale_token = manager.grant( + "stale-epoch", + stale.resource.clone(), + stale.action, + TokenConstraints::default(), + None, + ); + let epoch = manager.revoke_all("retire old epoch"); + store.revoke_all_granted().await; + // A grant minted before revocation can finish registering afterward. + store + .grant_request( + stale.id.as_str(), + stale_token.clone(), + GrantDuration::Session, + ) + .await + .unwrap(); + + let revoked = request_for(&store, "mixed").await; + let revoked_token = manager.grant( + "mixed", + revoked.resource.clone(), + revoked.action, + TokenConstraints::new(epoch, false, None, Some(2)), + None, + ); + store + .grant_request( + revoked.id.as_str(), + revoked_token.clone(), + GrantDuration::Session, + ) + .await + .unwrap(); + manager + .validate( + &revoked_token, + "mixed", + revoked.action, + &revoked.resource, + None, + ) + .await + .unwrap(); + manager + .revoke(*revoked_token.id(), "retire individual token") + .await; + + let live = request_for(&store, "mixed").await; + let live_token = manager.grant( + "mixed", + live.resource.clone(), + live.action, + TokenConstraints::default(), + None, + ); + store + .grant_request(live.id.as_str(), live_token.clone(), GrantDuration::Session) + .await + .unwrap(); + for index in 0..MAX_PENDING_REQUESTS - 3 { + assert!(request_for(&store, &format!("prefill-{index}")) + .await + .is_pending()); + } + let revoked_before = authority.get_revoked_tokens().await; + + assert_eq!(store.cleanup_retired_grants_at_capacity(&manager).await, 2); + assert_eq!(authority.current_epoch(), epoch); + assert_eq!(authority.get_revoked_tokens().await, revoked_before); + assert!(authority.is_token_revoked(revoked_token.id()).await); + assert!(!authority.is_token_revoked(stale_token.id()).await); + assert_eq!(authority.get_use_count(revoked_token.id()).await, 1); + for (token, session, resource) in [ + (&stale_token, "stale-epoch", &stale.resource), + (&revoked_token, "mixed", &revoked.resource), + ] { + assert!(matches!( + manager + .validate(token, session, Action::Read, resource, None) + .await, + Err(ValidationError::TokenRevoked) + )); + } + assert_eq!(authority.get_use_count(revoked_token.id()).await, 1); + manager + .validate(&live_token, "mixed", live.action, &live.resource, None) + .await + .unwrap(); + assert_eq!(store.list_session_granted("mixed").await[0].id, live.id); + let state = store.state.read().await; + assert!(!state.requests.contains_key(stale.id.as_str())); + assert!(!state.requests.contains_key(revoked.id.as_str())); + assert!(!state.session_requests.contains_key("stale-epoch")); + assert_eq!(state.session_requests["mixed"].len(), 1); + assert_eq!(state.requests.len(), MAX_PENDING_REQUESTS - 2); + assert_eq!( + state.session_requests.values().map(Vec::len).sum::(), + state.requests.len() + ); + drop(state); + + assert!(request_for(&store, "recovered").await.is_pending()); + assert!(request_for(&store, "recovered").await.is_pending()); + assert_eq!(store.cleanup_retired_grants_at_capacity(&manager).await, 0); + assert!(request_for(&store, "overflow").await.is_denied()); + } + #[tokio::test] async fn test_cleanup_reclaims_only_explicitly_expired_grants() { let store = create_test_store(); diff --git a/elastos/crates/elastos-server/src/api/gateway_browser_operator.rs b/elastos/crates/elastos-server/src/api/gateway_browser_operator.rs index f3eed6edc..d45a98ebd 100644 --- a/elastos/crates/elastos-server/src/api/gateway_browser_operator.rs +++ b/elastos/crates/elastos-server/src/api/gateway_browser_operator.rs @@ -332,6 +332,10 @@ pub(in crate::api::gateway) async fn request_admission( return failure(StatusCode::TOO_MANY_REQUESTS, "operator_requests_busy"); } service.pending.cleanup_old(60).await; + service + .pending + .cleanup_retired_grants_at_capacity(&service.capabilities) + .await; let pending = service .pending .create_request_with_reason( diff --git a/elastos/crates/elastos-server/src/api/handlers/capability.rs b/elastos/crates/elastos-server/src/api/handlers/capability.rs index cd5038467..618201d7c 100755 --- a/elastos/crates/elastos-server/src/api/handlers/capability.rs +++ b/elastos/crates/elastos-server/src/api/handlers/capability.rs @@ -119,6 +119,10 @@ pub async fn request_capability( // For now, all requests go to pending (no auto-grant policy yet) // Future: check if session already has this capability, or if policy allows auto-grant + state + .pending_store + .cleanup_retired_grants_at_capacity(&state.capability_manager) + .await; let request = state .pending_store .create_request_with_reason(session.id.clone(), resource, action, input.reason) @@ -776,6 +780,332 @@ mod tests { } } + #[tokio::test] + async fn test_request_capacity_recovers_consumed_once_grants_without_losing_live_grants() { + use elastos_runtime::capability::{ + manager::ValidationError, pending::MAX_PENDING_REQUESTS, CapabilityToken, + }; + use elastos_runtime::session::SessionId; + + async fn request(state: &CapabilityState, session: &Session) -> RequestCapabilityOutput { + request_capability( + State(state.clone()), + Extension(session.clone()), + Json(RequestCapabilityInput { + resource: "localhost://Users/self/Documents/test".to_string(), + action: "read".to_string(), + reason: "capacity regression".to_string(), + }), + ) + .await + .unwrap() + .0 + } + + async fn grant( + state: &CapabilityState, + session: &Session, + duration: &str, + ) -> (String, CapabilityToken) { + let output = request(state, session).await; + assert_eq!(output.status, "pending"); + let request_id = output.request_id.unwrap(); + let granted = grant_request( + State(state.clone()), + Extension(Session::new_shell("test-shell".to_string())), + Json(GrantRequestInput { + request_id: request_id.clone(), + duration: duration.to_string(), + rationale: Some("capacity regression".to_string()), + }), + ) + .await + .unwrap() + .0; + assert!(granted.success); + let token = CapabilityToken::from_base64(&granted.token.unwrap()).unwrap(); + (request_id, token) + } + + let state = test_state(); + let spent_session = Session::new_capsule("spent".to_string()); + let live_session = Session::new_capsule("live".to_string()); + let next_session = Session::new_capsule("next".to_string()); + let (spent_id, spent) = grant(&state, &spent_session, "once").await; + let (unused_id, unused) = grant(&state, &live_session, "once").await; + let (live_id, live) = grant(&state, &live_session, "session").await; + assert_eq!(spent.constraints().max_uses(), Some(1)); + assert!(spent.expiry().is_none()); + for index in 0..MAX_PENDING_REQUESTS - 3 { + let pending = state + .pending_store + .create_request( + SessionId::from_string(format!("prefill-{index}")), + spent.resource().clone(), + Action::Read, + ) + .await; + assert!(pending.is_pending()); + } + assert_eq!(request(&state, &next_session).await.status, "denied"); + + state + .capability_manager + .validate( + &spent, + spent_session.id.as_str(), + Action::Read, + spent.resource(), + None, + ) + .await + .unwrap(); + let recovered = request(&state, &next_session).await; + assert_eq!( + recovered.status, "pending", + "consumed grants must not permanently fill the store" + ); + assert!(state + .pending_store + .get_request(&recovered.request_id.unwrap()) + .await + .is_some()); + assert!(state.pending_store.get_request(&spent_id).await.is_none()); + assert!(state + .pending_store + .list_session_granted(spent_session.id.as_str()) + .await + .is_empty()); + assert_eq!( + state.pending_store.pending_count().await, + MAX_PENDING_REQUESTS - 2 + ); + assert_eq!(request(&state, &next_session).await.status, "denied"); + + let visible = list_capabilities(State(state.clone()), Extension(live_session.clone())) + .await + .0; + assert_eq!(visible.capabilities.len(), 2); + assert!(visible + .capabilities + .iter() + .any(|entry| entry.request_id == unused_id)); + assert!(visible + .capabilities + .iter() + .any(|entry| entry.request_id == live_id)); + // Pressure reconciliation must not spend an unused one-use allowance. + state + .capability_manager + .validate( + &unused, + live_session.id.as_str(), + Action::Read, + unused.resource(), + None, + ) + .await + .unwrap(); + assert!(matches!( + state + .capability_manager + .validate( + &unused, + live_session.id.as_str(), + Action::Read, + unused.resource(), + None + ) + .await, + Err(ValidationError::UseLimitExceeded { current: 1, max: 1 }) + )); + let revoked = revoke_capability( + State(state.clone()), + Extension(Session::new_shell("test-shell".to_string())), + Path(live_id), + ) + .await + .unwrap() + .0; + assert!(revoked.success); + assert!(matches!( + state + .capability_manager + .validate( + &live, + live_session.id.as_str(), + Action::Read, + live.resource(), + None + ) + .await, + Err(ValidationError::TokenRevoked) + )); + } + + #[tokio::test] + async fn test_request_capacity_recovers_late_registered_revoked_grant() { + use elastos_runtime::capability::{ + manager::ValidationError, pending::MAX_PENDING_REQUESTS, CapabilityToken, + }; + use elastos_runtime::session::SessionId; + + let state = test_state(); + let stale_session = Session::new_capsule("stale-epoch".to_string()); + let live_session = Session::new_capsule("live-epoch".to_string()); + let shell = Session::new_shell("test-shell".to_string()); + let resource = ResourceId::new("localhost://Users/self/Documents/test"); + let stale_request = state + .pending_store + .create_request(stale_session.id.clone(), resource.clone(), Action::Read) + .await; + // Reproduce the grant handler's mint/register split without timing sleeps. + let stale_token = state.capability_manager.grant( + stale_session.id.as_str(), + resource.clone(), + Action::Read, + TokenConstraints::default(), + None, + ); + let revoked = revoke_all_capabilities( + State(state.clone()), + Extension(shell.clone()), + Json(RevokeAllInput { + reason: "capacity regression".to_string(), + }), + ) + .await + .0; + assert!(state + .pending_store + .get_request(stale_request.id.as_str()) + .await + .unwrap() + .is_pending()); + state + .pending_store + .grant_request( + stale_request.id.as_str(), + stale_token.clone(), + GrantDuration::Session, + ) + .await + .unwrap(); + assert!(matches!( + state + .capability_manager + .validate( + &stale_token, + stale_session.id.as_str(), + Action::Read, + &resource, + None + ) + .await, + Err(ValidationError::TokenRevoked) + )); + + let live_request = state + .pending_store + .create_request(live_session.id.clone(), resource.clone(), Action::Read) + .await; + let live_grant = grant_request( + State(state.clone()), + Extension(shell), + Json(GrantRequestInput { + request_id: live_request.id.to_string(), + duration: "session".to_string(), + rationale: Some("capacity regression".to_string()), + }), + ) + .await + .unwrap() + .0; + let live_token = CapabilityToken::from_base64(&live_grant.token.unwrap()).unwrap(); + assert_eq!(live_token.constraints().epoch(), revoked.new_epoch); + assert_eq!(live_token.constraints().max_uses(), None); + assert!(live_token.expiry().is_none()); + for index in 0..MAX_PENDING_REQUESTS - 2 { + assert!(state + .pending_store + .create_request( + SessionId::from_string(format!("prefill-{index}")), + resource.clone(), + Action::Read + ) + .await + .is_pending()); + } + + let recovered = request_capability( + State(state.clone()), + Extension(Session::new_capsule("next".to_string())), + Json(RequestCapabilityInput { + resource: resource.to_string(), + action: "read".to_string(), + reason: "capacity regression".to_string(), + }), + ) + .await + .unwrap() + .0; + assert_eq!( + recovered.status, "pending", + "revoked grants must not permanently fill the store" + ); + assert!(state + .pending_store + .get_request(stale_request.id.as_str()) + .await + .is_none()); + assert_eq!( + state.pending_store.pending_count().await, + MAX_PENDING_REQUESTS - 1 + ); + assert!(state + .pending_store + .create_request( + SessionId::from_string("overflow"), + resource.clone(), + Action::Read + ) + .await + .is_denied()); + assert_eq!(state.capability_manager.current_epoch(), revoked.new_epoch); + assert!(matches!( + state + .capability_manager + .validate( + &stale_token, + stale_session.id.as_str(), + Action::Read, + &resource, + None + ) + .await, + Err(ValidationError::TokenRevoked) + )); + state + .capability_manager + .validate( + &live_token, + live_session.id.as_str(), + Action::Read, + &resource, + None, + ) + .await + .unwrap(); + let visible = list_capabilities(State(state), Extension(live_session)) + .await + .0; + assert_eq!(visible.capabilities.len(), 1); + assert_eq!( + visible.capabilities[0].request_id, + live_request.id.to_string() + ); + } + #[tokio::test] async fn test_request_capability_denies_ipfs_backend() { let output = request_capability( diff --git a/elastos/crates/elastos-server/src/resource_bridge.rs b/elastos/crates/elastos-server/src/resource_bridge.rs index c91eeb127..84eed0b25 100644 --- a/elastos/crates/elastos-server/src/resource_bridge.rs +++ b/elastos/crates/elastos-server/src/resource_bridge.rs @@ -840,6 +840,9 @@ async fn handle_request(line: &str, ctx: &Option) -> Result Date: Sat, 17 Oct 2026 14:21:47 +0000 Subject: [PATCH 04/11] Keep capability admission fair and report failed revocation Co-authored-by: Codex --- .../elastos-runtime/src/capability/manager.rs | 300 ++++++++++++-- .../elastos-runtime/src/capability/pending.rs | 382 +++++++++++++++--- .../elastos-runtime/src/capability/store.rs | 26 +- .../tests/http_api_integration.rs | 98 +++-- .../src/api/gateway_browser_operator.rs | 7 +- .../gateway_tests/collaboration_presence.rs | 9 +- .../src/api/handlers/capability.rs | 190 ++++++++- .../src/collaboration_history.rs | 12 +- .../src/collaboration_presence_grant.rs | 2 +- .../src/collaboration_transport.rs | 2 +- .../elastos-server/src/resource_bridge.rs | 6 +- .../crates/elastos-server/src/security_cmd.rs | 2 +- .../crates/elastos-server/src/server_infra.rs | 2 +- 13 files changed, 868 insertions(+), 170 deletions(-) diff --git a/elastos/crates/elastos-runtime/src/capability/manager.rs b/elastos/crates/elastos-runtime/src/capability/manager.rs index 0738f14f1..f45ba4e33 100644 --- a/elastos/crates/elastos-runtime/src/capability/manager.rs +++ b/elastos/crates/elastos-runtime/src/capability/manager.rs @@ -607,37 +607,65 @@ impl CapabilityManager { /// Revoke all tokens by advancing the epoch /// /// All tokens with epoch < new_epoch will be rejected. - /// Returns the new epoch. - pub fn revoke_all(&self, reason: &str) -> u64 { - let old_epoch = self.store.current_epoch(); - let new_epoch = self.store.advance_epoch(); - self.audit_log.epoch_advance(old_epoch, new_epoch, reason); - new_epoch + /// Returns the persisted epoch, or the write/rename/overflow error. + pub fn revoke_all(&self, reason: &str) -> std::io::Result { + let new_epoch = self.store.advance_epoch()?; + self.audit_log + .epoch_advance(new_epoch - 1, new_epoch, reason); + Ok(new_epoch) } - /// Rotate the signing key: generate a new key, advance the epoch, persist. + /// Rotate the signing key after durable epoch revocation. /// - /// All previously issued tokens become invalid (epoch revocation). - /// Returns the new public key bytes. - pub fn rotate_signing_key(&mut self, data_dir: &Path, reason: &str) -> [u8; 32] { + /// Epoch failure preserves the disk and memory key. If key publication + /// fails after revocation, the epoch stays advanced and the error reports + /// that partial result. This is not a multi-file atomic transaction. + pub fn rotate_signing_key( + &mut self, + data_dir: &Path, + reason: &str, + ) -> std::io::Result<[u8; 32]> { + self.rotate_signing_key_with_publish(data_dir, reason, |staged, key_path| { + staged + .persist(key_path) + .map(|_| ()) + .map_err(|error| error.error) + }) + } + + fn rotate_signing_key_with_publish( + &mut self, + data_dir: &Path, + reason: &str, + publish: impl FnOnce(tempfile::NamedTempFile, &Path) -> std::io::Result<()>, + ) -> std::io::Result<[u8; 32]> { + use std::io::Write as _; + let new_key = SigningKey::generate(&mut rand::thread_rng()); let new_pub = new_key.verifying_key(); - - // Persist new key with restrictive permissions let key_path = data_dir.join("signing_key"); - if let Err(e) = std::fs::write(&key_path, new_key.to_bytes()) { - tracing::error!("Failed to persist rotated key: {}", e); - } else { - #[cfg(unix)] - { - use std::os::unix::fs::PermissionsExt; - let _ = std::fs::set_permissions(&key_path, std::fs::Permissions::from_mode(0o600)); - } + let mut staged = tempfile::Builder::new() + .prefix(".signing-key-rotation-") + .tempfile_in(data_dir)?; + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + staged + .as_file() + .set_permissions(std::fs::Permissions::from_mode(0o600))?; } - - // Advance epoch to invalidate all existing tokens - let old_epoch = self.store.current_epoch(); - let new_epoch = self.store.advance_epoch(); + staged.write_all(&new_key.to_bytes())?; + staged.as_file().sync_all()?; + + // Until this succeeds, the live key file and in-memory key stay intact. + // NamedTempFile removes the staged key on either error path. + let new_epoch = self.revoke_all(reason)?; + let old_epoch = new_epoch - 1; + publish(staged, &key_path).map_err(|error| { + std::io::Error::new(error.kind(), format!( + "Capabilities were revoked at epoch {new_epoch}, but signing key publication failed: {error}" + )) + })?; self.audit_log.security_warning( "signing_key_rotated", @@ -650,8 +678,7 @@ impl CapabilityManager { let pub_bytes = new_pub.to_bytes(); self.signing_key = new_key; self.verifying_key = new_pub; - - pub_bytes + Ok(pub_bytes) } /// Get the current revocation epoch @@ -764,6 +791,221 @@ mod tests { assert!(matches!(result, Err(ValidationError::WrongAction { .. }))); } + #[tokio::test] + async fn test_revoke_all_propagates_persistence_failure_without_success_audit() { + let directory = tempfile::tempdir().unwrap(); + let store = Arc::new( + CapabilityStore::with_persistence(directory.path()) + .await + .unwrap(), + ); + let audit = Arc::new(AuditLog::new()); + let manager = CapabilityManager::new( + store.clone(), + audit.clone(), + Arc::new(MetricsManager::new()), + ); + let resource = ResourceId::new("localhost://Users/self/Documents/test"); + let token = manager.grant( + "owner", + resource.clone(), + Action::Read, + TokenConstraints::default(), + None, + ); + std::fs::create_dir(directory.path().join("epoch.tmp")).unwrap(); + assert!(manager.revoke_all("blocked write").is_err()); + assert_eq!(store.current_epoch(), 0); + assert!(audit + .recent_events_filtered(100, Some("epoch_advance")) + .is_empty()); + manager + .validate(&token, "owner", Action::Read, &resource, None) + .await + .unwrap(); + } + + #[tokio::test] + async fn test_rotation_epoch_failure_preserves_persisted_key_and_authority() { + for blocked_path in ["epoch.tmp", "epoch", "overflow"] { + let directory = tempfile::tempdir().unwrap(); + if blocked_path == "overflow" { + std::fs::write(directory.path().join("epoch"), u64::MAX.to_string()).unwrap(); + } + let store = Arc::new( + CapabilityStore::with_persistence(directory.path()) + .await + .unwrap(), + ); + let audit = Arc::new(AuditLog::new()); + let mut manager = CapabilityManager::load_or_generate( + directory.path(), + store.clone(), + audit.clone(), + Arc::new(MetricsManager::new()), + ); + let original_key = manager.public_key_bytes(); + let original_bytes = std::fs::read(directory.path().join("signing_key")).unwrap(); + let original_epoch = store.current_epoch(); + let resource = ResourceId::new("localhost://Users/self/Documents/test"); + let token = manager.grant( + "owner", + resource.clone(), + Action::Read, + TokenConstraints::default(), + None, + ); + if blocked_path != "overflow" { + std::fs::create_dir(directory.path().join(blocked_path)).unwrap(); + } + let audit_before = audit.recent_events(100).len(); + assert!(manager + .rotate_signing_key(directory.path(), "blocked epoch") + .is_err()); + assert_eq!(manager.public_key_bytes(), original_key); + assert_eq!(store.current_epoch(), original_epoch); + assert_eq!( + std::fs::read(directory.path().join("signing_key")).unwrap(), + original_bytes + ); + assert_eq!(audit.recent_events(100).len(), audit_before); + assert!(std::fs::read_dir(directory.path()).unwrap().all(|entry| { + !entry + .unwrap() + .file_name() + .to_string_lossy() + .starts_with(".signing-key-rotation-") + })); + if blocked_path != "overflow" { + std::fs::remove_dir(directory.path().join(blocked_path)).unwrap(); + } + let reloaded_store = Arc::new( + CapabilityStore::with_persistence(directory.path()) + .await + .unwrap(), + ); + let reloaded = CapabilityManager::load_or_generate( + directory.path(), + reloaded_store, + Arc::new(AuditLog::new()), + Arc::new(MetricsManager::new()), + ); + assert_eq!(reloaded.public_key_bytes(), original_key); + assert_eq!(reloaded.current_epoch(), original_epoch); + reloaded + .validate(&token, "owner", Action::Read, &resource, None) + .await + .unwrap(); + } + } + + #[tokio::test] + async fn test_rotation_publish_failure_keeps_key_and_reports_completed_revocation() { + let directory = tempfile::tempdir().unwrap(); + let store = Arc::new( + CapabilityStore::with_persistence(directory.path()) + .await + .unwrap(), + ); + let audit = Arc::new(AuditLog::new()); + let mut manager = CapabilityManager::load_or_generate( + directory.path(), + store.clone(), + audit.clone(), + Arc::new(MetricsManager::new()), + ); + let original_key = manager.public_key_bytes(); + let original_bytes = std::fs::read(directory.path().join("signing_key")).unwrap(); + let resource = ResourceId::new("localhost://Users/self/Documents/test"); + let token = manager.grant( + "owner", + resource.clone(), + Action::Read, + TokenConstraints::default(), + None, + ); + let error = manager + .rotate_signing_key_with_publish( + directory.path(), + "blocked rename", + |staged, key_path| { + assert_eq!(store.current_epoch(), 1, "revocation precedes publication"); + assert_eq!( + std::fs::read_to_string(directory.path().join("epoch")).unwrap(), + "1" + ); + assert_eq!(std::fs::read(key_path).unwrap(), original_bytes); + assert_eq!(staged.as_file().metadata().unwrap().len(), 32); + #[cfg(unix)] + { + use std::os::unix::fs::PermissionsExt; + assert_eq!( + staged.as_file().metadata().unwrap().permissions().mode() & 0o777, + 0o600 + ); + } + Err(std::io::Error::other("injected rename failure")) + }, + ) + .unwrap_err(); + assert!(error.to_string().contains("revoked at epoch 1")); + assert!(error.to_string().contains("publication failed")); + assert_eq!(manager.public_key_bytes(), original_key); + assert_eq!( + std::fs::read(directory.path().join("signing_key")).unwrap(), + original_bytes + ); + assert_eq!( + audit + .recent_events_filtered(100, Some("epoch_advance")) + .len(), + 1 + ); + assert!(std::fs::read_dir(directory.path()).unwrap().all(|entry| { + !entry + .unwrap() + .file_name() + .to_string_lossy() + .starts_with(".signing-key-rotation-") + })); + let reloaded_store = Arc::new( + CapabilityStore::with_persistence(directory.path()) + .await + .unwrap(), + ); + let mut reloaded = CapabilityManager::load_or_generate( + directory.path(), + reloaded_store, + Arc::new(AuditLog::new()), + Arc::new(MetricsManager::new()), + ); + assert_eq!(reloaded.public_key_bytes(), original_key); + assert_eq!(reloaded.current_epoch(), 1); + assert!(matches!( + reloaded + .validate(&token, "owner", Action::Read, &resource, None) + .await, + Err(ValidationError::TokenRevoked) + )); + let replacement = reloaded + .rotate_signing_key(directory.path(), "retry publication") + .unwrap(); + assert_ne!(replacement, original_key); + assert_eq!(reloaded.current_epoch(), 2); + let persisted = CapabilityManager::load_or_generate( + directory.path(), + Arc::new( + CapabilityStore::with_persistence(directory.path()) + .await + .unwrap(), + ), + Arc::new(AuditLog::new()), + Arc::new(MetricsManager::new()), + ); + assert_eq!(persisted.public_key_bytes(), replacement); + assert_eq!(persisted.current_epoch(), 2); + } + #[tokio::test] async fn test_epoch_revocation() { let manager = create_test_manager(); @@ -777,7 +1019,7 @@ mod tests { ); // Advance epoch (mass revocation) - manager.revoke_all("test revocation"); + manager.revoke_all("test revocation").unwrap(); // Token should now be invalid let result = manager @@ -880,7 +1122,7 @@ mod tests { .validate_recurring(&fresh, "principal", Action::Message, &resource, None, false) .await .is_ok()); - manager.revoke_all("epoch"); + manager.revoke_all("epoch").unwrap(); assert!(matches!( manager .validate_recurring(&fresh, "principal", Action::Message, &resource, None, false) @@ -1418,7 +1660,7 @@ mod tests { } // Advance epoch to invalidate all tokens - manager.revoke_all("epoch advance test"); + manager.revoke_all("epoch advance test").unwrap(); let mut successes = 0; let mut revoked = 0; diff --git a/elastos/crates/elastos-runtime/src/capability/pending.rs b/elastos/crates/elastos-runtime/src/capability/pending.rs index 7394f88bc..1981370ab 100644 --- a/elastos/crates/elastos-runtime/src/capability/pending.rs +++ b/elastos/crates/elastos-runtime/src/capability/pending.rs @@ -20,8 +20,8 @@ pub const DEFAULT_REQUEST_TIMEOUT_SECS: u64 = 300; /// Maximum number of pending requests before rejecting new ones pub const MAX_PENDING_REQUESTS: usize = 1024; -/// Maximum number of pending requests per session before rejecting new ones. -/// Prevents a single session from starving other sessions. +/// Maximum outstanding requests and retained live grants per session. +/// A session keeps its live grant records within its own bounded share. pub const MAX_PENDING_PER_SESSION: usize = 32; /// Unique identifier for a pending request @@ -202,6 +202,22 @@ struct PendingRequestState { } impl PendingRequestState { + fn session_admission_count(&self, session_id: &str, now: &SecureTimestamp) -> usize { + self.session_requests.get(session_id).map_or(0, |ids| { + ids.iter() + .filter(|id| { + self.requests + .get(*id) + .is_some_and(|request| match &request.status { + RequestStatus::Pending => !request.expires_at.is_before(now), + RequestStatus::Granted { .. } => true, + RequestStatus::Denied { .. } | RequestStatus::Expired => false, + }) + }) + .count() + }) + } + fn cleanup_expired(&mut self, now: &SecureTimestamp) -> usize { let mut expired = 0; for request in self.requests.values_mut() { @@ -284,12 +300,27 @@ impl PendingRequestStore { } } - /// Create a new pending request + /// Admit a request through Runtime-owned pressure reconciliation. /// - /// If the store is at capacity (MAX_PENDING_REQUESTS), expired requests are - /// cleaned up first. If still at capacity, the returned request is immediately - /// denied (not stored) to prevent request-flood DoS. - pub async fn create_request( + /// The manager must be the Runtime owner that issued this store's grants. + /// Reconciliation never consumes or grants authority. The final capacity + /// decision and insertion share one write lock after reconciliation. + pub async fn admit_request( + &self, + capability_manager: &CapabilityManager, + session_id: SessionId, + resource: ResourceId, + action: Action, + reason: String, + ) -> PendingCapabilityRequest { + self.cleanup_retired_grants_at_capacity(capability_manager, Some(session_id.as_str())) + .await; + self.create_request_with_reason(session_id, resource, action, reason) + .await + } + + #[cfg(test)] + async fn create_request( &self, session_id: SessionId, resource: ResourceId, @@ -299,7 +330,7 @@ impl PendingRequestStore { .await } - pub async fn create_request_with_reason( + async fn create_request_with_reason( &self, session_id: SessionId, resource: ResourceId, @@ -309,7 +340,9 @@ impl PendingRequestStore { // Admission, cleanup, and both index updates are one state transition. let mut state = self.state.write().await; let now = SecureTimestamp::now(); - if state.requests.len() >= MAX_PENDING_REQUESTS { + if state.requests.len() >= MAX_PENDING_REQUESTS + || state.session_admission_count(session_id.as_str(), &now) >= MAX_PENDING_PER_SESSION + { state.cleanup_expired(&now); state.cleanup_old(&now, 0); } @@ -317,18 +350,8 @@ impl PendingRequestStore { let denial = if state.requests.len() >= MAX_PENDING_REQUESTS { Some("Too many pending requests") } else { - let pending_count = state.session_requests.get(&session_id.0).map_or(0, |ids| { - ids.iter() - .filter(|id| { - state.requests.get(*id).is_some_and(|request| { - matches!(request.status, RequestStatus::Pending) - && !request.expires_at.is_before(&now) - }) - }) - .count() - }); - (pending_count >= MAX_PENDING_PER_SESSION) - .then_some("Too many pending requests for this session") + (state.session_admission_count(session_id.as_str(), &now) >= MAX_PENDING_PER_SESSION) + .then_some("Too many outstanding requests or grants for this session") }; let mut request = PendingCapabilityRequest::new_with_reason( @@ -345,6 +368,10 @@ impl PendingRequestStore { return request; } + // Exercise the decision/insertion boundary in race tests. The writer + // remains held across this yield, so contenders cannot share a slot. + #[cfg(test)] + tokio::task::yield_now().await; state.requests.insert(request.id.0.clone(), request.clone()); state .session_requests @@ -479,42 +506,79 @@ impl PendingRequestStore { state.cleanup_old(&SecureTimestamp::now(), retention_secs) } - /// Reclaim authoritatively revoked or exhausted grants when the store is full. - /// - /// The Runtime owner must supply the manager that issued and validates this - /// store's grants. Revocation and remaining uses come from that manager's - /// authoritative state, never the requested duration or session visibility. - /// This does not consume, revoke or authorize a use. - /// Admission still rechecks both limits under the same state lock afterward. - pub async fn cleanup_retired_grants_at_capacity( + /// Reclaim retired grants under global or requesting-session pressure. + async fn cleanup_retired_grants_at_capacity( &self, capability_manager: &CapabilityManager, + session_id: Option<&str>, ) -> usize { - if self.state.read().await.requests.len() < MAX_PENDING_REQUESTS { - return 0; - } - let mut state = self.state.write().await; - if state.requests.len() < MAX_PENDING_REQUESTS { - return 0; - } + self.reclaim_retired_grants_with(session_id, |token| async move { + capability_manager.is_revoked_or_exhausted(&token).await + }) + .await + } + async fn reclaim_retired_grants_with( + &self, + session_id: Option<&str>, + mut is_retired: F, + ) -> usize + where + F: FnMut(CapabilityToken) -> Fut, + Fut: std::future::Future, + { + let candidates = { + let state = self.state.read().await; + let global_pressure = state.requests.len() >= MAX_PENDING_REQUESTS; + let session_pressure = session_id.is_some_and(|session_id| { + state.session_admission_count(session_id, &SecureTimestamp::now()) + >= MAX_PENDING_PER_SESSION + }); + if !global_pressure && !session_pressure { + return 0; + } + state + .requests + .iter() + .filter(|(_, request)| { + global_pressure || Some(request.session_id.as_str()) == session_id + }) + .filter_map(|(id, request)| match &request.status { + RequestStatus::Granted { token, duration } => { + Some((id.clone(), token.as_ref().clone(), *duration)) + } + _ => None, + }) + .collect::>() + }; + + // Manager queries can await without holding any pending-state guard. let mut retired = Vec::new(); - for (id, request) in &state.requests { - if let RequestStatus::Granted { token, .. } = &request.status { - // The query reads only the manager's epoch/revocation/use state. - // Those guards are released before callers access pending state. - if capability_manager.is_revoked_or_exhausted(token).await { - retired.push(id.clone()); - } + for (id, token, duration) in candidates { + let token_id = *token.id(); + if is_retired(token).await { + retired.push((id, token_id, duration)); } } - for id in &retired { - state.requests.remove(id); + + let mut state = self.state.write().await; + let mut removed = 0; + for (id, token_id, duration) in retired { + // A granted record's fields are immutable until its status changes. + // Recheck status, token identity and duration before removing it. + let unchanged = state.requests.get(&id).is_some_and(|request| { + matches!(&request.status, RequestStatus::Granted { token, duration: current_duration } + if *token.id() == token_id && *current_duration == duration) + }); + if unchanged { + state.requests.remove(&id); + removed += 1; + } } - if !retired.is_empty() { + if removed != 0 { state.prune_session_index(); } - retired.len() + removed } /// Get the number of pending requests @@ -802,9 +866,9 @@ mod tests { } #[tokio::test] - async fn test_live_grants_fill_capacity_until_explicit_revocation() { + async fn test_retained_live_grants_fill_only_the_owning_session_share() { let store = create_test_store(); - for _ in 0..MAX_PENDING_REQUESTS { + for _ in 0..MAX_PENDING_PER_SESSION { let request = request_for(&store, "grant-session").await; assert!(request.is_pending()); store @@ -816,17 +880,192 @@ mod tests { .await .unwrap(); } - assert!(request_for(&store, "new-session").await.is_denied()); + assert!(request_for(&store, "grant-session").await.is_denied()); + assert!(request_for(&store, "new-session").await.is_pending()); assert_eq!( store.list_session_granted("grant-session").await.len(), - MAX_PENDING_REQUESTS + MAX_PENDING_PER_SESSION ); assert_eq!(store.cleanup_old(0).await, 0); store.revoke_all_granted().await; - assert!(request_for(&store, "new-session").await.is_pending()); + assert!(request_for(&store, "grant-session").await.is_pending()); assert!(store.list_session_granted("grant-session").await.is_empty()); - assert_eq!(store.state.read().await.requests.len(), 1); + } + + #[tokio::test] + async fn test_admission_reclaims_session_share_without_spending_live_grants() { + use crate::capability::{CapabilityStore, TokenConstraints}; + use crate::primitives::metrics::MetricsManager; + + for revoke in [false, true] { + let audit = Arc::new(AuditLog::new()); + let authority = Arc::new(CapabilityStore::new()); + let manager = CapabilityManager::new( + authority.clone(), + audit.clone(), + Arc::new(MetricsManager::new()), + ); + let store = PendingRequestStore::new(audit); + let mut grants = Vec::new(); + for _ in 0..MAX_PENDING_PER_SESSION { + let request = store + .admit_request( + &manager, + SessionId::from_string("owner"), + ResourceId::new("localhost://Users/self/Documents/test"), + Action::Read, + "bounded share".to_string(), + ) + .await; + assert!(request.is_pending()); + let token = manager.grant( + "owner", + request.resource.clone(), + request.action, + TokenConstraints::new(0, false, None, Some(1)), + None, + ); + store + .grant_request(request.id.as_str(), token.clone(), GrantDuration::Once) + .await + .unwrap(); + grants.push((request, token)); + } + let admit = |session| { + store.admit_request( + &manager, + SessionId::from_string(session), + ResourceId::new("localhost://Users/self/Documents/test"), + Action::Read, + String::new(), + ) + }; + assert!(admit("owner").await.is_denied()); + assert!(admit("another-session").await.is_pending()); + assert_eq!( + store.list_session_granted("owner").await.len(), + MAX_PENDING_PER_SESSION + ); + let (retired, token) = &grants[0]; + if revoke { + manager.revoke(*token.id(), "share recovery").await; + } else { + manager + .validate(token, "owner", Action::Read, &retired.resource, None) + .await + .unwrap(); + } + assert!(admit("owner").await.is_pending()); + assert!(store.get_request(retired.id.as_str()).await.is_none()); + assert_eq!( + store.list_session_granted("owner").await.len(), + MAX_PENDING_PER_SESSION - 1 + ); + assert_eq!( + authority.get_use_count(token.id()).await, + u32::from(!revoke) + ); + for (request, live_token) in &grants[1..] { + assert!(store + .get_request(request.id.as_str()) + .await + .unwrap() + .is_granted()); + assert_eq!(authority.get_use_count(live_token.id()).await, 0); + } + assert_eq!( + store.state.read().await.requests.len(), + MAX_PENDING_PER_SESSION + 1 + ); + } + } + + #[tokio::test] + async fn test_reclaim_queries_without_pending_lock_and_preserves_changed_records() { + let store = create_test_store(); + let mut granted = Vec::new(); + for _ in 0..4 { + let request = request_for(&store, "owner").await; + let token = token_for(&request); + store + .grant_request(request.id.as_str(), token.clone(), GrantDuration::Session) + .await + .unwrap(); + granted.push((request, token)); + } + for _ in 4..MAX_PENDING_PER_SESSION { + assert!(request_for(&store, "owner").await.is_pending()); + } + let removed = store + .reclaim_retired_grants_with(Some("owner"), |token| { + let store = &store; + let granted = &granted; + async move { + // This fails immediately if either a read or write guard escaped + // the snapshot scope. It does not rely on a timeout or scheduler. + let mut state = store + .state + .try_write() + .expect("manager query holds pending lock"); + let index = granted + .iter() + .position(|(_, old)| old.id() == token.id()) + .unwrap(); + let request = state + .requests + .get_mut(granted[index].0.id.as_str()) + .unwrap(); + match index { + 0 => {} + 1 => { + request.status = RequestStatus::Denied { + reason: "changed during query".into(), + } + } + 2 => { + request.status = RequestStatus::Granted { + token: Box::new(token_for(request)), + duration: GrantDuration::Session, + } + } + 3 => { + request.status = RequestStatus::Granted { + token: Box::new(token), + duration: GrantDuration::Once, + } + } + _ => unreachable!(), + } + drop(state); + tokio::task::yield_now().await; + true + } + }) + .await; + assert_eq!(removed, 1); + assert!(store.get_request(granted[0].0.id.as_str()).await.is_none()); + assert!(store + .get_request(granted[1].0.id.as_str()) + .await + .unwrap() + .is_denied()); + let replacement = store.get_request(granted[2].0.id.as_str()).await.unwrap(); + assert_ne!(replacement.granted_token().unwrap().id(), granted[2].1.id()); + assert!(matches!( + store + .get_request(granted[3].0.id.as_str()) + .await + .unwrap() + .status, + RequestStatus::Granted { + duration: GrantDuration::Once, + .. + } + )); + let state = store.state.read().await; + assert_eq!(state.requests.len(), MAX_PENDING_PER_SESSION - 1); + assert_eq!(state.session_requests["owner"].len(), state.requests.len()); } #[tokio::test] @@ -870,14 +1109,24 @@ mod tests { } grants.push((request, token, used)); } - assert_eq!(store.cleanup_retired_grants_at_capacity(&manager).await, 0); + assert_eq!( + store + .cleanup_retired_grants_at_capacity(&manager, None) + .await, + 0 + ); for index in 0..MAX_PENDING_REQUESTS - grants.len() { assert!(request_for(&store, &format!("prefill-{index}")) .await .is_pending()); } - assert_eq!(store.cleanup_retired_grants_at_capacity(&manager).await, 2); + assert_eq!( + store + .cleanup_retired_grants_at_capacity(&manager, None) + .await, + 2 + ); for (index, (request, token, used)) in grants.iter().enumerate() { assert_eq!( store.get_request(request.id.as_str()).await.is_some(), @@ -897,7 +1146,12 @@ mod tests { assert!(request_for(&store, "recovered").await.is_pending()); assert!(request_for(&store, "recovered").await.is_pending()); - assert_eq!(store.cleanup_retired_grants_at_capacity(&manager).await, 0); + assert_eq!( + store + .cleanup_retired_grants_at_capacity(&manager, None) + .await, + 0 + ); assert!(request_for(&store, "overflow").await.is_denied()); } @@ -922,7 +1176,7 @@ mod tests { TokenConstraints::default(), None, ); - let epoch = manager.revoke_all("retire old epoch"); + let epoch = manager.revoke_all("retire old epoch").unwrap(); store.revoke_all_granted().await; // A grant minted before revocation can finish registering afterward. store @@ -983,7 +1237,12 @@ mod tests { } let revoked_before = authority.get_revoked_tokens().await; - assert_eq!(store.cleanup_retired_grants_at_capacity(&manager).await, 2); + assert_eq!( + store + .cleanup_retired_grants_at_capacity(&manager, None) + .await, + 2 + ); assert_eq!(authority.current_epoch(), epoch); assert_eq!(authority.get_revoked_tokens().await, revoked_before); assert!(authority.is_token_revoked(revoked_token.id()).await); @@ -1020,7 +1279,12 @@ mod tests { assert!(request_for(&store, "recovered").await.is_pending()); assert!(request_for(&store, "recovered").await.is_pending()); - assert_eq!(store.cleanup_retired_grants_at_capacity(&manager).await, 0); + assert_eq!( + store + .cleanup_retired_grants_at_capacity(&manager, None) + .await, + 0 + ); assert!(request_for(&store, "overflow").await.is_denied()); } diff --git a/elastos/crates/elastos-runtime/src/capability/store.rs b/elastos/crates/elastos-runtime/src/capability/store.rs index 5fb2761a7..a178e3d08 100644 --- a/elastos/crates/elastos-runtime/src/capability/store.rs +++ b/elastos/crates/elastos-runtime/src/capability/store.rs @@ -81,21 +81,15 @@ impl CapabilityStore { /// if the process dies after persist but before in-memory update, /// the next startup loads the higher epoch and tokens stay revoked. /// Returns the new epoch value. - pub fn advance_epoch(&self) -> u64 { - match self.epoch_transaction( + pub fn advance_epoch(&self) -> std::io::Result { + self.epoch_transaction( |current| { current .checked_add(1) .ok_or_else(|| std::io::Error::other("capability epoch exhausted")) }, |epoch| self.write_epoch(epoch), - ) { - Ok(epoch) => epoch, - Err(error) => { - tracing::error!("CRITICAL: Capability epoch advance aborted: {}", error); - self.current_epoch() - } - } + ) } /// Load an already-persisted epoch during initialization. @@ -356,7 +350,7 @@ mod tests { handles.push(std::thread::spawn(move || { barrier.wait(); (0..ADVANCES) - .map(|_| store.advance_epoch()) + .map(|_| store.advance_epoch().unwrap()) .collect::>() })); } @@ -400,14 +394,14 @@ mod tests { .await .unwrap(); fs::create_dir(directory.path().join(blocked_path)).unwrap(); - assert_eq!(store.advance_epoch(), 0); + assert!(store.advance_epoch().is_err()); assert_eq!(store.current_epoch(), 0); // A synchronous writer guard must not make this async API !Send. assert_send(store.persist()); assert!(store.persist().await.is_err()); assert_eq!(store.current_epoch(), 0); fs::remove_dir(directory.path().join(blocked_path)).unwrap(); - assert_eq!(store.advance_epoch(), 1); + assert_eq!(store.advance_epoch().unwrap(), 1); let reloaded = CapabilityStore::with_persistence(directory.path()) .await .unwrap(); @@ -428,7 +422,7 @@ mod tests { let store = CapabilityStore::with_persistence(directory.path()) .await .unwrap(); - assert_eq!(store.advance_epoch(), u64::MAX); + assert!(store.advance_epoch().is_err()); assert_eq!(store.current_epoch(), u64::MAX); assert_eq!( fs::read_to_string(directory.path().join("epoch")).unwrap(), @@ -447,7 +441,7 @@ mod tests { assert_eq!(store.current_epoch(), 0); - let new_epoch = store.advance_epoch(); + let new_epoch = store.advance_epoch().unwrap(); assert_eq!(new_epoch, 1); assert_eq!(store.current_epoch(), 1); @@ -492,8 +486,8 @@ mod tests { let store = CapabilityStore::with_persistence(temp_dir.path()) .await .unwrap(); - store.advance_epoch(); - store.advance_epoch(); + store.advance_epoch().unwrap(); + store.advance_epoch().unwrap(); let token_id = TokenId::from_bytes([1u8; 16]); store.revoke_token(token_id).await; diff --git a/elastos/crates/elastos-runtime/tests/http_api_integration.rs b/elastos/crates/elastos-runtime/tests/http_api_integration.rs index 8e2d07c2a..b4608a45d 100755 --- a/elastos/crates/elastos-runtime/tests/http_api_integration.rs +++ b/elastos/crates/elastos-runtime/tests/http_api_integration.rs @@ -110,17 +110,19 @@ async fn test_invalidate_session() { #[tokio::test] async fn test_create_capability_request() { - let (session_registry, _, pending_store) = create_test_infra(); + let (session_registry, capability_manager, pending_store) = create_test_infra(); let session = session_registry .create_session(SessionType::Capsule, None) .await; let request = pending_store - .create_request( + .admit_request( + &capability_manager, session.id.clone(), ResourceId::new("localhost://Users/self/Documents/photos/*"), Action::Read, + String::new(), ) .await; @@ -135,7 +137,7 @@ async fn test_create_capability_request() { #[tokio::test] async fn test_list_pending_requests() { - let (session_registry, _, pending_store) = create_test_infra(); + let (session_registry, capability_manager, pending_store) = create_test_infra(); // Create two sessions with requests let session1 = session_registry @@ -146,18 +148,22 @@ async fn test_list_pending_requests() { .await; pending_store - .create_request( + .admit_request( + &capability_manager, session1.id.clone(), ResourceId::new("localhost://Users/self/Documents/photos/*"), Action::Read, + String::new(), ) .await; pending_store - .create_request( + .admit_request( + &capability_manager, session2.id.clone(), ResourceId::new("localhost://Users/self/Documents/documents/*"), Action::Write, + String::new(), ) .await; @@ -174,10 +180,12 @@ async fn test_grant_capability_request() { .await; let request = pending_store - .create_request( + .admit_request( + &capability_manager, session.id.clone(), ResourceId::new("localhost://Users/self/Documents/photos/*"), Action::Read, + String::new(), ) .await; @@ -217,17 +225,19 @@ async fn test_grant_capability_request() { #[tokio::test] async fn test_deny_capability_request() { - let (session_registry, _, pending_store) = create_test_infra(); + let (session_registry, capability_manager, pending_store) = create_test_infra(); let session = session_registry .create_session(SessionType::Capsule, None) .await; let request = pending_store - .create_request( + .admit_request( + &capability_manager, session.id.clone(), ResourceId::new("localhost://Users/self/Documents/sensitive/*"), Action::Delete, + String::new(), ) .await; @@ -290,10 +300,12 @@ async fn test_cannot_grant_already_granted_request() { .await; let request = pending_store - .create_request( + .admit_request( + &capability_manager, session.id.clone(), ResourceId::new("localhost://Users/self/Documents/photos/*"), Action::Read, + String::new(), ) .await; @@ -324,17 +336,19 @@ async fn test_cannot_grant_already_granted_request() { #[tokio::test] async fn test_cannot_deny_already_denied_request() { - let (session_registry, _, pending_store) = create_test_infra(); + let (session_registry, capability_manager, pending_store) = create_test_infra(); let session = session_registry .create_session(SessionType::Capsule, None) .await; let request = pending_store - .create_request( + .admit_request( + &capability_manager, session.id.clone(), ResourceId::new("localhost://Users/self/Documents/photos/*"), Action::Read, + String::new(), ) .await; @@ -362,10 +376,12 @@ async fn test_grant_once_duration() { .await; let request = pending_store - .create_request( + .admit_request( + &capability_manager, session.id.clone(), ResourceId::new("localhost://Users/self/Documents/photos/*"), Action::Read, + String::new(), ) .await; @@ -398,17 +414,19 @@ async fn test_grant_once_duration() { #[tokio::test] async fn test_request_expiry_detection() { - let (session_registry, _, pending_store) = create_test_infra(); + let (session_registry, capability_manager, pending_store) = create_test_infra(); let session = session_registry .create_session(SessionType::Capsule, None) .await; let request = pending_store - .create_request( + .admit_request( + &capability_manager, session.id.clone(), ResourceId::new("localhost://Users/self/Documents/photos/*"), Action::Read, + String::new(), ) .await; @@ -420,7 +438,7 @@ async fn test_request_expiry_detection() { #[tokio::test] async fn test_all_action_types() { - let (session_registry, _, pending_store) = create_test_infra(); + let (session_registry, capability_manager, pending_store) = create_test_infra(); let session = session_registry .create_session(SessionType::Capsule, None) @@ -437,10 +455,12 @@ async fn test_all_action_types() { for action in actions { let request = pending_store - .create_request( + .admit_request( + &capability_manager, session.id.clone(), ResourceId::new("localhost://Users/self/Documents/test/*"), action, + String::new(), ) .await; @@ -519,7 +539,7 @@ async fn test_cleanup_dead_vm_sessions() { #[tokio::test] async fn test_multiple_requests_same_session() { - let (session_registry, _, pending_store) = create_test_infra(); + let (session_registry, capability_manager, pending_store) = create_test_infra(); let session = session_registry .create_session(SessionType::Capsule, None) @@ -527,18 +547,22 @@ async fn test_multiple_requests_same_session() { // Create multiple requests from same session let request1 = pending_store - .create_request( + .admit_request( + &capability_manager, session.id.clone(), ResourceId::new("localhost://Users/self/Documents/photos/*"), Action::Read, + String::new(), ) .await; let request2 = pending_store - .create_request( + .admit_request( + &capability_manager, session.id.clone(), ResourceId::new("localhost://Users/self/Documents/documents/*"), Action::Write, + String::new(), ) .await; @@ -568,10 +592,12 @@ async fn test_full_capability_flow() { // 3. Capsule requests capability let request = pending_store - .create_request( + .admit_request( + &capability_manager, capsule.id.clone(), ResourceId::new("localhost://Users/self/Documents/photos/*"), Action::Read, + String::new(), ) .await; @@ -908,10 +934,12 @@ async fn test_session_info_capabilities_count() { // Create and grant first capability let req1 = pending_store - .create_request( + .admit_request( + &capability_manager, session.id.clone(), ResourceId::new("localhost://Users/self/Documents/photos/*"), Action::Read, + String::new(), ) .await; @@ -937,10 +965,12 @@ async fn test_session_info_capabilities_count() { // Create and grant second capability let req2 = pending_store - .create_request( + .admit_request( + &capability_manager, session.id.clone(), ResourceId::new("localhost://Users/self/Documents/docs/*"), Action::Write, + String::new(), ) .await; @@ -966,10 +996,12 @@ async fn test_session_info_capabilities_count() { // Pending request should NOT be counted pending_store - .create_request( + .admit_request( + &capability_manager, session.id.clone(), ResourceId::new("localhost://Users/self/Documents/music/*"), Action::Read, + String::new(), ) .await; @@ -985,10 +1017,12 @@ async fn test_session_info_capabilities_count() { .create_session(SessionType::Capsule, None) .await; let other_req = pending_store - .create_request( + .admit_request( + &capability_manager, other.id.clone(), ResourceId::new("localhost://Users/self/Documents/other/*"), Action::Read, + String::new(), ) .await; let other_token = capability_manager.grant( @@ -1060,10 +1094,12 @@ async fn test_shadow_divergence_visible_in_audit_log() { .await; let request = pending_store - .create_request( + .admit_request( + &capability_manager, capsule.id.clone(), ResourceId::new("localhost://Users/self/Documents/sensitive/*"), Action::Delete, + String::new(), ) .await; @@ -1129,10 +1165,12 @@ async fn test_no_shadow_no_divergence_events() { let evaluator = PolicyEvaluator::new(Box::new(ShellPassthroughVerifier), audit_log.clone()); let request = pending_store - .create_request( + .admit_request( + &capability_manager, elastos_runtime::session::SessionId::from_string("test-session"), ResourceId::new("localhost://Users/self/Documents/photos/*"), Action::Read, + String::new(), ) .await; @@ -1177,10 +1215,12 @@ async fn test_shadow_agreement_no_divergence() { ); let request = pending_store - .create_request( + .admit_request( + &capability_manager, elastos_runtime::session::SessionId::from_string("test-session"), ResourceId::new("localhost://Users/self/Documents/photos/*"), Action::Read, + String::new(), ) .await; @@ -1211,10 +1251,12 @@ async fn test_rules_shadow_diverges_on_blocked_resource() { // Request targets a blocklisted resource segment ("admin") let request = pending_store - .create_request( + .admit_request( + &capability_manager, elastos_runtime::session::SessionId::from_string("test-session"), ResourceId::new("localhost://Users/self/Documents/admin/secrets"), Action::Read, + String::new(), ) .await; diff --git a/elastos/crates/elastos-server/src/api/gateway_browser_operator.rs b/elastos/crates/elastos-server/src/api/gateway_browser_operator.rs index d45a98ebd..e9ac76f10 100644 --- a/elastos/crates/elastos-server/src/api/gateway_browser_operator.rs +++ b/elastos/crates/elastos-server/src/api/gateway_browser_operator.rs @@ -332,13 +332,10 @@ pub(in crate::api::gateway) async fn request_admission( return failure(StatusCode::TOO_MANY_REQUESTS, "operator_requests_busy"); } service.pending.cleanup_old(60).await; - service - .pending - .cleanup_retired_grants_at_capacity(&service.capabilities) - .await; let pending = service .pending - .create_request_with_reason( + .admit_request( + &service.capabilities, session.id.clone(), ResourceId::new(format!("elastos://browser/pages/{page_id}/input")), Action::Write, diff --git a/elastos/crates/elastos-server/src/api/gateway_tests/collaboration_presence.rs b/elastos/crates/elastos-server/src/api/gateway_tests/collaboration_presence.rs index 8657031c7..3708c590e 100644 --- a/elastos/crates/elastos-server/src/api/gateway_tests/collaboration_presence.rs +++ b/elastos/crates/elastos-server/src/api/gateway_tests/collaboration_presence.rs @@ -762,7 +762,8 @@ async fn heartbeat_is_strict_proof_bound_and_coalesces_exact_presentation() { .presence_authority() .unwrap() .manager() - .revoke_all("test epoch"); + .revoke_all("test epoch") + .unwrap(); assert_eq!( response_json(heartbeat().await.unwrap()).await["queued"], false @@ -1034,7 +1035,8 @@ async fn people_snapshot_filters_local_and_exposes_only_signed_live_remote_prese .presence_authority() .unwrap() .manager() - .revoke_all("read-only snapshot epoch"); + .revoke_all("read-only snapshot epoch") + .unwrap(); } let state_before = file_snapshot(local_dir.path()); let snapshot = local_presence.authorized_snapshot(now).await.unwrap(); @@ -1239,7 +1241,8 @@ async fn restarted_presence_get_keeps_legacy_core_bytes_until_lifecycle_repair() .presence_authority() .unwrap() .manager() - .revoke_all("restart GET denial"); + .revoke_all("restart GET denial") + .unwrap(); state.collaboration_presence_product_port = Some(reopened.clone()); let before = file_snapshot(dir.path()); assert!(reopened diff --git a/elastos/crates/elastos-server/src/api/handlers/capability.rs b/elastos/crates/elastos-server/src/api/handlers/capability.rs index 618201d7c..b647df5e4 100755 --- a/elastos/crates/elastos-server/src/api/handlers/capability.rs +++ b/elastos/crates/elastos-server/src/api/handlers/capability.rs @@ -118,23 +118,24 @@ pub async fn request_capability( // For now, all requests go to pending (no auto-grant policy yet) // Future: check if session already has this capability, or if policy allows auto-grant - - state - .pending_store - .cleanup_retired_grants_at_capacity(&state.capability_manager) - .await; let request = state .pending_store - .create_request_with_reason(session.id.clone(), resource, action, input.reason) + .admit_request( + &state.capability_manager, + session.id.clone(), + resource, + action, + input.reason, + ) .await; // If pre-denied (e.g. rate limit), surface the denial immediately - if request.is_denied() { + if let elastos_runtime::capability::RequestStatus::Denied { reason } = &request.status { return Ok(Json(RequestCapabilityOutput { status: "denied".to_string(), request_id: Some(request.id.to_string()), token: None, - reason: Some("Too many pending requests".to_string()), + reason: Some(reason.clone()), })); } @@ -542,17 +543,26 @@ pub async fn revoke_all_capabilities( State(state): State, Extension(_session): Extension, // Shell check done by middleware Json(input): Json, -) -> Json { - let new_epoch = state.capability_manager.revoke_all(&input.reason); +) -> Result, (StatusCode, String)> { + let new_epoch = state + .capability_manager + .revoke_all(&input.reason) + .map_err(|error| { + tracing::error!("Failed to persist capability revocation: {error}"); + ( + StatusCode::INTERNAL_SERVER_ERROR, + "Failed to persist capability revocation".to_string(), + ) + })?; // Mark all granted requests as revoked state.pending_store.revoke_all_granted().await; - Json(RevokeAllOutput { + Ok(Json(RevokeAllOutput { success: true, new_epoch, reason: input.reason, - }) + })) } // === Session Info === @@ -780,6 +790,129 @@ mod tests { } } + #[tokio::test] + async fn test_revoke_all_failure_returns_500_and_keeps_live_grants() { + use elastos_runtime::capability::{CapabilityStore, RequestStatus}; + use elastos_runtime::primitives::{audit::AuditEvent, metrics::MetricsManager}; + + for blocked_path in ["epoch.tmp", "epoch", "overflow"] { + let directory = tempfile::tempdir().unwrap(); + if blocked_path == "overflow" { + std::fs::write(directory.path().join("epoch"), u64::MAX.to_string()).unwrap(); + } + let authority = Arc::new( + CapabilityStore::with_persistence(directory.path()) + .await + .unwrap(), + ); + let mut state = test_state(); + let audit = state.pending_store.audit_log(); + state.capability_manager = Arc::new(CapabilityManager::new( + authority.clone(), + audit.clone(), + Arc::new(MetricsManager::new()), + )); + let session = Session::new_capsule("revocation-failure".to_string()); + let request = state + .pending_store + .admit_request( + &state.capability_manager, + session.id.clone(), + ResourceId::new("localhost://Users/self/Documents/test"), + Action::Read, + String::new(), + ) + .await; + let token = state.capability_manager.grant( + session.id.as_str(), + request.resource.clone(), + request.action, + TokenConstraints::default(), + None, + ); + state + .pending_store + .grant_request(request.id.as_str(), token.clone(), GrantDuration::Session) + .await + .unwrap(); + if blocked_path != "overflow" { + std::fs::create_dir(directory.path().join(blocked_path)).unwrap(); + } + let epoch = authority.current_epoch(); + let result = revoke_all_capabilities( + State(state.clone()), + Extension(Session::new_shell("test-shell".to_string())), + Json(RevokeAllInput { + reason: "persistence failure".to_string(), + }), + ) + .await; + let (status, message) = match result { + Err(error) => error, + Ok(_) => panic!("failed persistence must not report successful revocation"), + }; + assert_eq!(status, StatusCode::INTERNAL_SERVER_ERROR); + assert_eq!(message, "Failed to persist capability revocation"); + assert_eq!(authority.current_epoch(), epoch); + assert!(matches!( + state + .pending_store + .get_request(request.id.as_str()) + .await + .unwrap() + .status, + RequestStatus::Granted { .. } + )); + assert!(audit + .recent_events(100) + .iter() + .all(|event| !matches!(event, AuditEvent::EpochAdvance { .. }))); + state + .capability_manager + .validate( + &token, + session.id.as_str(), + request.action, + &request.resource, + None, + ) + .await + .unwrap(); + if blocked_path != "overflow" { + std::fs::remove_dir(directory.path().join(blocked_path)).unwrap(); + let recovered = revoke_all_capabilities( + State(state.clone()), + Extension(Session::new_shell("test-shell".to_string())), + Json(RevokeAllInput { + reason: "retry after repair".to_string(), + }), + ) + .await + .unwrap() + .0; + assert!(recovered.success); + assert_eq!(recovered.new_epoch, epoch + 1); + assert!(state + .pending_store + .get_request(request.id.as_str()) + .await + .unwrap() + .is_denied()); + assert!(state + .capability_manager + .validate( + &token, + session.id.as_str(), + request.action, + &request.resource, + None + ) + .await + .is_err()); + } + } + } + #[tokio::test] async fn test_request_capacity_recovers_consumed_once_grants_without_losing_live_grants() { use elastos_runtime::capability::{ @@ -839,10 +972,12 @@ mod tests { for index in 0..MAX_PENDING_REQUESTS - 3 { let pending = state .pending_store - .create_request( + .admit_request( + &state.capability_manager, SessionId::from_string(format!("prefill-{index}")), spent.resource().clone(), Action::Read, + String::new(), ) .await; assert!(pending.is_pending()); @@ -957,7 +1092,13 @@ mod tests { let resource = ResourceId::new("localhost://Users/self/Documents/test"); let stale_request = state .pending_store - .create_request(stale_session.id.clone(), resource.clone(), Action::Read) + .admit_request( + &state.capability_manager, + stale_session.id.clone(), + resource.clone(), + Action::Read, + String::new(), + ) .await; // Reproduce the grant handler's mint/register split without timing sleeps. let stale_token = state.capability_manager.grant( @@ -975,6 +1116,7 @@ mod tests { }), ) .await + .unwrap() .0; assert!(state .pending_store @@ -1007,7 +1149,13 @@ mod tests { let live_request = state .pending_store - .create_request(live_session.id.clone(), resource.clone(), Action::Read) + .admit_request( + &state.capability_manager, + live_session.id.clone(), + resource.clone(), + Action::Read, + String::new(), + ) .await; let live_grant = grant_request( State(state.clone()), @@ -1028,10 +1176,12 @@ mod tests { for index in 0..MAX_PENDING_REQUESTS - 2 { assert!(state .pending_store - .create_request( + .admit_request( + &state.capability_manager, SessionId::from_string(format!("prefill-{index}")), resource.clone(), - Action::Read + Action::Read, + String::new() ) .await .is_pending()); @@ -1064,10 +1214,12 @@ mod tests { ); assert!(state .pending_store - .create_request( + .admit_request( + &state.capability_manager, SessionId::from_string("overflow"), resource.clone(), - Action::Read + Action::Read, + String::new() ) .await .is_denied()); diff --git a/elastos/crates/elastos-server/src/collaboration_history.rs b/elastos/crates/elastos-server/src/collaboration_history.rs index 535d7c536..7b69a780a 100644 --- a/elastos/crates/elastos-server/src/collaboration_history.rs +++ b/elastos/crates/elastos-server/src/collaboration_history.rs @@ -1063,7 +1063,7 @@ mod tests { async fn a_history_request_carries_no_presence_after_an_epoch_bump() { let now = now_secs(); let (a, _b, plane) = reachable_pair("history-epoch", now).await; - a.authority.manager().revoke_all("test epoch"); + a.authority.manager().revoke_all("test epoch").unwrap(); a.service.fetch_once(now).await.unwrap(); assert_not_shown_and_nothing_sent(&a, &plane, now); } @@ -1175,7 +1175,10 @@ mod tests { ) .unwrap(); if epoch { - a.authority.manager().revoke_all("projection smoke epoch"); + a.authority + .manager() + .revoke_all("projection smoke epoch") + .unwrap(); } else { a.decide(PresenceDecision::Hidden, now + 1).await; } @@ -1237,7 +1240,10 @@ mod tests { .await .unwrap(); } else { - self.authority.manager().revoke_all("between history peers"); + self.authority + .manager() + .revoke_all("between history peers") + .unwrap(); } response } diff --git a/elastos/crates/elastos-server/src/collaboration_presence_grant.rs b/elastos/crates/elastos-server/src/collaboration_presence_grant.rs index 8a2fbdb99..6afdb2d16 100644 --- a/elastos/crates/elastos-server/src/collaboration_presence_grant.rs +++ b/elastos/crates/elastos-server/src/collaboration_presence_grant.rs @@ -1029,7 +1029,7 @@ mod tests { .await .is_some()); } - authority.manager().revoke_all("test epoch"); + authority.manager().revoke_all("test epoch").unwrap(); for person in [principal.as_str(), second] { assert_eq!(gate(&authority, data, person, ME, runtime, 60).await, None); assert_eq!(state(data, person, &scope()).unwrap(), None); diff --git a/elastos/crates/elastos-server/src/collaboration_transport.rs b/elastos/crates/elastos-server/src/collaboration_transport.rs index 2b7b04a79..dcf05d4f7 100644 --- a/elastos/crates/elastos-server/src/collaboration_transport.rs +++ b/elastos/crates/elastos-server/src/collaboration_transport.rs @@ -1510,7 +1510,7 @@ mod tests { granted_presence_driver(&a, ac.clone(), FakeCarrier::new()).await; queued_presence(&a, &port, "granted-presence").await; assert_eq!(driver.retry_outgoing_once(NOW).await.unwrap().attempted, 1); - authority.manager().revoke_all("test epoch"); + authority.manager().revoke_all("test epoch").unwrap(); tokio::time::advance(SHARED_CUSTODY_RETRY).await; assert_eq!( driver.retry_outgoing_once(NOW + 5).await.unwrap().attempted, diff --git a/elastos/crates/elastos-server/src/resource_bridge.rs b/elastos/crates/elastos-server/src/resource_bridge.rs index 84eed0b25..ca620bddb 100644 --- a/elastos/crates/elastos-server/src/resource_bridge.rs +++ b/elastos/crates/elastos-server/src/resource_bridge.rs @@ -840,12 +840,10 @@ async fn handle_request(line: &str, ctx: &Option) -> Result anyhow::Result<()> { &data_dir, store, audit_log, metrics, ); - let new_pub = mgr.rotate_signing_key(&data_dir, &reason); + let new_pub = mgr.rotate_signing_key(&data_dir, &reason)?; println!("Signing key rotated."); println!(" New public key: {}", hex::encode(new_pub)); println!(" Epoch advanced: all prior tokens invalidated."); diff --git a/elastos/crates/elastos-server/src/server_infra.rs b/elastos/crates/elastos-server/src/server_infra.rs index 33ac418a7..9c3089bc1 100644 --- a/elastos/crates/elastos-server/src/server_infra.rs +++ b/elastos/crates/elastos-server/src/server_infra.rs @@ -4259,7 +4259,7 @@ mod tests { ) .await .expect("grant still validates before revoke"); - manager.revoke_all("test restart revoke"); + manager.revoke_all("test restart revoke").unwrap(); token }; From 1b9cf8e907921502b43e9a72bb7b8c55c0dafb64 Mon Sep 17 00:00:00 2001 From: SashaMIT Date: Sat, 17 Oct 2026 14:57:32 +0000 Subject: [PATCH 05/11] Make capability admission race tests deterministic First poll all contenders before resuming the decision boundary so a split lock fails both session and global admission regressions. Describe signing key ordering without claiming power-loss durability. Co-authored-by: Codex --- .../elastos-runtime/src/capability/manager.rs | 2 +- .../elastos-runtime/src/capability/pending.rs | 34 ++++++++++++++----- 2 files changed, 26 insertions(+), 10 deletions(-) diff --git a/elastos/crates/elastos-runtime/src/capability/manager.rs b/elastos/crates/elastos-runtime/src/capability/manager.rs index f45ba4e33..6e47880ad 100644 --- a/elastos/crates/elastos-runtime/src/capability/manager.rs +++ b/elastos/crates/elastos-runtime/src/capability/manager.rs @@ -615,7 +615,7 @@ impl CapabilityManager { Ok(new_epoch) } - /// Rotate the signing key after durable epoch revocation. + /// Rotate the signing key after epoch advancement succeeds. /// /// Epoch failure preserves the disk and memory key. If key publication /// fails after revocation, the epoch stays advanced and the error reports diff --git a/elastos/crates/elastos-runtime/src/capability/pending.rs b/elastos/crates/elastos-runtime/src/capability/pending.rs index 1981370ab..8d1b5883e 100644 --- a/elastos/crates/elastos-runtime/src/capability/pending.rs +++ b/elastos/crates/elastos-runtime/src/capability/pending.rs @@ -686,23 +686,39 @@ mod tests { } async fn concurrent_admissions(store: Arc, shared_session: bool) -> usize { + use std::future::Future as _; + use std::task::{Context, Waker}; + const CONTENDERS: usize = 64; - let barrier = Arc::new(tokio::sync::Barrier::new(CONTENDERS + 1)); - let mut tasks = tokio::task::JoinSet::new(); + let mut contenders = Vec::new(); for index in 0..CONTENDERS { let store = Arc::clone(&store); - let barrier = Arc::clone(&barrier); - tasks.spawn(async move { - barrier.wait().await; + contenders.push(Box::pin(tokio::task::unconstrained(async move { let session = if shared_session { "shared-session".to_string() } else { format!("concurrent-{index}") }; request_for(&store, &session).await.is_pending() - }); + }))); + } + + // Poll every contender once before any can resume past the injected + // decision/insertion yield. Correct code keeps the first writer held, + // so the rest wait before checking capacity. If the writer is split at + // that boundary, every contender checks the same last slot first. + // Exclude Tokio's cooperative-budget yield from this first-poll probe; + // the only suspension points are the lock and our explicit boundary. + { + let mut context = Context::from_waker(Waker::noop()); + for contender in &mut contenders { + assert!(contender.as_mut().poll(&mut context).is_pending()); + } + } + let mut tasks = tokio::task::JoinSet::new(); + for contender in contenders { + tasks.spawn(contender); } - barrier.wait().await; let mut admitted = 0; while let Some(result) = tasks.join_next().await { admitted += usize::from(result.unwrap()); @@ -730,7 +746,7 @@ mod tests { assert_eq!(store.list_session_pending("shared-session").await.len(), 1); } - #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + #[tokio::test] async fn test_concurrent_admission_respects_session_limit() { let store = Arc::new(create_test_store()); for _ in 0..MAX_PENDING_PER_SESSION - 1 { @@ -746,7 +762,7 @@ mod tests { assert!(request_for(&store, "other-session").await.is_pending()); } - #[tokio::test(flavor = "multi_thread", worker_threads = 4)] + #[tokio::test] async fn test_concurrent_admission_respects_global_limit() { let store = Arc::new(create_test_store()); for index in 0..MAX_PENDING_REQUESTS - 1 { From 3c02e80add768b19d8936a7e0c8b718989608d6a Mon Sep 17 00:00:00 2001 From: SashaMIT Date: Sun, 18 Oct 2026 00:56:20 +0000 Subject: [PATCH 06/11] Reuse the platform-neutral Home link test fixture Apply the test-only System fixture from 619c0961964aa2a5da938af5cba622349edb1382 to the refreshed develop base. Preserve the existing authority assertions and production behavior. Co-authored-by: Codex --- .../src/api/gateway_tests/home_system.rs | 27 ++++++++++--------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/elastos/crates/elastos-server/src/api/gateway_tests/home_system.rs b/elastos/crates/elastos-server/src/api/gateway_tests/home_system.rs index a30bded9c..02d08fc68 100644 --- a/elastos/crates/elastos-server/src/api/gateway_tests/home_system.rs +++ b/elastos/crates/elastos-server/src/api/gateway_tests/home_system.rs @@ -11945,9 +11945,10 @@ async fn home_external_link_names_the_launched_app_and_the_exact_address() { }) }; - let browser_token = launched(BROWSER_CAPSULE_ID); + // System is launchable on every host; Browser availability is platform-specific. + let system_token = launched(SYSTEM_CAPSULE_ID); for (app_token, app_id, app_title) in [ - (browser_token.clone(), BROWSER_CAPSULE_ID, "Browser"), + (system_token.clone(), SYSTEM_CAPSULE_ID, "System"), ( launched(HOME_CLI_CAPSULE_ID_FOR_TEST), "home-cli", @@ -11979,13 +11980,13 @@ async fn home_external_link_names_the_launched_app_and_the_exact_address() { "foreign origin", "https://evil.example", home_token.clone(), - browser_token.clone(), + system_token.clone(), ), ( "an app frame calling directly", "null", home_token.clone(), - browser_token.clone(), + system_token.clone(), ), ( "app token from another Home session", @@ -11993,8 +11994,8 @@ async fn home_external_link_names_the_launched_app_and_the_exact_address() { home_token.clone(), issue_home_projection_launch_token_with_context( dir.path(), - BROWSER_CAPSULE_ID, - BROWSER_CAPSULE_ID, + SYSTEM_CAPSULE_ID, + SYSTEM_CAPSULE_ID, &other_session, ) .unwrap(), @@ -12004,13 +12005,13 @@ async fn home_external_link_names_the_launched_app_and_the_exact_address() { "http://localhost:61180", issue_home_launch_token_with_context(dir.path(), HOME_CAPSULE_ID, &other_session) .unwrap(), - browser_token.clone(), + system_token.clone(), ), ( "app token Home did not mint", "http://localhost:61180", home_token.clone(), - issue_home_launch_token_with_context(dir.path(), BROWSER_CAPSULE_ID, &context).unwrap(), + issue_home_launch_token_with_context(dir.path(), SYSTEM_CAPSULE_ID, &context).unwrap(), ), ( "Home GUI is not a link sender", @@ -12037,7 +12038,7 @@ async fn home_external_link_names_the_launched_app_and_the_exact_address() { issue_home_projection_launch_token_with_context( dir.path(), "assistant", - BROWSER_CAPSULE_ID, + SYSTEM_CAPSULE_ID, &context, ) .unwrap(), @@ -12061,7 +12062,7 @@ async fn home_external_link_names_the_launched_app_and_the_exact_address() { } let expired = - issue_expired_home_launch_token_with_context(dir.path(), BROWSER_CAPSULE_ID, &context) + issue_expired_home_launch_token_with_context(dir.path(), SYSTEM_CAPSULE_ID, &context) .unwrap(); let (status, body) = home_external_link_raw( &app, @@ -12088,7 +12089,7 @@ async fn home_external_link_names_the_launched_app_and_the_exact_address() { &app, "http://localhost:61180", &home_token, - request(&browser_token, url), + request(&system_token, url), ) .await; assert_eq!(status, StatusCode::BAD_REQUEST, "{url:?}: {body}"); @@ -12099,13 +12100,13 @@ async fn home_external_link_names_the_launched_app_and_the_exact_address() { &home_token, json!({ "schema": "elastos.home.external-link.request/v2", - "app_token": browser_token, + "app_token": system_token, "url": "https://example.com/", }), ) .await; assert_eq!(status, StatusCode::BAD_REQUEST); - let mut extra = request(&browser_token, "https://example.com/"); + let mut extra = request(&system_token, "https://example.com/"); extra["app_id"] = json!("wallet"); let (status, _) = home_external_link_raw(&app, "http://localhost:61180", &home_token, extra).await; From ae2a9a5323018a660585229aba3712faf5b9c3b5 Mon Sep 17 00:00:00 2001 From: SashaMIT Date: Sun, 18 Oct 2026 01:04:25 +0000 Subject: [PATCH 07/11] Preserve durable capability state across process lifecycles Serialize persistent state updates across processes, retain the durable maximum, and keep command and Runtime lifecycle state consistent. Add focused persistence, failure, and shutdown regressions. Co-authored-by: Codex --- .../src/bootstrap/runtime_builder.rs | 33 ++ .../elastos-runtime/src/capability/manager.rs | 7 + .../elastos-runtime/src/capability/store.rs | 339 +++++++++++++++++- elastos/crates/elastos-server/src/main.rs | 2 +- .../crates/elastos-server/src/security_cmd.rs | 86 ++++- 5 files changed, 444 insertions(+), 23 deletions(-) diff --git a/elastos/crates/elastos-runtime/src/bootstrap/runtime_builder.rs b/elastos/crates/elastos-runtime/src/bootstrap/runtime_builder.rs index 488756b0e..0d7813af0 100755 --- a/elastos/crates/elastos-runtime/src/bootstrap/runtime_builder.rs +++ b/elastos/crates/elastos-runtime/src/bootstrap/runtime_builder.rs @@ -575,6 +575,39 @@ mod tests { assert!(runtime.stop().await.is_err()); } + #[tokio::test] + async fn test_runtime_shutdown_preserves_an_externally_advanced_epoch() { + let directory = tempfile::tempdir().unwrap(); + let config = RuntimeConfig { + data_dir: directory.path().to_path_buf(), + enable_audit: false, + ..Default::default() + }; + let mut runtime = + ElastosRuntime::build(config, Arc::new(MockComputeProvider), Arc::new(NullFetcher)) + .await + .unwrap(); + runtime.start().await.unwrap(); + let store_path = directory.path().join("capability_store"); + let external = CapabilityStore::with_persistence(&store_path) + .await + .unwrap(); + assert_eq!(external.advance_epoch().unwrap(), 1); + assert_eq!(runtime.capability_store.current_epoch(), 0); + + runtime.stop().await.unwrap(); + + let reloaded = CapabilityStore::with_persistence(&store_path) + .await + .unwrap(); + assert_eq!( + reloaded.current_epoch(), + 1, + "shutdown must preserve the externally advanced epoch" + ); + assert!(!reloaded.is_epoch_valid(0)); + } + #[tokio::test] async fn test_runtime_accessors() { let temp_dir = tempfile::tempdir().unwrap(); diff --git a/elastos/crates/elastos-runtime/src/capability/manager.rs b/elastos/crates/elastos-runtime/src/capability/manager.rs index 6e47880ad..a15e505d6 100644 --- a/elastos/crates/elastos-runtime/src/capability/manager.rs +++ b/elastos/crates/elastos-runtime/src/capability/manager.rs @@ -856,6 +856,9 @@ mod tests { None, ); if blocked_path != "overflow" { + if blocked_path == "epoch" { + std::fs::remove_file(directory.path().join("epoch")).unwrap(); + } std::fs::create_dir(directory.path().join(blocked_path)).unwrap(); } let audit_before = audit.recent_events(100).len(); @@ -878,6 +881,10 @@ mod tests { })); if blocked_path != "overflow" { std::fs::remove_dir(directory.path().join(blocked_path)).unwrap(); + if blocked_path == "epoch" { + std::fs::write(directory.path().join("epoch"), original_epoch.to_string()) + .unwrap(); + } } let reloaded_store = Arc::new( CapabilityStore::with_persistence(directory.path()) diff --git a/elastos/crates/elastos-runtime/src/capability/store.rs b/elastos/crates/elastos-runtime/src/capability/store.rs index a178e3d08..e020e1071 100644 --- a/elastos/crates/elastos-runtime/src/capability/store.rs +++ b/elastos/crates/elastos-runtime/src/capability/store.rs @@ -7,6 +7,7 @@ use std::collections::{HashMap, HashSet}; use std::fs; +use std::io::Write as _; use std::path::{Path, PathBuf}; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::Mutex; @@ -14,6 +15,43 @@ use tokio::sync::RwLock; use super::token::TokenId; +/// The stable lock file orders epoch writers across Runtime processes. +struct EpochFileLock(fs::File); + +impl EpochFileLock { + fn acquire(path: &Path, initialize: bool) -> std::io::Result<(Self, bool)> { + let lock_path = path.join("epoch.lock"); + let mut options = fs::OpenOptions::new(); + options.read(true).write(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt as _; + options.mode(0o600); + } + let (file, created) = if initialize { + match options.create_new(true).open(&lock_path) { + Ok(file) => (file, true), + Err(error) if error.kind() == std::io::ErrorKind::AlreadyExists => { + (options.create_new(false).open(&lock_path)?, false) + } + Err(error) => return Err(error), + } + } else { + (options.open(&lock_path)?, false) + }; + file.lock()?; + Ok((Self(file), created)) + } +} + +impl Drop for EpochFileLock { + fn drop(&mut self) { + // Release at scope exit even if a concurrently spawned child briefly + // inherited this open description before its exec closes the fd. + let _ = self.0.unlock(); + } +} + /// Capability store - manages token state /// /// Owns the revocation epoch as an instance field (not process-global), @@ -69,7 +107,7 @@ impl CapabilityStore { Ok(store) } - /// Get the current revocation epoch + /// Get the last observed revocation epoch. Validation refreshes disk state. pub fn current_epoch(&self) -> u64 { self.epoch.load(Ordering::SeqCst) } @@ -98,12 +136,15 @@ impl CapabilityStore { /// Runtime caller is construction-time loading; use `advance_epoch` for /// revocation, which persists the new value before publishing it. pub fn set_epoch(&self, epoch: u64) { - self.epoch_transaction(|current| Ok(current.max(epoch)), |_| Ok(())) - .expect("initializing an epoch cannot fail"); + let _epoch_writer = self + .epoch_writer + .lock() + .unwrap_or_else(|poisoned| poisoned.into_inner()); + self.epoch.fetch_max(epoch, Ordering::SeqCst); } - /// All epoch writers share this synchronous transaction. The update and - /// writer run under one lock; memory is published only after writer success. + /// Serialize the durable snapshot, update and write across instances and + /// processes. A stale instance always starts from the durable maximum. fn epoch_transaction( &self, update: impl FnOnce(u64) -> std::io::Result, @@ -113,24 +154,72 @@ impl CapabilityStore { .epoch_writer .lock() .unwrap_or_else(|poisoned| poisoned.into_inner()); + let _file_lock = self + .storage_path + .as_ref() + .map(|path| EpochFileLock::acquire(path, false)) + .transpose()?; + if let Some(path) = &self.storage_path { + self.epoch + .fetch_max(Self::read_epoch(path)?, Ordering::SeqCst); + } let next = update(self.current_epoch())?; - writer(next)?; - self.epoch.store(next, Ordering::SeqCst); + if let Err(error) = writer(next) { + // Rename can succeed before directory synchronization fails. Keep + // the advanced state while reporting the durability error. + if let Some(path) = &self.storage_path { + if let Ok(persisted) = Self::read_epoch(path) { + self.epoch.fetch_max(persisted, Ordering::SeqCst); + } + } + return Err(error); + } + self.epoch.fetch_max(next, Ordering::SeqCst); Ok(next) } + fn read_epoch(path: &Path) -> std::io::Result { + fs::read_to_string(path.join("epoch"))? + .trim() + .parse() + .map_err(|_| { + std::io::Error::new(std::io::ErrorKind::InvalidData, "invalid capability epoch") + }) + } + fn write_epoch(&self, epoch: u64) -> std::io::Result<()> { if let Some(path) = &self.storage_path { let epoch_path = path.join("epoch"); let tmp_path = path.join("epoch.tmp"); - fs::write(&tmp_path, epoch.to_string())?; + let mut options = fs::OpenOptions::new(); + options.create(true).truncate(true).write(true); + #[cfg(unix)] + { + use std::os::unix::fs::OpenOptionsExt as _; + options.mode(0o600); + } + let mut staged = options.open(&tmp_path)?; + staged.write_all(epoch.to_string().as_bytes())?; + staged.sync_all()?; fs::rename(&tmp_path, &epoch_path)?; + #[cfg(unix)] + fs::File::open(path).and_then(|directory| directory.sync_all()).map_err(|error| { + std::io::Error::new(error.kind(), format!( + "Capability epoch was written, but directory synchronization failed; durability is uncertain: {error}" + )) + })?; } Ok(()) } /// Check if a token's epoch is valid pub fn is_epoch_valid(&self, token_epoch: u64) -> bool { + if let Some(path) = &self.storage_path { + let Ok(persisted) = Self::read_epoch(path) else { + return false; + }; + self.epoch.fetch_max(persisted, Ordering::SeqCst); + } token_epoch >= self.current_epoch() } @@ -214,14 +303,20 @@ impl CapabilityStore { /// Load state from disk async fn load_state(&self) -> std::io::Result<()> { if let Some(path) = &self.storage_path { - // Load epoch - let epoch_path = path.join("epoch"); - if epoch_path.exists() { - if let Ok(content) = fs::read_to_string(&epoch_path) { - if let Ok(epoch) = content.trim().parse::() { - self.set_epoch(epoch); + // Only the creator of the stable lock can initialize an absent + // epoch. An interrupted initialization or removed epoch requires + // explicit repair instead of silently restoring epoch zero. + { + let (_file_lock, created) = EpochFileLock::acquire(path, true)?; + let epoch = match Self::read_epoch(path) { + Ok(epoch) => epoch, + Err(error) if created && error.kind() == std::io::ErrorKind::NotFound => { + self.write_epoch(0)?; + 0 } - } + Err(error) => return Err(error), + }; + self.set_epoch(epoch); } // Load revoked tokens @@ -393,6 +488,9 @@ mod tests { let store = CapabilityStore::with_persistence(directory.path()) .await .unwrap(); + if blocked_path == "epoch" { + fs::remove_file(directory.path().join("epoch")).unwrap(); + } fs::create_dir(directory.path().join(blocked_path)).unwrap(); assert!(store.advance_epoch().is_err()); assert_eq!(store.current_epoch(), 0); @@ -401,6 +499,9 @@ mod tests { assert!(store.persist().await.is_err()); assert_eq!(store.current_epoch(), 0); fs::remove_dir(directory.path().join(blocked_path)).unwrap(); + if blocked_path == "epoch" { + fs::write(directory.path().join("epoch"), "0").unwrap(); + } assert_eq!(store.advance_epoch().unwrap(), 1); let reloaded = CapabilityStore::with_persistence(directory.path()) .await @@ -435,6 +536,214 @@ mod tests { assert_eq!(reloaded.current_epoch(), u64::MAX); } + #[tokio::test] + async fn test_independent_epoch_writers_preserve_every_advance() { + use std::sync::{Arc, Barrier}; + const WRITERS: usize = 4; + const ADVANCES: usize = 4; + let directory = tempfile::tempdir().unwrap(); + let mut stores = Vec::new(); + for _ in 0..WRITERS { + stores.push( + CapabilityStore::with_persistence(directory.path()) + .await + .unwrap(), + ); + } + let barrier = Arc::new(Barrier::new(WRITERS)); + let handles = stores + .into_iter() + .map(|store| { + let barrier = barrier.clone(); + std::thread::spawn(move || { + barrier.wait(); + (0..ADVANCES) + .map(|_| store.advance_epoch().unwrap()) + .collect::>() + }) + }) + .collect::>(); + let mut epochs = handles + .into_iter() + .flat_map(|handle| handle.join().unwrap()) + .collect::>(); + epochs.sort_unstable(); + assert_eq!( + epochs, + (1..=(WRITERS * ADVANCES) as u64).collect::>() + ); + let reloaded = CapabilityStore::with_persistence(directory.path()) + .await + .unwrap(); + assert_eq!(reloaded.current_epoch(), (WRITERS * ADVANCES) as u64); + } + + #[test] + fn test_epoch_file_lock_is_process_wide() { + const CHILD_PATH: &str = "ELASTOS_EPOCH_LOCK_TEST_PATH"; + if let Some(path) = std::env::var_os(CHILD_PATH) { + let file = fs::OpenOptions::new() + .read(true) + .write(true) + .open(path) + .unwrap(); + assert!( + matches!(file.try_lock(), Err(fs::TryLockError::WouldBlock)), + "a second process entered an active epoch transaction" + ); + println!("EPOCH_FILE_LOCK_CHILD_VERIFIED"); + return; + } + let runtime = tokio::runtime::Builder::new_current_thread() + .build() + .unwrap(); + let directory = tempfile::tempdir().unwrap(); + let store = runtime + .block_on(CapabilityStore::with_persistence(directory.path())) + .unwrap(); + store + .epoch_transaction( + |current| { + let result = std::process::Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "capability::store::tests::test_epoch_file_lock_is_process_wide", + "--nocapture", + ]) + .env(CHILD_PATH, directory.path().join("epoch.lock")) + .output() + .unwrap(); + assert!( + result.status.success(), + "cross-process lock probe failed: {}", + String::from_utf8_lossy(&result.stderr) + ); + let output = String::from_utf8_lossy(&result.stdout); + assert!( + output.contains("running 1 test") + && output.contains("EPOCH_FILE_LOCK_CHILD_VERIFIED"), + "the exact child lock probe did not execute: {output}" + ); + Ok(current) + }, + |_| Ok(()), + ) + .unwrap(); + } + + #[tokio::test] + async fn test_external_epoch_advance_retires_old_tokens_in_a_running_store() { + use crate::capability::manager::ValidationError; + use crate::capability::{Action, CapabilityManager, ResourceId, TokenConstraints}; + use crate::primitives::{audit::AuditLog, metrics::MetricsManager}; + use std::sync::Arc; + + let directory = tempfile::tempdir().unwrap(); + let running = Arc::new( + CapabilityStore::with_persistence(directory.path()) + .await + .unwrap(), + ); + let manager = CapabilityManager::new( + running.clone(), + Arc::new(AuditLog::new()), + Arc::new(MetricsManager::new()), + ); + let resource = ResourceId::new("localhost://Users/self/Documents/test"); + let tokens = ["first", "second"].map(|capsule| { + manager.grant( + capsule, + resource.clone(), + Action::Read, + TokenConstraints::default(), + None, + ) + }); + assert_ne!(tokens[0].id, tokens[1].id); + let external = CapabilityStore::with_persistence(directory.path()) + .await + .unwrap(); + assert!(running.is_epoch_valid(0)); + assert_eq!(external.advance_epoch().unwrap(), 1); + for token in tokens { + assert!( + matches!( + manager + .validate(&token, &token.capsule, Action::Read, &resource, None) + .await, + Err(ValidationError::TokenRevoked) + ), + "a running store must reject tokens from the earlier durable epoch" + ); + } + assert!(running.is_epoch_valid(1)); + assert_eq!(running.current_epoch(), 1); + } + + #[tokio::test] + async fn test_missing_or_invalid_epoch_fails_closed_until_repaired() { + for missing in [true, false] { + let directory = tempfile::tempdir().unwrap(); + let store = CapabilityStore::with_persistence(directory.path()) + .await + .unwrap(); + let epoch_path = directory.path().join("epoch"); + if missing { + fs::remove_file(&epoch_path).unwrap(); + } else { + fs::write(&epoch_path, "invalid").unwrap(); + } + assert!(!store.is_epoch_valid(0)); + assert!(store.advance_epoch().is_err()); + assert!(store.persist().await.is_err()); + assert!(CapabilityStore::with_persistence(directory.path()) + .await + .is_err()); + assert_eq!(store.current_epoch(), 0); + fs::write(&epoch_path, "0").unwrap(); + assert_eq!(store.advance_epoch().unwrap(), 1); + } + } + + #[tokio::test] + async fn test_interrupted_epoch_initialization_requires_explicit_repair() { + let directory = tempfile::tempdir().unwrap(); + fs::File::create(directory.path().join("epoch.lock")).unwrap(); + assert!( + CapabilityStore::with_persistence(directory.path()) + .await + .is_err(), + "interrupted initialization must require explicit repair" + ); + assert!(!directory.path().join("epoch").exists()); + } + + #[tokio::test] + async fn test_post_write_error_keeps_the_observed_epoch_advanced() { + let directory = tempfile::tempdir().unwrap(); + let store = CapabilityStore::with_persistence(directory.path()) + .await + .unwrap(); + let error = store + .epoch_transaction( + |current| Ok(current + 1), + |next| { + store.write_epoch(next)?; + Err(std::io::Error::other("fixture: durability is uncertain")) + }, + ) + .unwrap_err(); + assert!(error.to_string().contains("durability is uncertain")); + assert_eq!( + store.current_epoch(), + 1, + "a post-write error must retain the observed advanced epoch" + ); + assert_eq!(CapabilityStore::read_epoch(directory.path()).unwrap(), 1); + store.persist().await.unwrap(); + assert_eq!(CapabilityStore::read_epoch(directory.path()).unwrap(), 1); + } + #[test] fn test_epoch() { let store = CapabilityStore::new(); diff --git a/elastos/crates/elastos-server/src/main.rs b/elastos/crates/elastos-server/src/main.rs index 6c9e5b513..a00396676 100755 --- a/elastos/crates/elastos-server/src/main.rs +++ b/elastos/crates/elastos-server/src/main.rs @@ -1446,7 +1446,7 @@ async fn run() -> anyhow::Result<()> { } Commands::Emergency(cmd) => { - security_cmd::run_emergency(cmd)?; + security_cmd::run_emergency(cmd).await?; } Commands::Config(cmd) => { diff --git a/elastos/crates/elastos-server/src/security_cmd.rs b/elastos/crates/elastos-server/src/security_cmd.rs index 7b22c1956..fe4e4a79b 100644 --- a/elastos/crates/elastos-server/src/security_cmd.rs +++ b/elastos/crates/elastos-server/src/security_cmd.rs @@ -37,20 +37,26 @@ pub fn run_tls(tls_cmd: TlsCommand) -> anyhow::Result<()> { Ok(()) } -pub fn run_emergency(cmd: EmergencyCommand) -> anyhow::Result<()> { +pub async fn run_emergency(cmd: EmergencyCommand) -> anyhow::Result<()> { + let data_dir = elastos_server::sources::default_data_dir(); + run_emergency_in(&data_dir, cmd).await +} + +async fn run_emergency_in(data_dir: &std::path::Path, cmd: EmergencyCommand) -> anyhow::Result<()> { match cmd { EmergencyCommand::Rotate { reason } => { - let data_dir = elastos_server::sources::default_data_dir(); - let _ = std::fs::create_dir_all(&data_dir); - - let store = Arc::new(capability::CapabilityStore::new()); + std::fs::create_dir_all(data_dir)?; + let store = Arc::new( + capability::CapabilityStore::with_persistence(data_dir.join("capability_store")) + .await?, + ); let audit_log = Arc::new(primitives::audit::AuditLog::new()); let metrics = Arc::new(primitives::metrics::MetricsManager::new()); let mut mgr = capability::CapabilityManager::load_or_generate( - &data_dir, store, audit_log, metrics, + data_dir, store, audit_log, metrics, ); - let new_pub = mgr.rotate_signing_key(&data_dir, &reason)?; + let new_pub = mgr.rotate_signing_key(data_dir, &reason)?; println!("Signing key rotated."); println!(" New public key: {}", hex::encode(new_pub)); println!(" Epoch advanced: all prior tokens invalidated."); @@ -62,3 +68,69 @@ pub fn run_emergency(cmd: EmergencyCommand) -> anyhow::Result<()> { Ok(()) } + +#[cfg(test)] +mod tests { + use super::*; + + #[tokio::test] + async fn emergency_rotation_uses_the_commands_persistent_store() { + let data_dir = tempfile::tempdir().unwrap(); + let store_path = data_dir.path().join("capability_store"); + let store = capability::CapabilityStore::with_persistence(&store_path) + .await + .unwrap(); + for _ in 0..7 { + store.advance_epoch().unwrap(); + } + for expected in [8, 9] { + run_emergency_in( + data_dir.path(), + EmergencyCommand::Rotate { + reason: "isolated command fixture".into(), + }, + ) + .await + .unwrap(); + let reloaded = capability::CapabilityStore::with_persistence(&store_path) + .await + .unwrap(); + assert_eq!( + reloaded.current_epoch(), + expected, + "the command must advance the durable epoch" + ); + assert!(data_dir.path().join("signing_key").is_file()); + } + // A separately loaded, older store also preserves the command result. + store.persist().await.unwrap(); + let reloaded = capability::CapabilityStore::with_persistence(&store_path) + .await + .unwrap(); + assert_eq!(reloaded.current_epoch(), 9); + } + + #[tokio::test] + async fn emergency_rotation_refuses_unreadable_state_before_creating_a_key() { + let data_dir = tempfile::tempdir().unwrap(); + let store_path = data_dir.path().join("capability_store"); + std::fs::create_dir(&store_path).unwrap(); + std::fs::write(store_path.join("epoch"), "invalid").unwrap(); + assert!( + run_emergency_in( + data_dir.path(), + EmergencyCommand::Rotate { + reason: "isolated command fixture".into(), + }, + ) + .await + .is_err(), + "the command must refuse malformed durable state" + ); + assert!(!data_dir.path().join("signing_key").exists()); + assert_eq!( + std::fs::read_to_string(store_path.join("epoch")).unwrap(), + "invalid" + ); + } +} From 081bc7c3f70f5906e0cf831bd6a0a7eed53a4be0 Mon Sep 17 00:00:00 2001 From: SashaMIT Date: Sun, 18 Oct 2026 01:07:40 +0000 Subject: [PATCH 08/11] Retire grants with their session and revocation epoch Bind authenticated session admission to registry lifecycle, retain cleanup ownership across cancellation, and separate Component request scopes. Preserve grants issued after epoch advancement and retire unreturned tokens when registration fails. Add capacity, cancellation, ownership, and epoch-race regressions. Co-authored-by: Codex --- .../elastos-runtime/src/capability/pending.rs | 663 ++++++++++++++++-- .../elastos-runtime/src/session/registry.rs | 322 ++++++--- .../src/api/handlers/capability.rs | 122 +++- .../elastos-server/src/resource_bridge.rs | 4 +- .../crates/elastos-server/src/server_infra.rs | 13 +- 5 files changed, 944 insertions(+), 180 deletions(-) diff --git a/elastos/crates/elastos-runtime/src/capability/pending.rs b/elastos/crates/elastos-runtime/src/capability/pending.rs index 8d1b5883e..d03316beb 100644 --- a/elastos/crates/elastos-runtime/src/capability/pending.rs +++ b/elastos/crates/elastos-runtime/src/capability/pending.rs @@ -2,7 +2,7 @@ //! //! Tracks capability requests that are awaiting user approval (grant/deny). //! Requests have a timeout and are cleaned up periodically. -use std::collections::HashMap; +use std::collections::{HashMap, HashSet}; use std::sync::Arc; use tokio::sync::RwLock; @@ -24,6 +24,20 @@ pub const MAX_PENDING_REQUESTS: usize = 1024; /// A session keeps its live grant records within its own bounded share. pub const MAX_PENDING_PER_SESSION: usize = 32; +/// Runtime-owned request scope. Session scopes end with their authenticated +/// connection; Component Bus scopes retain their existing capsule ownership. +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +pub enum RequestScope { + Session(SessionId), + Capsule(String), +} + +impl From for RequestScope { + fn from(session: SessionId) -> Self { + Self::Session(session) + } +} + /// Unique identifier for a pending request #[derive(Debug, Clone, PartialEq, Eq, Hash, Serialize, Deserialize)] pub struct RequestId(pub String); @@ -112,6 +126,9 @@ pub struct PendingCapabilityRequest { /// Session that made the request pub session_id: SessionId, + /// Only authenticated Session scopes participate in registry retirement. + scope: RequestScope, + /// Requested resource pub resource: ResourceId, @@ -154,6 +171,7 @@ impl PendingCapabilityRequest { Self { id: RequestId::new(), + scope: RequestScope::Session(session_id.clone()), session_id, resource, action, @@ -198,12 +216,21 @@ struct PendingRequestState { /// Requests and their index change together under the store's lock. requests: HashMap, /// Index from session ID to request IDs (for listing) - session_requests: HashMap>, + session_requests: HashMap>, + /// Present only when this store belongs to a capability-enabled registry. + /// Ended IDs leave this set instead of accumulating as tombstones. + active_sessions: Option>, } impl PendingRequestState { - fn session_admission_count(&self, session_id: &str, now: &SecureTimestamp) -> usize { - self.session_requests.get(session_id).map_or(0, |ids| { + fn session_is_active(&self, session_id: &str) -> bool { + self.active_sessions + .as_ref() + .is_none_or(|active| active.contains(session_id)) + } + + fn session_admission_count(&self, scope: &RequestScope, now: &SecureTimestamp) -> usize { + self.session_requests.get(scope).map_or(0, |ids| { ids.iter() .filter(|id| { self.requests @@ -286,6 +313,76 @@ impl PendingRequestStore { } } + pub(crate) fn with_session_lifecycle(audit_log: Arc) -> Self { + Self { + state: RwLock::new(PendingRequestState { + active_sessions: Some(HashSet::new()), + ..PendingRequestState::default() + }), + audit_log, + timeout_secs: DEFAULT_REQUEST_TIMEOUT_SECS, + } + } + + pub(crate) async fn register_session(&self, session_id: &str) { + let mut state = self.state.write().await; + if let Some(active) = &mut state.active_sessions { + active.insert(session_id.to_string()); + } + } + + pub(crate) async fn session_is_active(&self, session_id: &str) -> bool { + self.state.read().await.session_is_active(session_id) + } + + pub(crate) async fn deactivate_session(&self, session_id: &str) { + let mut state = self.state.write().await; + if let Some(active) = &mut state.active_sessions { + active.remove(session_id); + } + } + + pub(crate) async fn retire_session(&self, manager: &CapabilityManager, session_id: &str) { + self.retire_session_with(session_id, |token_id| async move { + manager.revoke(token_id, "Session ended").await; + }) + .await; + } + + async fn retire_session_with(&self, session_id: &str, mut revoke: F) + where + F: FnMut(super::token::TokenId) -> Fut, + Fut: std::future::Future, + { + let tokens = { + let mut state = self.state.write().await; + if let Some(active) = &mut state.active_sessions { + active.remove(session_id); + } + state + .requests + .values() + .filter(|request| { + matches!(&request.scope, RequestScope::Session(id) if id.as_str() == session_id) + }) + .filter_map(|request| request.granted_token().map(|token| *token.id())) + .collect::>() + }; + + // Inactive scopes cannot acquire new requests or grants. Retain their + // records until revocation finishes so cancellation can be retried. + // The registry retains its inactive session entry until this completes. + for token_id in tokens { + revoke(token_id).await; + } + + let mut state = self.state.write().await; + state.requests.retain(|_, request| { + !(matches!(&request.scope, RequestScope::Session(id) if id.as_str() == session_id)) + }); + state.prune_session_index(); + } + /// Shared audit sink used by Runtime-owned bridge adapters. pub fn audit_log(&self) -> Arc { self.audit_log.clone() @@ -308,14 +405,15 @@ impl PendingRequestStore { pub async fn admit_request( &self, capability_manager: &CapabilityManager, - session_id: SessionId, + scope: impl Into, resource: ResourceId, action: Action, reason: String, ) -> PendingCapabilityRequest { - self.cleanup_retired_grants_at_capacity(capability_manager, Some(session_id.as_str())) + let scope = scope.into(); + self.cleanup_retired_grants_at_capacity(capability_manager, Some(&scope)) .await; - self.create_request_with_reason(session_id, resource, action, reason) + self.create_request_with_reason(scope, resource, action, reason) .await } @@ -326,31 +424,39 @@ impl PendingRequestStore { resource: ResourceId, action: Action, ) -> PendingCapabilityRequest { - self.create_request_with_reason(session_id, resource, action, String::new()) + self.create_request_with_reason(session_id.into(), resource, action, String::new()) .await } async fn create_request_with_reason( &self, - session_id: SessionId, + scope: RequestScope, resource: ResourceId, action: Action, reason: String, ) -> PendingCapabilityRequest { + let session_id = match &scope { + RequestScope::Session(session_id) => session_id.clone(), + RequestScope::Capsule(capsule_id) => SessionId(capsule_id.clone()), + }; // Admission, cleanup, and both index updates are one state transition. let mut state = self.state.write().await; let now = SecureTimestamp::now(); if state.requests.len() >= MAX_PENDING_REQUESTS - || state.session_admission_count(session_id.as_str(), &now) >= MAX_PENDING_PER_SESSION + || state.session_admission_count(&scope, &now) >= MAX_PENDING_PER_SESSION { state.cleanup_expired(&now); state.cleanup_old(&now, 0); } - let denial = if state.requests.len() >= MAX_PENDING_REQUESTS { + let denial = if matches!(scope, RequestScope::Session(_)) + && !state.session_is_active(session_id.as_str()) + { + Some("Session has ended") + } else if state.requests.len() >= MAX_PENDING_REQUESTS { Some("Too many pending requests") } else { - (state.session_admission_count(session_id.as_str(), &now) >= MAX_PENDING_PER_SESSION) + (state.session_admission_count(&scope, &now) >= MAX_PENDING_PER_SESSION) .then_some("Too many outstanding requests or grants for this session") }; @@ -361,6 +467,7 @@ impl PendingRequestStore { reason, self.timeout_secs, ); + request.scope = scope.clone(); if let Some(reason) = denial { request.status = RequestStatus::Denied { reason: reason.to_string(), @@ -375,7 +482,7 @@ impl PendingRequestStore { state.requests.insert(request.id.0.clone(), request.clone()); state .session_requests - .entry(session_id.0.clone()) + .entry(scope) .or_default() .push(request.id.0.clone()); drop(state); @@ -408,6 +515,16 @@ impl PendingRequestStore { ) -> Result<(), String> { let mut state = self.state.write().await; + let request = state + .requests + .get(request_id) + .ok_or_else(|| format!("Request not found: {}", request_id))?; + if matches!(request.scope, RequestScope::Session(_)) + && !state.session_is_active(request.session_id.as_str()) + { + return Err("Session has ended".to_string()); + } + let request = state .requests .get_mut(request_id) @@ -478,7 +595,7 @@ impl PendingRequestStore { state .session_requests - .get(session_id) + .get(&RequestScope::Session(SessionId::from_string(session_id))) .map(|ids| { ids.iter() .filter_map(|id| state.requests.get(id)) @@ -510,9 +627,9 @@ impl PendingRequestStore { async fn cleanup_retired_grants_at_capacity( &self, capability_manager: &CapabilityManager, - session_id: Option<&str>, + scope: Option<&RequestScope>, ) -> usize { - self.reclaim_retired_grants_with(session_id, |token| async move { + self.reclaim_retired_grants_with(scope, |token| async move { capability_manager.is_revoked_or_exhausted(&token).await }) .await @@ -520,7 +637,7 @@ impl PendingRequestStore { async fn reclaim_retired_grants_with( &self, - session_id: Option<&str>, + scope: Option<&RequestScope>, mut is_retired: F, ) -> usize where @@ -530,8 +647,8 @@ impl PendingRequestStore { let candidates = { let state = self.state.read().await; let global_pressure = state.requests.len() >= MAX_PENDING_REQUESTS; - let session_pressure = session_id.is_some_and(|session_id| { - state.session_admission_count(session_id, &SecureTimestamp::now()) + let session_pressure = scope.is_some_and(|scope| { + state.session_admission_count(scope, &SecureTimestamp::now()) >= MAX_PENDING_PER_SESSION }); if !global_pressure && !session_pressure { @@ -540,9 +657,7 @@ impl PendingRequestStore { state .requests .iter() - .filter(|(_, request)| { - global_pressure || Some(request.session_id.as_str()) == session_id - }) + .filter(|(_, request)| global_pressure || Some(&request.scope) == scope) .filter_map(|(id, request)| match &request.status { RequestStatus::Granted { token, duration } => { Some((id.clone(), token.as_ref().clone(), *duration)) @@ -592,7 +707,7 @@ impl PendingRequestStore { let state = self.state.read().await; state .session_requests - .get(session_id) + .get(&RequestScope::Session(SessionId::from_string(session_id))) .into_iter() .flatten() .filter_map(|id| state.requests.get(id)) @@ -626,16 +741,19 @@ impl PendingRequestStore { } } - /// Mark all granted requests as revoked + /// Mark grants from epochs older than the persisted revocation epoch revoked. /// /// Called when epoch is advanced to revoke all capabilities - pub async fn revoke_all_granted(&self) { + pub async fn revoke_all_granted_before_epoch(&self, new_epoch: u64) { let mut state = self.state.write().await; let now = SecureTimestamp::now(); let mut events = Vec::new(); for (request_id, request) in state.requests.iter_mut() { - if request.is_granted() { + if request + .granted_token() + .is_some_and(|token| token.constraints().epoch() < new_epoch) + { request.status = RequestStatus::Denied { reason: "Epoch advanced - all capabilities revoked".to_string(), }; @@ -658,6 +776,377 @@ impl PendingRequestStore { #[cfg(test)] mod tests { use super::*; + use crate::capability::{CapabilityStore, TokenConstraints}; + use crate::primitives::metrics::MetricsManager; + use crate::session::{SessionRegistry, SessionType}; + + fn lifecycle_store() -> ( + SessionRegistry, + Arc, + Arc, + ) { + let audit = Arc::new(AuditLog::new()); + let manager = Arc::new(CapabilityManager::new( + Arc::new(CapabilityStore::new()), + audit.clone(), + Arc::new(MetricsManager::new()), + )); + let (registry, pending) = SessionRegistry::with_capabilities(audit, manager.clone()); + (registry, pending, manager) + } + + async fn lifecycle_grant( + pending: &PendingRequestStore, + manager: &CapabilityManager, + scope: impl Into, + ) -> (PendingCapabilityRequest, CapabilityToken) { + let request = pending + .admit_request( + manager, + scope, + ResourceId::new("localhost://Users/self/Documents/lifecycle"), + Action::Read, + "session lifecycle".to_string(), + ) + .await; + assert!(request.is_pending()); + let token = manager.grant( + request.session_id.as_str(), + request.resource.clone(), + request.action, + TokenConstraints::default(), + None, + ); + pending + .grant_request(request.id.as_str(), token.clone(), GrantDuration::Session) + .await + .unwrap(); + (request, token) + } + + #[tokio::test] + async fn test_ended_sessions_release_full_global_grant_capacity() { + let (registry, pending, manager) = lifecycle_store(); + let mut grants = Vec::new(); + for _ in 0..MAX_PENDING_REQUESTS / MAX_PENDING_PER_SESSION { + let session = registry.create_session(SessionType::Capsule, None).await; + for _ in 0..MAX_PENDING_PER_SESSION { + grants.push(lifecycle_grant(&pending, &manager, session.id.clone()).await); + } + registry + .get_session_mut(&session.token, |session| { + session.last_active = SecureTimestamp::at(0); + }) + .await + .unwrap(); + } + assert_eq!(grants.len(), 32 * 32); + assert_eq!( + pending.state.read().await.requests.len(), + MAX_PENDING_REQUESTS + ); + let fresh = registry.create_session(SessionType::Capsule, None).await; + assert!(request_for(&pending, fresh.id.as_str()).await.is_denied()); + + assert_eq!(registry.cleanup_stale_sessions(600).await, 32); + let admitted = pending + .admit_request( + &manager, + fresh.id.clone(), + ResourceId::new("localhost://Users/self/Documents/fresh"), + Action::Read, + String::new(), + ) + .await; + assert!( + admitted.is_pending(), + "ended sessions must release global grant capacity" + ); + for (request, token) in grants { + assert!(pending.get_request(request.id.as_str()).await.is_none()); + assert!(manager + .validate( + &token, + request.session_id.as_str(), + request.action, + &request.resource, + None + ) + .await + .is_err()); + } + let state = pending.state.read().await; + assert_eq!(state.requests.len(), 1); + assert_eq!(state.session_requests.len(), 1); + assert_eq!(state.active_sessions.as_ref().unwrap().len(), 1); + assert_eq!( + state.session_requests[&RequestScope::Session(fresh.id.clone())], + vec![admitted.id.0] + ); + } + + #[tokio::test] + async fn test_all_session_end_paths_retire_grants_and_preserve_active_sessions() { + for mode in 0..4 { + let (registry, pending, manager) = lifecycle_store(); + let ended = registry + .create_session(SessionType::Capsule, Some("ended".to_string())) + .await; + let active = registry + .create_session(SessionType::Capsule, Some("active".to_string())) + .await; + let (request, token) = lifecycle_grant(&pending, &manager, ended.id.clone()).await; + let (live_request, live_token) = + lifecycle_grant(&pending, &manager, active.id.clone()).await; + match mode { + 0 => { + registry.invalidate_session(&ended.token).await.unwrap(); + } + 1 => { + registry.invalidate_vm_session("ended").await.unwrap(); + } + 2 => { + assert_eq!( + registry + .cleanup_dead_vm_sessions(&["active".to_string()]) + .await, + 1 + ); + } + _ => { + registry + .get_session_mut(&ended.token, |session| { + session.last_active = SecureTimestamp::at(0) + }) + .await + .unwrap(); + assert_eq!(registry.cleanup_stale_sessions(600).await, 1); + } + } + assert!(registry.validate_token(&ended.token).await.is_none()); + assert!(registry.get_vm_token("ended").await.is_none()); + assert_eq!(registry.session_count().await, 1); + assert!(pending.get_request(request.id.as_str()).await.is_none()); + assert!(request_for(&pending, ended.id.as_str()).await.is_denied()); + assert!(manager + .validate( + &token, + ended.id.as_str(), + request.action, + &request.resource, + None + ) + .await + .is_err()); + assert!(registry.validate_token(&active.token).await.is_some()); + assert_eq!( + pending.list_session_granted(active.id.as_str()).await.len(), + 1 + ); + manager + .validate( + &live_token, + active.id.as_str(), + live_request.action, + &live_request.resource, + None, + ) + .await + .unwrap(); + } + } + + #[tokio::test] + async fn test_canceled_session_retirement_fails_closed_and_cleanup_retries() { + use std::future::Future as _; + use std::task::{Context, Waker}; + + let (registry, pending, manager) = lifecycle_store(); + let session = registry + .create_session(SessionType::Shell, Some("retiring".to_string())) + .await; + let (granted, token) = lifecycle_grant(&pending, &manager, session.id.clone()).await; + let late = request_for(&pending, session.id.as_str()).await; + let late_token = manager.grant( + session.id.as_str(), + late.resource.clone(), + late.action, + TokenConstraints::default(), + None, + ); + let mut retiring = Box::pin(tokio::task::unconstrained( + pending.retire_session_with(session.id.as_str(), |_| std::future::pending::<()>()), + )); + assert!(retiring + .as_mut() + .poll(&mut Context::from_waker(Waker::noop())) + .is_pending()); + drop(retiring); + + assert!(registry.validate_token(&session.token).await.is_none()); + assert!(!registry.is_shell(&session.token).await); + assert!(!registry.has_vm_session("retiring").await); + assert!(registry.get_vm_token("retiring").await.is_none()); + assert!(registry.list_sessions().await.is_empty()); + assert_eq!(registry.session_count().await, 0); + assert!(registry + .get_session_mut(&session.token, |_| ()) + .await + .is_none()); + registry.touch_session(&session.token).await; + assert!( + request_for(&pending, session.id.as_str()).await.is_denied(), + "ended session must reject late admission" + ); + assert!( + pending + .grant_request(late.id.as_str(), late_token, GrantDuration::Session) + .await + .is_err(), + "ended session must reject a late grant" + ); + // The canceled operation preserves the identities needed for retry. + assert!(pending.get_request(granted.id.as_str()).await.is_some()); + assert!(pending.get_request(late.id.as_str()).await.is_some()); + assert_eq!(registry.cleanup_stale_sessions(u64::MAX).await, 1); + assert!(manager + .validate( + &token, + session.id.as_str(), + granted.action, + &granted.resource, + None + ) + .await + .is_err()); + let state = pending.state.read().await; + assert!(state.requests.is_empty()); + assert!(state.session_requests.is_empty()); + assert!(state.active_sessions.as_ref().unwrap().is_empty()); + } + + #[tokio::test] + async fn test_session_retirement_preserves_capsule_scope_with_same_owner_string() { + let (registry, pending, manager) = lifecycle_store(); + let session = registry.create_session(SessionType::Capsule, None).await; + let mut session_grants = Vec::new(); + for _ in 0..MAX_PENDING_PER_SESSION { + session_grants.push(lifecycle_grant(&pending, &manager, session.id.clone()).await); + } + assert!(request_for(&pending, session.id.as_str()).await.is_denied()); + let (capsule_request, capsule_token) = lifecycle_grant( + &pending, + &manager, + RequestScope::Capsule(session.id.to_string()), + ) + .await; + assert_eq!( + pending + .list_session_granted(session.id.as_str()) + .await + .len(), + MAX_PENDING_PER_SESSION + ); + assert_eq!(pending.state.read().await.session_requests.len(), 2); + registry.invalidate_session(&session.token).await.unwrap(); + for (session_request, session_token) in session_grants { + assert!(pending + .get_request(session_request.id.as_str()) + .await + .is_none()); + assert!(manager + .validate( + &session_token, + session.id.as_str(), + session_request.action, + &session_request.resource, + None + ) + .await + .is_err()); + } + assert!(pending + .list_session_granted(session.id.as_str()) + .await + .is_empty()); + assert!(pending + .get_request(capsule_request.id.as_str()) + .await + .unwrap() + .is_granted()); + manager + .validate( + &capsule_token, + session.id.as_str(), + capsule_request.action, + &capsule_request.resource, + None, + ) + .await + .unwrap(); + assert_eq!( + pending.state.read().await.session_requests + [&RequestScope::Capsule(session.id.to_string())], + vec![capsule_request.id.0] + ); + let second_capsule = lifecycle_grant( + &pending, + &manager, + RequestScope::Capsule(session.id.to_string()), + ) + .await; + assert!(pending + .get_request(second_capsule.0.id.as_str()) + .await + .unwrap() + .is_granted()); + } + + #[tokio::test] + async fn test_epoch_retirement_preserves_grant_landing_after_advance() { + let (registry, pending, manager) = lifecycle_store(); + let session = registry.create_session(SessionType::Capsule, None).await; + let (old_request, old_token) = + lifecycle_grant(&pending, &manager, session.id.clone()).await; + let epoch = manager.revoke_all("test epoch race").unwrap(); + // Deterministically place a new-epoch grant in the handler's gap. + let (new_request, new_token) = + lifecycle_grant(&pending, &manager, session.id.clone()).await; + assert_eq!(new_token.constraints().epoch(), epoch); + pending.revoke_all_granted_before_epoch(epoch).await; + assert!(pending + .get_request(old_request.id.as_str()) + .await + .unwrap() + .is_denied()); + let visible = pending.list_session_granted(session.id.as_str()).await; + assert_eq!( + visible.len(), + 1, + "new-epoch grant must remain visible and revocable" + ); + assert_eq!(visible[0].id, new_request.id); + assert!(manager + .validate( + &old_token, + session.id.as_str(), + old_request.action, + &old_request.resource, + None + ) + .await + .is_err()); + manager + .validate( + &new_token, + session.id.as_str(), + new_request.action, + &new_request.resource, + None, + ) + .await + .unwrap(); + } fn create_test_store() -> PendingRequestStore { PendingRequestStore::new(Arc::new(AuditLog::new())) @@ -829,20 +1318,24 @@ mod tests { .read() .await .session_requests - .contains_key("expired-session")); + .contains_key(&RequestScope::Session(SessionId::from_string( + "expired-session" + )))); // Individual and epoch revocation make grant records eligible for cleanup. store.revoke_request(once.id.as_str()).await; assert_eq!(store.cleanup_old(0).await, 1); assert!(store.list_session_granted("mixed-session").await.is_empty()); - store.revoke_all_granted().await; + store.revoke_all_granted_before_epoch(1).await; assert_eq!(store.cleanup_old(0).await, 1); assert!(!store .state .read() .await .session_requests - .contains_key("grant-session")); + .contains_key(&RequestScope::Session(SessionId::from_string( + "grant-session" + )))); assert_eq!(store.cleanup_old(0).await, 0); assert_eq!( store.list_session_pending("mixed-session").await[0].id, @@ -904,7 +1397,7 @@ mod tests { ); assert_eq!(store.cleanup_old(0).await, 0); - store.revoke_all_granted().await; + store.revoke_all_granted_before_epoch(1).await; assert!(request_for(&store, "grant-session").await.is_pending()); assert!(store.list_session_granted("grant-session").await.is_empty()); } @@ -1014,50 +1507,53 @@ mod tests { assert!(request_for(&store, "owner").await.is_pending()); } let removed = store - .reclaim_retired_grants_with(Some("owner"), |token| { - let store = &store; - let granted = &granted; - async move { - // This fails immediately if either a read or write guard escaped - // the snapshot scope. It does not rely on a timeout or scheduler. - let mut state = store - .state - .try_write() - .expect("manager query holds pending lock"); - let index = granted - .iter() - .position(|(_, old)| old.id() == token.id()) - .unwrap(); - let request = state - .requests - .get_mut(granted[index].0.id.as_str()) - .unwrap(); - match index { - 0 => {} - 1 => { - request.status = RequestStatus::Denied { - reason: "changed during query".into(), + .reclaim_retired_grants_with( + Some(&RequestScope::Session(SessionId::from_string("owner"))), + |token| { + let store = &store; + let granted = &granted; + async move { + // This fails immediately if either a read or write guard escaped + // the snapshot scope. It does not rely on a timeout or scheduler. + let mut state = store + .state + .try_write() + .expect("manager query holds pending lock"); + let index = granted + .iter() + .position(|(_, old)| old.id() == token.id()) + .unwrap(); + let request = state + .requests + .get_mut(granted[index].0.id.as_str()) + .unwrap(); + match index { + 0 => {} + 1 => { + request.status = RequestStatus::Denied { + reason: "changed during query".into(), + } } - } - 2 => { - request.status = RequestStatus::Granted { - token: Box::new(token_for(request)), - duration: GrantDuration::Session, + 2 => { + request.status = RequestStatus::Granted { + token: Box::new(token_for(request)), + duration: GrantDuration::Session, + } } - } - 3 => { - request.status = RequestStatus::Granted { - token: Box::new(token), - duration: GrantDuration::Once, + 3 => { + request.status = RequestStatus::Granted { + token: Box::new(token), + duration: GrantDuration::Once, + } } + _ => unreachable!(), } - _ => unreachable!(), + drop(state); + tokio::task::yield_now().await; + true } - drop(state); - tokio::task::yield_now().await; - true - } - }) + }, + ) .await; assert_eq!(removed, 1); assert!(store.get_request(granted[0].0.id.as_str()).await.is_none()); @@ -1081,7 +1577,10 @@ mod tests { )); let state = store.state.read().await; assert_eq!(state.requests.len(), MAX_PENDING_PER_SESSION - 1); - assert_eq!(state.session_requests["owner"].len(), state.requests.len()); + assert_eq!( + state.session_requests[&RequestScope::Session(SessionId::from_string("owner"))].len(), + state.requests.len() + ); } #[tokio::test] @@ -1151,8 +1650,13 @@ mod tests { assert_eq!(counts.get_use_count(token.id()).await, *used); } let state = store.state.read().await; - assert!(!state.session_requests.contains_key("exhausted-only")); - assert_eq!(state.session_requests["mixed"].len(), 3); + assert!(!state.session_requests.contains_key(&RequestScope::Session( + SessionId::from_string("exhausted-only") + ))); + assert_eq!( + state.session_requests[&RequestScope::Session(SessionId::from_string("mixed"))].len(), + 3 + ); assert_eq!(state.requests.len(), MAX_PENDING_REQUESTS - 2); assert_eq!( state.session_requests.values().map(Vec::len).sum::(), @@ -1193,7 +1697,7 @@ mod tests { None, ); let epoch = manager.revoke_all("retire old epoch").unwrap(); - store.revoke_all_granted().await; + store.revoke_all_granted_before_epoch(epoch).await; // A grant minted before revocation can finish registering afterward. store .grant_request( @@ -1284,8 +1788,13 @@ mod tests { let state = store.state.read().await; assert!(!state.requests.contains_key(stale.id.as_str())); assert!(!state.requests.contains_key(revoked.id.as_str())); - assert!(!state.session_requests.contains_key("stale-epoch")); - assert_eq!(state.session_requests["mixed"].len(), 1); + assert!(!state.session_requests.contains_key(&RequestScope::Session( + SessionId::from_string("stale-epoch") + ))); + assert_eq!( + state.session_requests[&RequestScope::Session(SessionId::from_string("mixed"))].len(), + 1 + ); assert_eq!(state.requests.len(), MAX_PENDING_REQUESTS - 2); assert_eq!( state.session_requests.values().map(Vec::len).sum::(), @@ -1333,7 +1842,9 @@ mod tests { .read() .await .session_requests - .contains_key("expired-grant-session")); + .contains_key(&RequestScope::Session(SessionId::from_string( + "expired-grant-session" + )))); } #[tokio::test] diff --git a/elastos/crates/elastos-runtime/src/session/registry.rs b/elastos/crates/elastos-runtime/src/session/registry.rs index 316ccbcb6..a7ef1ea0e 100644 --- a/elastos/crates/elastos-runtime/src/session/registry.rs +++ b/elastos/crates/elastos-runtime/src/session/registry.rs @@ -7,6 +7,7 @@ use std::sync::Arc; use tokio::sync::RwLock; use super::{Session, SessionType}; +use crate::capability::{CapabilityManager, PendingRequestStore}; use crate::primitives::audit::AuditLog; use crate::primitives::time::SecureTimestamp; @@ -23,6 +24,10 @@ pub struct SessionRegistry { /// Audit log for session events audit_log: Arc, + + /// Capability ownership is present for Runtime session APIs. A gateway + /// control registry without capability APIs needs only session validation. + capabilities: Option<(Arc, Arc)>, } impl SessionRegistry { @@ -33,6 +38,34 @@ impl SessionRegistry { vm_sessions: RwLock::new(HashMap::new()), default_owner: RwLock::new(None), audit_log, + capabilities: None, + } + } + + /// Build one lifecycle owner for authenticated sessions and their grants. + /// Returning the bound pending store keeps server wiring on this owner. + pub fn with_capabilities( + audit_log: Arc, + manager: Arc, + ) -> (Self, Arc) { + let pending = Arc::new(PendingRequestStore::with_session_lifecycle( + audit_log.clone(), + )); + let mut registry = Self::new(audit_log); + registry.capabilities = Some((manager, pending.clone())); + (registry, pending) + } + + async fn session_is_active(&self, session: &Session) -> bool { + match &self.capabilities { + Some((_, pending)) => pending.session_is_active(session.id.as_str()).await, + None => true, + } + } + + async fn retire_capabilities(&self, session: &Session) { + if let Some((manager, pending)) = &self.capabilities { + pending.retire_session(manager, session.id.as_str()).await; } } @@ -59,6 +92,11 @@ impl SessionRegistry { // Store session { let mut sessions = self.sessions.write().await; + if let Some((_, pending)) = &self.capabilities { + pending.register_session(session.id.as_str()).await; + } + // Registration and publication have no intervening await. A + // canceled creation cannot leave an unowned active scope. sessions.insert(session.token.clone(), session.clone()); } @@ -83,7 +121,10 @@ impl SessionRegistry { /// Validate a bearer token and return the session if valid pub async fn validate_token(&self, token: &str) -> Option { let sessions = self.sessions.read().await; - sessions.get(token).cloned() + let session = sessions.get(token)?; + self.session_is_active(session) + .await + .then(|| session.clone()) } /// Get a session by token (mutable access for updates) @@ -92,43 +133,106 @@ impl SessionRegistry { F: FnOnce(&mut Session) -> R, { let mut sessions = self.sessions.write().await; - sessions.get_mut(token).map(f) + let session = sessions.get_mut(token)?; + self.session_is_active(session).await.then(|| f(session)) } /// Update last activity for a session pub async fn touch_session(&self, token: &str) { let mut sessions = self.sessions.write().await; if let Some(session) = sessions.get_mut(token) { - session.touch(); + if self.session_is_active(session).await { + session.touch(); + } } } /// Check if a token belongs to a shell session pub async fn is_shell(&self, token: &str) -> bool { - let sessions = self.sessions.read().await; - sessions.get(token).map(|s| s.is_shell()).unwrap_or(false) + self.validate_token(token) + .await + .is_some_and(|s| s.is_shell()) } /// Invalidate a session by token pub async fn invalidate_session(&self, token: &str) -> Option { + self.invalidate_session_with_reason(token, "invalidated".to_string(), None) + .await + } + + async fn invalidate_session_with_reason( + &self, + token: &str, + reason: String, + max_idle_secs: Option, + ) -> Option { + self.invalidate_session_with_retirement( + token, + reason, + max_idle_secs, + |session| async move { + self.retire_capabilities(&session).await; + }, + ) + .await + } + + async fn invalidate_session_with_retirement( + &self, + token: &str, + reason: String, + max_idle_secs: Option, + retire: F, + ) -> Option + where + F: FnOnce(Session) -> Fut, + Fut: std::future::Future, + { let session = { - let mut sessions = self.sessions.write().await; - sessions.remove(token) + let sessions = self.sessions.write().await; + let session = sessions.get(token)?; + if let Some(max_idle_secs) = max_idle_secs { + let idle_secs = SecureTimestamp::now() + .unix_secs + .saturating_sub(session.last_active.unix_secs); + if idle_secs <= max_idle_secs && self.session_is_active(session).await { + return None; + } + } + if let Some((_, pending)) = &self.capabilities { + pending.deactivate_session(session.id.as_str()).await; + } + session.clone() }; - if let Some(ref session) = session { - // Remove VM mapping if exists - if let Some(ref vm_id) = session.vm_id { - let mut vm_sessions = self.vm_sessions.write().await; - vm_sessions.remove(vm_id); + // Deactivation is serialized with touch/creation; revocation needs no + // registry or pending-state guard. Cancellation leaves the inactive + // entry for cleanup to retry without blocking unrelated sessions. + retire(session).await; + let session = { + // Acquire both guards before removing the retry owner. Session + // and matching VM mapping removal then have no cancellation gap. + let mut sessions = self.sessions.write().await; + let mut vm_sessions = self.vm_sessions.write().await; + let session = sessions.remove(token); + if let Some(vm_id) = session.as_ref().and_then(|session| session.vm_id.as_ref()) { + if vm_sessions + .get(vm_id) + .is_some_and(|current| current == token) + { + vm_sessions.remove(vm_id); + } } + session + }; + if let Some(ref session) = session { // Audit self.audit_log .emit(crate::primitives::audit::AuditEvent::SessionDestroyed { timestamp: SecureTimestamp::now(), session_id: session.id.to_string(), - reason: "invalidated".to_string(), + reason, }); } @@ -143,28 +247,8 @@ impl SessionRegistry { }; if let Some(token) = token { - let session = { - let mut sessions = self.sessions.write().await; - sessions.remove(&token) - }; - - // Remove VM mapping - { - let mut vm_sessions = self.vm_sessions.write().await; - vm_sessions.remove(vm_id); - } - - if let Some(ref session) = session { - // Audit - self.audit_log - .emit(crate::primitives::audit::AuditEvent::SessionDestroyed { - timestamp: SecureTimestamp::now(), - session_id: session.id.to_string(), - reason: format!("vm_terminated:{}", vm_id), - }); - } - - session + self.invalidate_session_with_reason(&token, format!("vm_terminated:{}", vm_id), None) + .await } else { None } @@ -173,72 +257,53 @@ impl SessionRegistry { /// Get all active sessions (for debugging/admin) pub async fn list_sessions(&self) -> Vec { let sessions = self.sessions.read().await; - sessions.values().cloned().collect() + let mut active = Vec::new(); + for session in sessions.values() { + if self.session_is_active(session).await { + active.push(session.clone()); + } + } + active } /// Get session count pub async fn session_count(&self) -> usize { - let sessions = self.sessions.read().await; - sessions.len() + self.list_sessions().await.len() } /// Check if a VM has an active session pub async fn has_vm_session(&self, vm_id: &str) -> bool { - let vm_sessions = self.vm_sessions.read().await; - vm_sessions.contains_key(vm_id) + self.get_vm_token(vm_id).await.is_some() } /// Get session token for a VM pub async fn get_vm_token(&self, vm_id: &str) -> Option { - let vm_sessions = self.vm_sessions.read().await; - vm_sessions.get(vm_id).cloned() + let token = self.vm_sessions.read().await.get(vm_id).cloned()?; + self.validate_token(&token).await.map(|_| token) } /// Cleanup stale sessions (called periodically) /// /// Removes sessions that haven't been active for the specified duration pub async fn cleanup_stale_sessions(&self, max_idle_secs: u64) -> usize { - let now = SecureTimestamp::now(); - let mut removed = Vec::new(); - - { - let mut sessions = self.sessions.write().await; - sessions.retain(|token, session| { - let idle_secs = now.unix_secs.saturating_sub(session.last_active.unix_secs); - if idle_secs > max_idle_secs { - removed.push((token.clone(), session.clone())); - false - } else { - true - } - }); - } - - // Remove VM mappings and audit - for (token, session) in &removed { - if let Some(ref vm_id) = session.vm_id { - let mut vm_sessions = self.vm_sessions.write().await; - vm_sessions.remove(vm_id); + let tokens: Vec<_> = self.sessions.read().await.keys().cloned().collect(); + let mut removed = 0; + for token in tokens { + // The helper rechecks activity while holding the writer, then + // retries inactive sessions regardless of their last-active time. + if self + .invalidate_session_with_reason( + &token, + "stale_cleanup".to_string(), + Some(max_idle_secs), + ) + .await + .is_some() + { + removed += 1; } - - self.audit_log - .emit(crate::primitives::audit::AuditEvent::SessionDestroyed { - timestamp: SecureTimestamp::now(), - session_id: session.id.to_string(), - reason: format!( - "stale_cleanup:idle_{}s", - now.unix_secs.saturating_sub(session.last_active.unix_secs) - ), - }); - - tracing::info!( - session_id = %session.id, - token = %token[..8.min(token.len())], - "Cleaned up stale session" - ); } - - removed.len() + removed } /// Check if sessions exist for a list of VM IDs and remove dead ones @@ -256,8 +321,12 @@ impl SessionRegistry { }; let mut count = 0; - for (vm_id, _) in vm_tokens { - if self.invalidate_vm_session(&vm_id).await.is_some() { + for (vm_id, token) in vm_tokens { + if self + .invalidate_session_with_reason(&token, format!("vm_terminated:{vm_id}"), None) + .await + .is_some() + { count += 1; } } @@ -274,6 +343,93 @@ mod tests { SessionRegistry::new(Arc::new(AuditLog::new())) } + #[tokio::test] + async fn test_canceled_registry_retirement_keeps_retry_owner_and_other_sessions_responsive() { + use crate::capability::{ + Action, CapabilityStore, GrantDuration, ResourceId, TokenConstraints, + }; + use crate::primitives::metrics::MetricsManager; + use std::future::Future as _; + use std::task::{Context, Waker}; + + let audit = Arc::new(AuditLog::new()); + let manager = Arc::new(CapabilityManager::new( + Arc::new(CapabilityStore::new()), + audit.clone(), + Arc::new(MetricsManager::new()), + )); + let (registry, pending) = SessionRegistry::with_capabilities(audit, manager.clone()); + let ended = registry + .create_session(SessionType::Capsule, Some("reused-vm".to_string())) + .await; + let request = pending + .admit_request( + &manager, + ended.id.clone(), + ResourceId::new("localhost://Users/self/Documents/test"), + Action::Read, + String::new(), + ) + .await; + let token = manager.grant( + ended.id.as_str(), + request.resource.clone(), + request.action, + TokenConstraints::default(), + None, + ); + pending + .grant_request(request.id.as_str(), token.clone(), GrantDuration::Session) + .await + .unwrap(); + + let mut retirement = Box::pin(tokio::task::unconstrained( + registry.invalidate_session_with_retirement( + &ended.token, + "test paused retirement".to_string(), + None, + |_| std::future::pending::<()>(), + ), + )); + assert!(retirement + .as_mut() + .poll(&mut Context::from_waker(Waker::noop())) + .is_pending()); + // The real registry path releases its writer before async revocation. + assert!( + registry.sessions.try_read().is_ok(), + "revocation must release the registry writer" + ); + let replacement = registry + .create_session(SessionType::Capsule, Some("reused-vm".to_string())) + .await; + assert!(registry.validate_token(&replacement.token).await.is_some()); + assert!(registry.validate_token(&ended.token).await.is_none()); + drop(retirement); + assert!( + registry.sessions.read().await.contains_key(&ended.token), + "canceled retirement must retain its retry owner" + ); + assert!(pending.get_request(request.id.as_str()).await.is_some()); + assert_eq!(registry.cleanup_stale_sessions(u64::MAX).await, 1); + assert!(manager + .validate( + &token, + ended.id.as_str(), + request.action, + &request.resource, + None + ) + .await + .is_err()); + assert!(pending.get_request(request.id.as_str()).await.is_none()); + assert!(!registry.sessions.read().await.contains_key(&ended.token)); + assert_eq!( + registry.get_vm_token("reused-vm").await, + Some(replacement.token) + ); + } + #[tokio::test] async fn test_create_session() { let registry = create_test_registry(); diff --git a/elastos/crates/elastos-server/src/api/handlers/capability.rs b/elastos/crates/elastos-server/src/api/handlers/capability.rs index b647df5e4..d975d7e5c 100755 --- a/elastos/crates/elastos-server/src/api/handlers/capability.rs +++ b/elastos/crates/elastos-server/src/api/handlers/capability.rs @@ -13,8 +13,8 @@ use serde::{Deserialize, Serialize}; use elastos_common::localhost::{is_supported_resource_scheme, is_system_only_backend_resource}; use elastos_runtime::capability::{ - pending::PendingRequestStore, Action, CapabilityManager, GrantDuration, PolicyEvaluator, - PolicyOutcome, ResourceId, TokenConstraints, + pending::PendingRequestStore, Action, CapabilityManager, CapabilityToken, GrantDuration, + PolicyEvaluator, PolicyOutcome, ResourceId, TokenConstraints, }; use elastos_runtime::session::Session; @@ -350,12 +350,7 @@ pub async fn grant_request( None, // No expiry for now (session-scoped) ); - // Mark request as granted - state - .pending_store - .grant_request(&input.request_id, token.clone(), duration) - .await - .map_err(|e| (StatusCode::INTERNAL_SERVER_ERROR, e))?; + record_grant(&state, &input.request_id, &token, duration).await?; Ok(Json(GrantRequestOutput { success: true, @@ -364,6 +359,29 @@ pub async fn grant_request( })) } +async fn record_grant( + state: &CapabilityState, + request_id: &str, + token: &CapabilityToken, + duration: GrantDuration, +) -> Result<(), (StatusCode, String)> { + if let Err(error) = state + .pending_store + .grant_request(request_id, token.clone(), duration) + .await + { + // The owner can end while the Shell's decision is in flight. Retire + // the minted token when its request can no longer record that grant. + state + .capability_manager + .revoke(*token.id(), "Grant registration failed") + .await; + return Err((StatusCode::INTERNAL_SERVER_ERROR, error)); + } + + Ok(()) +} + // === Deny Request (Shell Only) === #[derive(Debug, Deserialize)] @@ -555,8 +573,11 @@ pub async fn revoke_all_capabilities( ) })?; - // Mark all granted requests as revoked - state.pending_store.revoke_all_granted().await; + // A grant issued after advancement keeps its authority and visible record. + state + .pending_store + .revoke_all_granted_before_epoch(new_epoch) + .await; Ok(Json(RevokeAllOutput { success: true, @@ -790,6 +811,57 @@ mod tests { } } + #[tokio::test] + async fn test_failed_grant_registration_retires_unreturned_token() { + use elastos_runtime::session::{SessionRegistry, SessionType}; + + let mut state = test_state(); + let (registry, pending) = SessionRegistry::with_capabilities( + state.pending_store.audit_log(), + state.capability_manager.clone(), + ); + state.pending_store = pending; + let session = registry.create_session(SessionType::Capsule, None).await; + let request = state + .pending_store + .admit_request( + &state.capability_manager, + session.id.clone(), + ResourceId::new("localhost://Users/self/Documents/test"), + Action::Read, + String::new(), + ) + .await; + // The handler has read the pending request and minted a token before + // the session ends; recording that stale decision must retire it. + let token = state.capability_manager.grant( + session.id.as_str(), + request.resource.clone(), + request.action, + TokenConstraints::default(), + None, + ); + registry.invalidate_session(&session.token).await.unwrap(); + let error = record_grant(&state, request.id.as_str(), &token, GrantDuration::Session) + .await + .unwrap_err(); + assert_eq!(error.0, StatusCode::INTERNAL_SERVER_ERROR); + assert!( + state + .capability_manager + .validate( + &token, + session.id.as_str(), + request.action, + &request.resource, + None + ) + .await + .is_err(), + "unreturned token must be retired when grant registration fails" + ); + } + #[tokio::test] async fn test_revoke_all_failure_returns_500_and_keeps_live_grants() { use elastos_runtime::capability::{CapabilityStore, RequestStatus}; @@ -836,6 +908,9 @@ mod tests { .await .unwrap(); if blocked_path != "overflow" { + if blocked_path == "epoch" { + std::fs::remove_file(directory.path().join("epoch")).unwrap(); + } std::fs::create_dir(directory.path().join(blocked_path)).unwrap(); } let epoch = authority.current_epoch(); @@ -867,7 +942,7 @@ mod tests { .recent_events(100) .iter() .all(|event| !matches!(event, AuditEvent::EpochAdvance { .. }))); - state + let validation = state .capability_manager .validate( &token, @@ -876,10 +951,31 @@ mod tests { &request.resource, None, ) - .await - .unwrap(); + .await; + if blocked_path == "epoch" { + assert!( + validation.is_err(), + "unreadable epoch state must fail closed" + ); + } else { + validation.unwrap(); + } if blocked_path != "overflow" { std::fs::remove_dir(directory.path().join(blocked_path)).unwrap(); + if blocked_path == "epoch" { + std::fs::write(directory.path().join("epoch"), epoch.to_string()).unwrap(); + } + state + .capability_manager + .validate( + &token, + session.id.as_str(), + request.action, + &request.resource, + None, + ) + .await + .unwrap(); let recovered = revoke_all_capabilities( State(state.clone()), Extension(Session::new_shell("test-shell".to_string())), diff --git a/elastos/crates/elastos-server/src/resource_bridge.rs b/elastos/crates/elastos-server/src/resource_bridge.rs index ca620bddb..7e3cbf349 100644 --- a/elastos/crates/elastos-server/src/resource_bridge.rs +++ b/elastos/crates/elastos-server/src/resource_bridge.rs @@ -844,7 +844,9 @@ async fn handle_request(line: &str, ctx: &Option) -> Result { From 867fc12ec87e2a7fc2fe91e58eb91d60a2fd389c Mon Sep 17 00:00:00 2001 From: SashaMIT Date: Sun, 18 Oct 2026 13:27:20 +0000 Subject: [PATCH 09/11] Clarify rotation restart and capability store recovery Describe the durable epoch contract, state when a restart is required, and give actionable startup recovery guidance. Exercise command output and preserved epoch recovery. Co-authored-by: Codex --- SECURITY.md | 48 ++++-- docs/COMMAND_MATRIX.md | 2 +- .../elastos-runtime/src/capability/store.rs | 72 ++++++++- .../crates/elastos-server/src/security_cmd.rs | 138 +++++++++++++++--- 4 files changed, 220 insertions(+), 40 deletions(-) diff --git a/SECURITY.md b/SECURITY.md index 3249cd152..87028e8d3 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -48,24 +48,44 @@ operator. Use hosted accounts only for public demos; keep wallets, private data and recovery material on your own device until hosted protection passes its acceptance gate. Independent review and installed proof remain security gates. +## Capability revocation and key rotation + +Runtime stores the capability revocation epoch in `capability_store/epoch` +inside its data home. A successful epoch advance commits the new value before +publishing it in memory. Capability validation reads the stored epoch and +rejects tokens with a lower epoch, including after another process advances the +same store. +Epoch writes are serialized across processes and preserve the highest stored +or observed value. A stale process that persists its state preserves a newer +epoch already committed by another process. + +`elastos emergency rotate` advances that durable epoch before publishing the +replacement signing key. An epoch error stops rotation before key publication. +If key publication fails after the epoch advances, the command reports the +partial result and retains the higher epoch. The epoch and key are separate +writes. + +A running Runtime loads the replacement signing key only after restart. For a +complete rotation, stop Runtime for the affected data home, run +`elastos emergency rotate`, then start Runtime with the same data home. Confirm +that Runtime loads the stored key and starts successfully before resuming use. +Command completion confirms the persisted changes; the operator owns the +restart. Runtime can use an in-memory key when it cannot load or save a signing +key, so resolve signing-key warnings before relying on authority across restarts. + +An established capability store requires a readable, valid epoch at startup. +Capability validation also fails closed when that epoch is unavailable or +invalid. For recovery, stop all Runtime processes using the affected data home and preserve +the capability store. Restore valid state with its latest committed epoch from +trusted, current recovery data. Keep the epoch and lock files; resetting the +epoch or deleting the store can discard committed revocations. If the latest +committed epoch cannot be established, obtain recovery assistance before +starting Runtime. + ## Open Findings The following security-relevant findings remain open in the current runtime and are documented here for transparency. -### Capability state and key rotation are not restart safe - -**Severity:** Medium -**Files:** `elastos/crates/elastos-runtime/src/capability/manager.rs`, `elastos/crates/elastos-server/src/security_cmd.rs` -**Status:** Open - -Capability signing-key creation and `elastos emergency rotate` log persistence -errors but continue with in-memory state. The emergency command can therefore -report success even when the new key was not written. Rotation also advances a -fresh in-process capability store rather than a durable epoch record. Until the -key and revocation state are committed atomically, operators must verify the -persisted key and restart result instead of treating command completion as a -durable rotation receipt. - ### Host-plane Carrier providers use Runtime admission **Severity:** Medium diff --git a/docs/COMMAND_MATRIX.md b/docs/COMMAND_MATRIX.md index 2d558b953..d86590c82 100644 --- a/docs/COMMAND_MATRIX.md +++ b/docs/COMMAND_MATRIX.md @@ -53,7 +53,7 @@ target. | Configuration | `elastos config show`, `elastos config set` | Reads or writes local configuration. | | Identity | `elastos identity show`, `elastos identity nickname get`, `elastos identity nickname set`, `elastos identity arm-owner`, `elastos identity arm-owner-recovery` | Reads or updates the local DID-backed profile. `nickname set` prompts only when no value is supplied and a TTY is available. `arm-owner` admits one first-owner passkey at an exact public HTTPS Home address. `arm-owner-recovery` prints a one-time code for recovering this Home's admin with the Recovery Kit; use the command shown on the sign-in screen. | | TLS | `elastos tls trust`, `elastos tls regen` | Prints trust instructions or regenerates the local leaf certificate. | -| Emergency | `elastos emergency rotate` | Attempts to persist a new signing key. The active Runtime changes only after restart, and current persistence failure is not fail-closed. See the [security finding](../SECURITY.md#capability-state-and-key-rotation-are-not-restart-safe). | +| Emergency | `elastos emergency rotate` | Advances the durable capability revocation epoch, then persists a new signing key. Runtime validation reads the stored epoch; a running Runtime requires a restart to load the new key. The command reports persistence errors, including an epoch advance followed by a key publication failure. See [rotation and recovery](../SECURITY.md#capability-revocation-and-key-rotation). | | Trusted sources | `elastos source add`, `elastos source list`, `elastos source show`, `elastos source switch-channel`, `elastos source verify` | Manages local trusted-release source state. | | Updates | `elastos update` | Checks the trusted source over Carrier only. No local runtime is required. See [INSTALL.md](INSTALL.md#how-updates-reach-your-home). | | Offline principal-root maintenance | hidden `elastos principal-root-migrate`, hidden `elastos principal-root-upgrade` | Operates on an explicit data directory. The Runtime must be offline and the command requires explicit backup inputs. | diff --git a/elastos/crates/elastos-runtime/src/capability/store.rs b/elastos/crates/elastos-runtime/src/capability/store.rs index e020e1071..10c22ab04 100644 --- a/elastos/crates/elastos-runtime/src/capability/store.rs +++ b/elastos/crates/elastos-runtime/src/capability/store.rs @@ -314,7 +314,15 @@ impl CapabilityStore { self.write_epoch(0)?; 0 } - Err(error) => return Err(error), + Err(error) => { + return Err(std::io::Error::new(error.kind(), format!( + "Capability store startup refused: cannot load revocation epoch at {}: {error}. \ + Stop all Runtime processes that share this data directory and preserve the capability store. \ + Restore the latest committed epoch from trusted valid state. \ + If that value cannot be established, keep Runtime stopped and seek recovery help.", + path.join("epoch").display() + ))); + } }; self.set_epoch(epoch); } @@ -687,21 +695,71 @@ mod tests { let store = CapabilityStore::with_persistence(directory.path()) .await .unwrap(); + for _ in 0..7 { + store.advance_epoch().unwrap(); + } let epoch_path = directory.path().join("epoch"); + let trusted_epoch = fs::read(&epoch_path).unwrap(); if missing { fs::remove_file(&epoch_path).unwrap(); } else { fs::write(&epoch_path, "invalid").unwrap(); } - assert!(!store.is_epoch_valid(0)); + assert!(!store.is_epoch_valid(7)); assert!(store.advance_epoch().is_err()); assert!(store.persist().await.is_err()); - assert!(CapabilityStore::with_persistence(directory.path()) + let error = CapabilityStore::with_persistence(directory.path()) .await - .is_err()); - assert_eq!(store.current_epoch(), 0); - fs::write(&epoch_path, "0").unwrap(); - assert_eq!(store.advance_epoch().unwrap(), 1); + .err() + .expect("startup must refuse unreadable durable state"); + assert_eq!( + error.kind(), + if missing { + std::io::ErrorKind::NotFound + } else { + std::io::ErrorKind::InvalidData + } + ); + let message = error.to_string(); + assert!( + message.contains("Capability store startup refused"), + "{message}" + ); + assert!( + message.contains(&epoch_path.display().to_string()), + "{message}" + ); + assert!(message.contains("Stop all Runtime processes"), "{message}"); + assert!( + message.contains("preserve the capability store"), + "{message}" + ); + assert!( + message.contains("Restore the latest committed epoch from trusted valid state"), + "{message}" + ); + assert!( + message.contains("keep Runtime stopped and seek recovery help"), + "{message}" + ); + if missing { + assert!(!epoch_path.exists()); + } else { + assert_eq!(fs::read_to_string(&epoch_path).unwrap(), "invalid"); + } + assert!(directory.path().join("epoch.lock").is_file()); + assert_eq!(store.current_epoch(), 7); + drop(store); + + // Restore the last committed value from this fixture's trusted state. + fs::write(&epoch_path, trusted_epoch).unwrap(); + let repaired = CapabilityStore::with_persistence(directory.path()) + .await + .unwrap(); + assert_eq!(repaired.current_epoch(), 7); + assert!(!repaired.is_epoch_valid(6)); + assert!(repaired.is_epoch_valid(7)); + assert_eq!(repaired.advance_epoch().unwrap(), 8); } } diff --git a/elastos/crates/elastos-server/src/security_cmd.rs b/elastos/crates/elastos-server/src/security_cmd.rs index fe4e4a79b..32c399085 100644 --- a/elastos/crates/elastos-server/src/security_cmd.rs +++ b/elastos/crates/elastos-server/src/security_cmd.rs @@ -13,7 +13,7 @@ pub enum TlsCommand { #[derive(Subcommand)] pub enum EmergencyCommand { - /// Rotate signing key and invalidate all sessions/tokens + /// Save a new signing key and advance the token epoch; restart Runtime to apply the key Rotate { /// Reason for the emergency rotation #[arg(long, default_value = "key compromise")] @@ -57,12 +57,13 @@ async fn run_emergency_in(data_dir: &std::path::Path, cmd: EmergencyCommand) -> ); let new_pub = mgr.rotate_signing_key(data_dir, &reason)?; - println!("Signing key rotated."); + println!("New signing key saved to disk."); println!(" New public key: {}", hex::encode(new_pub)); - println!(" Epoch advanced: all prior tokens invalidated."); + println!(" Durable epoch advanced: tokens from earlier epochs are rejected."); println!(" Reason: {}", reason); println!(); - println!("Restart the runtime to apply the new key."); + println!("Restart required: restart every Runtime that uses this data directory to apply the new key."); + println!("Running Runtime processes retain their loaded signing key until restart."); } } @@ -73,6 +74,92 @@ async fn run_emergency_in(data_dir: &std::path::Path, cmd: EmergencyCommand) -> mod tests { use super::*; + #[test] + fn emergency_rotation_cli_reports_restart_requirement() { + const CHILD_DIR: &str = "ELASTOS_EMERGENCY_ROTATION_TEST_DIR"; + if let Some(directory) = std::env::var_os(CHILD_DIR) { + use clap::Parser as _; + + let cli = crate::Cli::try_parse_from([ + "elastos", + "emergency", + "rotate", + "--reason", + "isolated command fixture", + ]) + .unwrap(); + let Some(crate::Commands::Emergency(command)) = cli.command else { + panic!("the CLI must dispatch emergency rotation"); + }; + let runtime = tokio::runtime::Builder::new_current_thread() + .build() + .unwrap(); + runtime + .block_on(run_emergency_in(std::path::Path::new(&directory), command)) + .unwrap(); + return; + } + + let data_dir = tempfile::tempdir().unwrap(); + let runtime = tokio::runtime::Builder::new_current_thread() + .build() + .unwrap(); + let store_path = data_dir.path().join("capability_store"); + let store = runtime + .block_on(capability::CapabilityStore::with_persistence(&store_path)) + .unwrap(); + assert_eq!(store.advance_epoch().unwrap(), 1); + + let result = std::process::Command::new(std::env::current_exe().unwrap()) + .args([ + "--exact", + "security_cmd::tests::emergency_rotation_cli_reports_restart_requirement", + "--nocapture", + ]) + .env(CHILD_DIR, data_dir.path()) + .output() + .unwrap(); + assert!( + result.status.success(), + "rotation command failed: {}", + String::from_utf8_lossy(&result.stderr) + ); + let output = String::from_utf8_lossy(&result.stdout); + assert!(output.contains("running 1 test"), "{output}"); + assert!( + output.contains("New signing key saved to disk."), + "{output}" + ); + assert!( + output.contains("Durable epoch advanced: tokens from earlier epochs are rejected."), + "{output}" + ); + assert!( + output.contains("Restart required: restart every Runtime that uses this data directory to apply the new key."), + "{output}" + ); + assert!( + output.contains( + "Running Runtime processes retain their loaded signing key until restart." + ), + "{output}" + ); + assert!(!output.contains("all prior tokens invalidated"), "{output}"); + assert_eq!( + runtime + .block_on(capability::CapabilityStore::with_persistence(&store_path)) + .unwrap() + .current_epoch(), + 2 + ); + assert_eq!( + std::fs::read(data_dir.path().join("signing_key")) + .unwrap() + .len(), + 32 + ); + } + #[tokio::test] async fn emergency_rotation_uses_the_commands_persistent_store() { let data_dir = tempfile::tempdir().unwrap(); @@ -112,25 +199,40 @@ mod tests { #[tokio::test] async fn emergency_rotation_refuses_unreadable_state_before_creating_a_key() { - let data_dir = tempfile::tempdir().unwrap(); - let store_path = data_dir.path().join("capability_store"); - std::fs::create_dir(&store_path).unwrap(); - std::fs::write(store_path.join("epoch"), "invalid").unwrap(); - assert!( - run_emergency_in( + for missing in [true, false] { + let data_dir = tempfile::tempdir().unwrap(); + let store_path = data_dir.path().join("capability_store"); + let store = capability::CapabilityStore::with_persistence(&store_path) + .await + .unwrap(); + assert_eq!(store.advance_epoch().unwrap(), 1); + let epoch_path = store_path.join("epoch"); + if missing { + std::fs::remove_file(&epoch_path).unwrap(); + } else { + std::fs::write(&epoch_path, "invalid").unwrap(); + } + let error = run_emergency_in( data_dir.path(), EmergencyCommand::Rotate { reason: "isolated command fixture".into(), }, ) .await - .is_err(), - "the command must refuse malformed durable state" - ); - assert!(!data_dir.path().join("signing_key").exists()); - assert_eq!( - std::fs::read_to_string(store_path.join("epoch")).unwrap(), - "invalid" - ); + .expect_err("the command must refuse unreadable durable state") + .to_string(); + assert!( + error.contains("Capability store startup refused"), + "{error}" + ); + assert!(error.contains(&epoch_path.display().to_string()), "{error}"); + assert!(error.contains("latest committed epoch"), "{error}"); + assert!(!data_dir.path().join("signing_key").exists()); + if missing { + assert!(!epoch_path.exists()); + } else { + assert_eq!(std::fs::read_to_string(&epoch_path).unwrap(), "invalid"); + } + } } } From 4159892c004524904710801c3b75f2ada0b02d02 Mon Sep 17 00:00:00 2001 From: SashaMIT Date: Mon, 19 Oct 2026 10:01:22 +0000 Subject: [PATCH 10/11] Clarify Runtime recovery and rotation guidance Co-authored-by: Codex --- SECURITY.md | 12 ++- .../elastos-runtime/src/capability/manager.rs | 24 ++++- .../elastos-runtime/src/capability/store.rs | 99 +++++++++++++++++-- 3 files changed, 119 insertions(+), 16 deletions(-) diff --git a/SECURITY.md b/SECURITY.md index 87028e8d3..858eab393 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -76,11 +76,13 @@ key, so resolve signing-key warnings before relying on authority across restarts An established capability store requires a readable, valid epoch at startup. Capability validation also fails closed when that epoch is unavailable or invalid. For recovery, stop all Runtime processes using the affected data home and preserve -the capability store. Restore valid state with its latest committed epoch from -trusted, current recovery data. Keep the epoch and lock files; resetting the -epoch or deleting the store can discard committed revocations. If the latest -committed epoch cannot be established, obtain recovery assistance before -starting Runtime. +the capability store. Restore a valid unsigned 64-bit epoch at or above the +latest committed epoch established from trusted, current recovery data. A higher +epoch preserves committed revocations and can invalidate additional tokens; the +maximum value leaves no room for another epoch advance. Keep the epoch and lock +files; resetting the epoch or deleting the store can discard committed +revocations. If the latest committed epoch cannot be established, obtain +recovery assistance before starting Runtime. ## Open Findings diff --git a/elastos/crates/elastos-runtime/src/capability/manager.rs b/elastos/crates/elastos-runtime/src/capability/manager.rs index a15e505d6..f72d731fa 100644 --- a/elastos/crates/elastos-runtime/src/capability/manager.rs +++ b/elastos/crates/elastos-runtime/src/capability/manager.rs @@ -670,7 +670,7 @@ impl CapabilityManager { self.audit_log.security_warning( "signing_key_rotated", &format!( - "Signing key rotated (reason: {}). Epoch advanced {} → {}. All prior tokens invalidated.", + "Signing key rotated (reason: {}). Epoch advanced {} → {}; tokens from earlier epochs are rejected. Other Runtime processes using this data directory require restart to load the new key.", reason, old_epoch, new_epoch ), ); @@ -999,6 +999,28 @@ mod tests { .unwrap(); assert_ne!(replacement, original_key); assert_eq!(reloaded.current_epoch(), 2); + let rotation_details = reloaded + .audit_log() + .recent_events(100) + .into_iter() + .find_map(|event| match event { + crate::primitives::audit::AuditEvent::SecurityWarning { + warning_type, + details, + .. + } if warning_type == "signing_key_rotated" => Some(details), + _ => None, + }) + .expect("successful rotation must state its scope in the audit log"); + assert!(rotation_details.contains("Epoch advanced 1 → 2")); + assert!( + rotation_details.contains("tokens from earlier epochs are rejected"), + "rotation audit must scope epoch rejection: {rotation_details}" + ); + assert!(rotation_details.contains( + "Other Runtime processes using this data directory require restart to load the new key" + )); + assert!(!rotation_details.contains("All prior tokens invalidated")); let persisted = CapabilityManager::load_or_generate( directory.path(), Arc::new( diff --git a/elastos/crates/elastos-runtime/src/capability/store.rs b/elastos/crates/elastos-runtime/src/capability/store.rs index 10c22ab04..ba8606f56 100644 --- a/elastos/crates/elastos-runtime/src/capability/store.rs +++ b/elastos/crates/elastos-runtime/src/capability/store.rs @@ -20,6 +20,19 @@ struct EpochFileLock(fs::File); impl EpochFileLock { fn acquire(path: &Path, initialize: bool) -> std::io::Result<(Self, bool)> { + Self::acquire_inner(path, initialize).map_err(|error| { + std::io::Error::new(error.kind(), format!( + "Capability store refused: cannot acquire revocation epoch lock at {}: {error}. \ + Stop all Runtime processes that share this data directory and preserve the capability store. \ + Check directory ownership, permissions, and filesystem lock support. \ + Keep the epoch and lock files in place; retry after repairing access. \ + If repair is uncertain, keep Runtime stopped and seek recovery help.", + path.join("epoch.lock").display() + )) + }) + } + + fn acquire_inner(path: &Path, initialize: bool) -> std::io::Result<(Self, bool)> { let lock_path = path.join("epoch.lock"); let mut options = fs::OpenOptions::new(); options.read(true).write(true); @@ -318,7 +331,7 @@ impl CapabilityStore { return Err(std::io::Error::new(error.kind(), format!( "Capability store startup refused: cannot load revocation epoch at {}: {error}. \ Stop all Runtime processes that share this data directory and preserve the capability store. \ - Restore the latest committed epoch from trusted valid state. \ + Restore a valid unsigned 64-bit epoch at or above the latest committed epoch established from trusted valid state. \ If that value cannot be established, keep Runtime stopped and seek recovery help.", path.join("epoch").display() ))); @@ -689,8 +702,71 @@ mod tests { } #[tokio::test] - async fn test_missing_or_invalid_epoch_fails_closed_until_repaired() { + async fn test_epoch_lock_failure_preserves_state_and_explains_recovery() { for missing in [true, false] { + let directory = tempfile::tempdir().unwrap(); + let store = CapabilityStore::with_persistence(directory.path()) + .await + .unwrap(); + assert_eq!(store.advance_epoch().unwrap(), 1); + let epoch_path = directory.path().join("epoch"); + let lock_path = directory.path().join("epoch.lock"); + fs::remove_file(&lock_path).unwrap(); + if !missing { + fs::create_dir(&lock_path).unwrap(); + } + let expected_kind = fs::OpenOptions::new() + .read(true) + .write(true) + .open(&lock_path) + .unwrap_err() + .kind(); + let mut errors = vec![ + store.advance_epoch().unwrap_err(), + store.persist().await.unwrap_err(), + ]; + if !missing { + errors.push( + CapabilityStore::with_persistence(directory.path()) + .await + .err() + .expect("startup must refuse an unusable epoch lock"), + ); + } + for error in errors { + assert_eq!(error.kind(), expected_kind); + let message = error.to_string(); + assert!( + message.contains("cannot acquire revocation epoch lock"), + "epoch lock failure needs context: {message}" + ); + assert!( + message.contains(&lock_path.display().to_string()), + "{message}" + ); + assert!(message.contains("Stop all Runtime processes"), "{message}"); + assert!( + message.contains("preserve the capability store"), + "{message}" + ); + assert!( + message.contains("ownership, permissions, and filesystem lock support"), + "{message}" + ); + assert!( + message.contains("Keep the epoch and lock files in place"), + "{message}" + ); + } + assert_eq!(store.current_epoch(), 1); + assert_eq!(fs::read_to_string(&epoch_path).unwrap(), "1"); + assert_eq!(lock_path.exists(), !missing); + } + } + + #[tokio::test] + async fn test_missing_or_invalid_epoch_fails_closed_until_repaired() { + for (missing, repaired_epoch) in [(true, 7_u64), (false, 7), (true, 9), (false, 9)] { let directory = tempfile::tempdir().unwrap(); let store = CapabilityStore::with_persistence(directory.path()) .await @@ -699,7 +775,7 @@ mod tests { store.advance_epoch().unwrap(); } let epoch_path = directory.path().join("epoch"); - let trusted_epoch = fs::read(&epoch_path).unwrap(); + let trusted_epoch: u64 = fs::read_to_string(&epoch_path).unwrap().parse().unwrap(); if missing { fs::remove_file(&epoch_path).unwrap(); } else { @@ -735,7 +811,7 @@ mod tests { "{message}" ); assert!( - message.contains("Restore the latest committed epoch from trusted valid state"), + message.contains("Restore a valid unsigned 64-bit epoch at or above the latest committed epoch established from trusted valid state"), "{message}" ); assert!( @@ -751,15 +827,18 @@ mod tests { assert_eq!(store.current_epoch(), 7); drop(store); - // Restore the last committed value from this fixture's trusted state. - fs::write(&epoch_path, trusted_epoch).unwrap(); + // The fixture's latest trusted epoch is the lower bound for repair. + assert!(repaired_epoch >= trusted_epoch); + fs::write(&epoch_path, repaired_epoch.to_string()).unwrap(); let repaired = CapabilityStore::with_persistence(directory.path()) .await .unwrap(); - assert_eq!(repaired.current_epoch(), 7); - assert!(!repaired.is_epoch_valid(6)); - assert!(repaired.is_epoch_valid(7)); - assert_eq!(repaired.advance_epoch().unwrap(), 8); + assert_eq!(repaired.current_epoch(), repaired_epoch); + for revoked_epoch in 0..repaired_epoch { + assert!(!repaired.is_epoch_valid(revoked_epoch)); + } + assert!(repaired.is_epoch_valid(repaired_epoch)); + assert_eq!(repaired.advance_epoch().unwrap(), repaired_epoch + 1); } } From 629afe5bf6db8b106ad7680bc9e869848ce02661 Mon Sep 17 00:00:00 2001 From: SashaMIT Date: Mon, 19 Oct 2026 14:33:07 +0000 Subject: [PATCH 11/11] Clarify Runtime upgrade and recovery guidance Document legacy-state recovery and verify the documented restart path. Co-authored-by: Codex --- SECURITY.md | 23 +++++++--- docs/COMMAND_MATRIX.md | 2 +- .../elastos-runtime/src/capability/store.rs | 44 ++++++++++++++----- 3 files changed, 51 insertions(+), 18 deletions(-) diff --git a/SECURITY.md b/SECURITY.md index 858eab393..89e72b459 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -75,14 +75,23 @@ key, so resolve signing-key warnings before relying on authority across restarts An established capability store requires a readable, valid epoch at startup. Capability validation also fails closed when that epoch is unavailable or -invalid. For recovery, stop all Runtime processes using the affected data home and preserve -the capability store. Restore a valid unsigned 64-bit epoch at or above the -latest committed epoch established from trusted, current recovery data. A higher +invalid. For recovery, stop all Runtime processes using the affected data home +and preserve the capability store. Restore a valid unsigned 64-bit epoch at or +above the latest committed epoch established from trusted, current recovery data. A higher epoch preserves committed revocations and can invalidate additional tokens; the -maximum value leaves no room for another epoch advance. Keep the epoch and lock -files; resetting the epoch or deleting the store can discard committed -revocations. If the latest committed epoch cannot be established, obtain -recovery assistance before starting Runtime. +maximum value leaves no room for another epoch advance. Preserve the epoch and +any existing lock file. If only `epoch.lock` is missing, stop every Runtime +process using that data home, then restart Runtime to recreate the lock without +changing the valid stored epoch. Resetting the epoch or deleting the store can +discard committed revocations. If the latest committed epoch cannot be +established, obtain recovery assistance before starting Runtime. + +Upgrade note: Older Runtime versions could start at epoch zero when a stored +epoch was empty or malformed. Runtime now refuses that state; if an upgrade +exposes it, preserve the store and use the recovery procedure above before +restarting. A truncated value that still parses as a number cannot be detected +from its syntax alone, so establish the recovery lower bound from trusted, +current data. ## Open Findings diff --git a/docs/COMMAND_MATRIX.md b/docs/COMMAND_MATRIX.md index d86590c82..ecdff1f60 100644 --- a/docs/COMMAND_MATRIX.md +++ b/docs/COMMAND_MATRIX.md @@ -53,7 +53,7 @@ target. | Configuration | `elastos config show`, `elastos config set` | Reads or writes local configuration. | | Identity | `elastos identity show`, `elastos identity nickname get`, `elastos identity nickname set`, `elastos identity arm-owner`, `elastos identity arm-owner-recovery` | Reads or updates the local DID-backed profile. `nickname set` prompts only when no value is supplied and a TTY is available. `arm-owner` admits one first-owner passkey at an exact public HTTPS Home address. `arm-owner-recovery` prints a one-time code for recovering this Home's admin with the Recovery Kit; use the command shown on the sign-in screen. | | TLS | `elastos tls trust`, `elastos tls regen` | Prints trust instructions or regenerates the local leaf certificate. | -| Emergency | `elastos emergency rotate` | Advances the durable capability revocation epoch, then persists a new signing key. Runtime validation reads the stored epoch; a running Runtime requires a restart to load the new key. The command reports persistence errors, including an epoch advance followed by a key publication failure. See [rotation and recovery](../SECURITY.md#capability-revocation-and-key-rotation). | +| Emergency | `elastos emergency rotate` | Advances the durable capability revocation epoch, then persists a new signing key. Runtime validation reads the stored epoch; a running Runtime requires a restart to load the new key. The command reports persistence errors, including an epoch advance followed by a key publication failure. See [rotation, upgrade and recovery guidance](../SECURITY.md#capability-revocation-and-key-rotation). | | Trusted sources | `elastos source add`, `elastos source list`, `elastos source show`, `elastos source switch-channel`, `elastos source verify` | Manages local trusted-release source state. | | Updates | `elastos update` | Checks the trusted source over Carrier only. No local runtime is required. See [INSTALL.md](INSTALL.md#how-updates-reach-your-home). | | Offline principal-root maintenance | hidden `elastos principal-root-migrate`, hidden `elastos principal-root-upgrade` | Operates on an explicit data directory. The Runtime must be offline and the command requires explicit backup inputs. | diff --git a/elastos/crates/elastos-runtime/src/capability/store.rs b/elastos/crates/elastos-runtime/src/capability/store.rs index ba8606f56..9fe15fa07 100644 --- a/elastos/crates/elastos-runtime/src/capability/store.rs +++ b/elastos/crates/elastos-runtime/src/capability/store.rs @@ -25,7 +25,9 @@ impl EpochFileLock { "Capability store refused: cannot acquire revocation epoch lock at {}: {error}. \ Stop all Runtime processes that share this data directory and preserve the capability store. \ Check directory ownership, permissions, and filesystem lock support. \ - Keep the epoch and lock files in place; retry after repairing access. \ + Preserve the epoch and any existing lock file. \ + If only the lock file is missing and the stored epoch is valid, restart Runtime to recreate the lock. \ + Otherwise repair access before restarting. \ If repair is uncertain, keep Runtime stopped and seek recovery help.", path.join("epoch.lock").display() )) @@ -754,19 +756,37 @@ mod tests { "{message}" ); assert!( - message.contains("Keep the epoch and lock files in place"), + message.contains("Preserve the epoch and any existing lock file"), "{message}" ); } assert_eq!(store.current_epoch(), 1); assert_eq!(fs::read_to_string(&epoch_path).unwrap(), "1"); assert_eq!(lock_path.exists(), !missing); + if missing { + drop(store); + let restarted = CapabilityStore::with_persistence(directory.path()) + .await + .unwrap(); + assert!(lock_path.is_file()); + assert_eq!(fs::read_to_string(&epoch_path).unwrap(), "1"); + assert_eq!(restarted.current_epoch(), 1); + assert!(!restarted.is_epoch_valid(0)); + assert_eq!(restarted.advance_epoch().unwrap(), 2); + } } } #[tokio::test] async fn test_missing_or_invalid_epoch_fails_closed_until_repaired() { - for (missing, repaired_epoch) in [(true, 7_u64), (false, 7), (true, 9), (false, 9)] { + for (contents, missing_lock, repaired_epoch) in [ + (None, false, 7_u64), + (Some("invalid"), false, 7), + (None, false, 9), + (Some("invalid"), false, 9), + (Some(""), true, 7), + (Some(""), true, 9), + ] { let directory = tempfile::tempdir().unwrap(); let store = CapabilityStore::with_persistence(directory.path()) .await @@ -776,10 +796,14 @@ mod tests { } let epoch_path = directory.path().join("epoch"); let trusted_epoch: u64 = fs::read_to_string(&epoch_path).unwrap().parse().unwrap(); - if missing { - fs::remove_file(&epoch_path).unwrap(); + if let Some(contents) = contents { + fs::write(&epoch_path, contents).unwrap(); } else { - fs::write(&epoch_path, "invalid").unwrap(); + fs::remove_file(&epoch_path).unwrap(); + } + if missing_lock { + // Older stores can have an empty epoch without a lock file. + fs::remove_file(directory.path().join("epoch.lock")).unwrap(); } assert!(!store.is_epoch_valid(7)); assert!(store.advance_epoch().is_err()); @@ -790,7 +814,7 @@ mod tests { .expect("startup must refuse unreadable durable state"); assert_eq!( error.kind(), - if missing { + if contents.is_none() { std::io::ErrorKind::NotFound } else { std::io::ErrorKind::InvalidData @@ -818,10 +842,10 @@ mod tests { message.contains("keep Runtime stopped and seek recovery help"), "{message}" ); - if missing { - assert!(!epoch_path.exists()); + if let Some(contents) = contents { + assert_eq!(fs::read_to_string(&epoch_path).unwrap(), contents); } else { - assert_eq!(fs::read_to_string(&epoch_path).unwrap(), "invalid"); + assert!(!epoch_path.exists()); } assert!(directory.path().join("epoch.lock").is_file()); assert_eq!(store.current_epoch(), 7);