diff --git a/src/Aspire.Hosting.Radius/Publishing/SealedSecretApplyStep.cs b/src/Aspire.Hosting.Radius/Publishing/SealedSecretApplyStep.cs index 576ecef16e7..bb1c0532ec8 100644 --- a/src/Aspire.Hosting.Radius/Publishing/SealedSecretApplyStep.cs +++ b/src/Aspire.Hosting.Radius/Publishing/SealedSecretApplyStep.cs @@ -124,11 +124,13 @@ private static async Task ApplyStoreAsync( // fresh one. The apply and verify kubectl calls are wrapped with the same remaining-budget // helper the poll loop already uses, which cancels the linked token (killing the child // process) when the budget is exhausted. - var deadline = DateTimeOffset.UtcNow + store.MaterializationTimeout; + var timeProvider = TimeProvider.System; + var deadline = timeProvider.GetUtcNow() + store.MaterializationTimeout; var appliedGeneration = await InvokeProbeWithRemainingBudgetAsync( ct => ApplyManifestAsync(validated.Content, applyNamespace, kubeContext, store.Name, metadata.Namespace, metadata.Name, manifestPath, logger, ct), - RemainingBudget(deadline), + RemainingBudget(deadline, timeProvider), + timeProvider, cancellationToken, () => CreateOperationTimeoutException(store.Name, metadata.Namespace, metadata.Name, "apply", store.MaterializationTimeout)) .ConfigureAwait(false); @@ -137,6 +139,7 @@ await WaitForSealedSecretSyncedAsync( store.Name, metadata.Namespace, metadata.Name, appliedGeneration, deadline, store.MaterializationTimeout, s_pollInterval, ct => GetSealedSecretStatusAsync(metadata.Namespace, metadata.Name, kubeContext, ct), ct => SecretExistsAsync(metadata.Namespace, metadata.Name, kubeContext, ct), + timeProvider, cancellationToken).ConfigureAwait(false); // The SealedSecret controller can report Synced=True and create a Secret that is missing keys @@ -147,7 +150,8 @@ await WaitForSealedSecretSyncedAsync( { var dataKeys = await InvokeProbeWithRemainingBudgetAsync( ct => GetSecretDataKeysAsync(metadata.Namespace, metadata.Name, kubeContext, ct), - RemainingBudget(deadline), + RemainingBudget(deadline, timeProvider), + timeProvider, cancellationToken, () => CreateOperationTimeoutException(store.Name, metadata.Namespace, metadata.Name, "verify", store.MaterializationTimeout)) .ConfigureAwait(false); @@ -268,13 +272,15 @@ internal static async Task WaitForSealedSecretSyncedAsync( TimeSpan interval, Func> getStatus, Func> secretExists, + TimeProvider timeProvider, CancellationToken cancellationToken) { while (true) { var status = await InvokeProbeWithRemainingBudgetAsync( getStatus, - RemainingBudget(deadline), + RemainingBudget(deadline, timeProvider), + timeProvider, cancellationToken, () => CreateSealedSecretSyncTimeoutException(storeName, ns, name, appliedGeneration, timeout)) .ConfigureAwait(false); @@ -283,7 +289,8 @@ internal static async Task WaitForSealedSecretSyncedAsync( { if (await InvokeProbeWithRemainingBudgetAsync( secretExists, - RemainingBudget(deadline), + RemainingBudget(deadline, timeProvider), + timeProvider, cancellationToken, () => CreateSealedSecretSyncTimeoutException(storeName, ns, name, appliedGeneration, timeout)) .ConfigureAwait(false)) @@ -298,7 +305,7 @@ internal static async Task WaitForSealedSecretSyncedAsync( $"failed to sync generation {appliedGeneration}: {decision.Message}. Diagnostic: ASPIRERADIUS058."); } - var remaining = RemainingBudget(deadline); + var remaining = RemainingBudget(deadline, timeProvider); if (remaining <= TimeSpan.Zero) { throw CreateSealedSecretSyncTimeoutException(storeName, ns, name, appliedGeneration, timeout); @@ -306,7 +313,7 @@ internal static async Task WaitForSealedSecretSyncedAsync( try { - await Task.Delay(remaining < interval ? remaining : interval, cancellationToken).ConfigureAwait(false); + await Task.Delay(remaining < interval ? remaining : interval, timeProvider, cancellationToken).ConfigureAwait(false); } catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested) { @@ -318,6 +325,7 @@ internal static async Task WaitForSealedSecretSyncedAsync( internal static async Task InvokeProbeWithRemainingBudgetAsync( Func> probe, TimeSpan remaining, + TimeProvider timeProvider, CancellationToken cancellationToken, Func createTimeoutException) { @@ -326,20 +334,20 @@ internal static async Task InvokeProbeWithRemainingBudgetAsync( throw createTimeoutException(); } - using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken); - linkedCts.CancelAfter(remaining); + using var timeoutCts = new CancellationTokenSource(remaining, timeProvider); + using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, timeoutCts.Token); try { return await probe(linkedCts.Token).ConfigureAwait(false); } - catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested && linkedCts.IsCancellationRequested) + catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested && timeoutCts.IsCancellationRequested) { throw createTimeoutException(); } } - private static TimeSpan RemainingBudget(DateTimeOffset deadline) => deadline - DateTimeOffset.UtcNow; + private static TimeSpan RemainingBudget(DateTimeOffset deadline, TimeProvider timeProvider) => deadline - timeProvider.GetUtcNow(); private static InvalidOperationException CreateSealedSecretSyncTimeoutException( string storeName, diff --git a/tests/Aspire.Hosting.Radius.Tests/Aspire.Hosting.Radius.Tests.csproj b/tests/Aspire.Hosting.Radius.Tests/Aspire.Hosting.Radius.Tests.csproj index d85f7bee14f..5591130f1f2 100644 --- a/tests/Aspire.Hosting.Radius.Tests/Aspire.Hosting.Radius.Tests.csproj +++ b/tests/Aspire.Hosting.Radius.Tests/Aspire.Hosting.Radius.Tests.csproj @@ -15,6 +15,7 @@ + diff --git a/tests/Aspire.Hosting.Radius.Tests/Secrets/SealedSecretApplyStepTests.cs b/tests/Aspire.Hosting.Radius.Tests/Secrets/SealedSecretApplyStepTests.cs index 790ff65259a..76ca271af64 100644 --- a/tests/Aspire.Hosting.Radius.Tests/Secrets/SealedSecretApplyStepTests.cs +++ b/tests/Aspire.Hosting.Radius.Tests/Secrets/SealedSecretApplyStepTests.cs @@ -5,12 +5,15 @@ using System.Text.Json; using Aspire.Hosting.Radius.Publishing; +using Microsoft.Extensions.Time.Testing; using Xunit.Sdk; namespace Aspire.Hosting.Radius.Tests.Secrets; public class SealedSecretApplyStepTests { + private static readonly TimeSpan s_hangGuard = TimeSpan.FromSeconds(30); + [Fact] public void BuildApplyArgs_WithContext_PassesContextExplicitly() { @@ -317,6 +320,7 @@ await SealedSecretApplyStep.WaitForSealedSecretSyncedAsync( """, StdErr: "")); }), secretExists: _ => Task.FromResult(true), + timeProvider: TimeProvider.System, cancellationToken: default); Assert.Equal(2, statusProbeCalls); @@ -474,6 +478,7 @@ await SealedSecretApplyStep.WaitForSealedSecretSyncedAsync( secretCalls++; return Task.FromResult(secretCalls >= 2); }, + timeProvider: TimeProvider.System, cancellationToken: default); Assert.True(statusCalls >= 3); @@ -494,6 +499,7 @@ public async Task WaitForSealedSecretSynced_FailsFastWhenSyncedFalseForAppliedGe 4, [new SealedSecretApplyStep.SealedSecretCondition("Synced", "False", "no key could decrypt secret")])), secretExists: _ => Task.FromResult(false), + timeProvider: TimeProvider.System, cancellationToken: default)); Assert.Contains("ASPIRERADIUS058", ex.Message); @@ -511,6 +517,7 @@ public async Task WaitForSealedSecretSynced_TimesOutWhenStatusNeverMatches_Throw interval: TimeSpan.FromMilliseconds(1), getStatus: _ => Task.FromResult(new SealedSecretApplyStep.SealedSecretStatusSnapshot(4, 3, [])), secretExists: _ => Task.FromResult(false), + timeProvider: TimeProvider.System, cancellationToken: default)); Assert.Contains("ASPIRERADIUS058", ex.Message); @@ -536,45 +543,55 @@ await SealedSecretApplyStep.WaitForSealedSecretSyncedAsync( 5, [new SealedSecretApplyStep.SealedSecretCondition("Synced", "True", null)])), secretExists: _ => Task.FromResult(true), + timeProvider: TimeProvider.System, cancellationToken: default); } [Fact] public async Task WaitForSealedSecretSynced_StatusMatchesButSecretAbsent_KeepsWaitingThenTimesOut() { + // The budget is driven by a FakeTimeProvider rather than the wall clock. A wall-clock budget + // (previously 10ms, then 250ms) was flaky under CI load: a slow first iteration (cold JIT, + // thread-pool starvation) could consume the whole budget before the second poll, so the + // loop timed out after observing the absent Secret only once. With fake time, the budget only + // elapses when the test advances it, after the loop has demonstrably kept waiting. + var timeProvider = new FakeTimeProvider(); + var budget = TimeSpan.FromMilliseconds(250); + var secondProbeStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var secretCalls = 0; - var ex = await Assert.ThrowsAsync(() => - SealedSecretApplyStep.WaitForSealedSecretSyncedAsync( - "db-creds", "app", "db-creds", appliedGeneration: 4, - // Use a comfortable timeout (not a tiny wall-clock budget) and drive the timeout - // deterministically from the probe. A previous 10ms budget was flaky under CI load: - // a slow cold-JIT first iteration could consume the whole budget before a second - // poll, so 'secretCalls > 1' occasionally saw only one call. Here the first probe - // reports the Secret absent (so the loop keeps waiting past one iteration), then the - // second probe hangs until the remaining-budget guard cancels it and surfaces the - // ASPIRERADIUS058 timeout — guaranteeing at least two polls regardless of runner speed. - deadline: DateTimeOffset.UtcNow + TimeSpan.FromMilliseconds(250), - timeout: TimeSpan.FromMilliseconds(250), - interval: TimeSpan.FromMilliseconds(1), - getStatus: _ => Task.FromResult(new SealedSecretApplyStep.SealedSecretStatusSnapshot( - 4, - 4, - [new SealedSecretApplyStep.SealedSecretCondition("Synced", "True", null)])), - secretExists: async ct => - { - secretCalls++; - if (secretCalls == 1) - { - return false; - } - await Task.Delay(Timeout.Infinite, ct).ConfigureAwait(false); + var waitTask = SealedSecretApplyStep.WaitForSealedSecretSyncedAsync( + "db-creds", "app", "db-creds", appliedGeneration: 4, + deadline: timeProvider.GetUtcNow() + budget, + timeout: budget, + interval: TimeSpan.Zero, + getStatus: _ => Task.FromResult(new SealedSecretApplyStep.SealedSecretStatusSnapshot( + 4, + 4, + [new SealedSecretApplyStep.SealedSecretCondition("Synced", "True", null)])), + secretExists: async ct => + { + if (Interlocked.Increment(ref secretCalls) == 1) + { return false; - }, - cancellationToken: default)); + } + + secondProbeStarted.TrySetResult(); + await Task.Delay(Timeout.Infinite, ct).ConfigureAwait(false); + return false; + }, + timeProvider: timeProvider, + cancellationToken: default); + + await secondProbeStarted.Task.WaitAsync(s_hangGuard); + Assert.False(waitTask.IsCompleted); + + // Exhausting the budget cancels the hanging probe and surfaces the ASPIRERADIUS058 timeout. + timeProvider.Advance(budget); + var ex = await Assert.ThrowsAsync(() => waitTask.WaitAsync(s_hangGuard)); Assert.Contains("ASPIRERADIUS058", ex.Message); - Assert.True(secretCalls > 1); + Assert.Equal(2, Volatile.Read(ref secretCalls)); } [Fact] @@ -593,6 +610,7 @@ public async Task WaitForSealedSecretSynced_HangingProbeTimesOutWith_ASPIRERADIU return new SealedSecretApplyStep.SealedSecretStatusSnapshot(4, null, []); }, secretExists: _ => Task.FromResult(false), + timeProvider: TimeProvider.System, cancellationToken: default)); stopwatch.Stop(); @@ -614,6 +632,7 @@ await Assert.ThrowsAnyAsync(() => interval: TimeSpan.FromMilliseconds(1), getStatus: _ => Task.FromResult(new SealedSecretApplyStep.SealedSecretStatusSnapshot(4, null, [])), secretExists: _ => Task.FromResult(false), + timeProvider: TimeProvider.System, cancellationToken: cts.Token)); } @@ -868,6 +887,7 @@ public async Task InvokeProbeWithRemainingBudget_HangingApply_CancelledWithin_AS return 0; }, remaining: TimeSpan.FromMilliseconds(50), + timeProvider: TimeProvider.System, cancellationToken: default, createTimeoutException: () => SealedSecretApplyStep.CreateOperationTimeoutException( "db-creds", "app", "db-creds", "apply", TimeSpan.FromMilliseconds(50)))); @@ -894,6 +914,7 @@ await Assert.ThrowsAnyAsync(() => return 0; }, remaining: TimeSpan.FromSeconds(5), + timeProvider: TimeProvider.System, cancellationToken: cts.Token, createTimeoutException: () => SealedSecretApplyStep.CreateOperationTimeoutException( "db-creds", "app", "db-creds", "verify", TimeSpan.FromSeconds(5))));