Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 19 additions & 11 deletions src/Aspire.Hosting.Radius/Publishing/SealedSecretApplyStep.cs
Original file line number Diff line number Diff line change
Expand Up @@ -124,11 +124,13 @@ private static async Task ApplyStoreAsync(
// fresh one. The apply and verify kubectl calls are wrapped with the same remaining-budget
// helper the poll loop already uses, which cancels the linked token (killing the child
// process) when the budget is exhausted.
var deadline = DateTimeOffset.UtcNow + store.MaterializationTimeout;
var timeProvider = TimeProvider.System;
var deadline = timeProvider.GetUtcNow() + store.MaterializationTimeout;

var appliedGeneration = await InvokeProbeWithRemainingBudgetAsync(
ct => ApplyManifestAsync(validated.Content, applyNamespace, kubeContext, store.Name, metadata.Namespace, metadata.Name, manifestPath, logger, ct),
RemainingBudget(deadline),
RemainingBudget(deadline, timeProvider),
timeProvider,
cancellationToken,
() => CreateOperationTimeoutException(store.Name, metadata.Namespace, metadata.Name, "apply", store.MaterializationTimeout))
.ConfigureAwait(false);
Expand All @@ -137,6 +139,7 @@ await WaitForSealedSecretSyncedAsync(
store.Name, metadata.Namespace, metadata.Name, appliedGeneration, deadline, store.MaterializationTimeout, s_pollInterval,
ct => GetSealedSecretStatusAsync(metadata.Namespace, metadata.Name, kubeContext, ct),
ct => SecretExistsAsync(metadata.Namespace, metadata.Name, kubeContext, ct),
timeProvider,
cancellationToken).ConfigureAwait(false);

// The SealedSecret controller can report Synced=True and create a Secret that is missing keys
Expand All @@ -147,7 +150,8 @@ await WaitForSealedSecretSyncedAsync(
{
var dataKeys = await InvokeProbeWithRemainingBudgetAsync(
ct => GetSecretDataKeysAsync(metadata.Namespace, metadata.Name, kubeContext, ct),
RemainingBudget(deadline),
RemainingBudget(deadline, timeProvider),
timeProvider,
cancellationToken,
() => CreateOperationTimeoutException(store.Name, metadata.Namespace, metadata.Name, "verify", store.MaterializationTimeout))
.ConfigureAwait(false);
Expand Down Expand Up @@ -268,13 +272,15 @@ internal static async Task WaitForSealedSecretSyncedAsync(
TimeSpan interval,
Func<CancellationToken, Task<SealedSecretStatusSnapshot>> getStatus,
Func<CancellationToken, Task<bool>> secretExists,
TimeProvider timeProvider,
CancellationToken cancellationToken)
{
while (true)
{
var status = await InvokeProbeWithRemainingBudgetAsync(
getStatus,
RemainingBudget(deadline),
RemainingBudget(deadline, timeProvider),
timeProvider,
cancellationToken,
() => CreateSealedSecretSyncTimeoutException(storeName, ns, name, appliedGeneration, timeout))
.ConfigureAwait(false);
Expand All @@ -283,7 +289,8 @@ internal static async Task WaitForSealedSecretSyncedAsync(
{
if (await InvokeProbeWithRemainingBudgetAsync(
secretExists,
RemainingBudget(deadline),
RemainingBudget(deadline, timeProvider),
timeProvider,
cancellationToken,
() => CreateSealedSecretSyncTimeoutException(storeName, ns, name, appliedGeneration, timeout))
.ConfigureAwait(false))
Expand All @@ -298,15 +305,15 @@ internal static async Task WaitForSealedSecretSyncedAsync(
$"failed to sync generation {appliedGeneration}: {decision.Message}. Diagnostic: ASPIRERADIUS058.");
}

var remaining = RemainingBudget(deadline);
var remaining = RemainingBudget(deadline, timeProvider);
if (remaining <= TimeSpan.Zero)
{
throw CreateSealedSecretSyncTimeoutException(storeName, ns, name, appliedGeneration, timeout);
}

try
{
await Task.Delay(remaining < interval ? remaining : interval, cancellationToken).ConfigureAwait(false);
await Task.Delay(remaining < interval ? remaining : interval, timeProvider, cancellationToken).ConfigureAwait(false);
}
catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested)
{
Expand All @@ -318,6 +325,7 @@ internal static async Task WaitForSealedSecretSyncedAsync(
internal static async Task<T> InvokeProbeWithRemainingBudgetAsync<T>(
Func<CancellationToken, Task<T>> probe,
TimeSpan remaining,
TimeProvider timeProvider,
CancellationToken cancellationToken,
Func<InvalidOperationException> createTimeoutException)
{
Expand All @@ -326,20 +334,20 @@ internal static async Task<T> InvokeProbeWithRemainingBudgetAsync<T>(
throw createTimeoutException();
}

using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
linkedCts.CancelAfter(remaining);
using var timeoutCts = new CancellationTokenSource(remaining, timeProvider);
using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken, timeoutCts.Token);

try
{
return await probe(linkedCts.Token).ConfigureAwait(false);
}
catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested && linkedCts.IsCancellationRequested)
catch (OperationCanceledException) when (!cancellationToken.IsCancellationRequested && timeoutCts.IsCancellationRequested)
{
throw createTimeoutException();
}
}

private static TimeSpan RemainingBudget(DateTimeOffset deadline) => deadline - DateTimeOffset.UtcNow;
private static TimeSpan RemainingBudget(DateTimeOffset deadline, TimeProvider timeProvider) => deadline - timeProvider.GetUtcNow();

private static InvalidOperationException CreateSealedSecretSyncTimeoutException(
string storeName,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
</PropertyGroup>

<ItemGroup>
<PackageReference Include="Microsoft.Extensions.TimeProvider.Testing" />
<PackageReference Include="Verify.XunitV3" />
</ItemGroup>

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -5,12 +5,15 @@

using System.Text.Json;
using Aspire.Hosting.Radius.Publishing;
using Microsoft.Extensions.Time.Testing;
using Xunit.Sdk;

namespace Aspire.Hosting.Radius.Tests.Secrets;

public class SealedSecretApplyStepTests
{
private static readonly TimeSpan s_hangGuard = TimeSpan.FromSeconds(30);

[Fact]
public void BuildApplyArgs_WithContext_PassesContextExplicitly()
{
Expand Down Expand Up @@ -317,6 +320,7 @@ await SealedSecretApplyStep.WaitForSealedSecretSyncedAsync(
""", StdErr: ""));
}),
secretExists: _ => Task.FromResult(true),
timeProvider: TimeProvider.System,
cancellationToken: default);

Assert.Equal(2, statusProbeCalls);
Expand Down Expand Up @@ -474,6 +478,7 @@ await SealedSecretApplyStep.WaitForSealedSecretSyncedAsync(
secretCalls++;
return Task.FromResult(secretCalls >= 2);
},
timeProvider: TimeProvider.System,
cancellationToken: default);

Assert.True(statusCalls >= 3);
Expand All @@ -494,6 +499,7 @@ public async Task WaitForSealedSecretSynced_FailsFastWhenSyncedFalseForAppliedGe
4,
[new SealedSecretApplyStep.SealedSecretCondition("Synced", "False", "no key could decrypt secret")])),
secretExists: _ => Task.FromResult(false),
timeProvider: TimeProvider.System,
cancellationToken: default));

Assert.Contains("ASPIRERADIUS058", ex.Message);
Expand All @@ -511,6 +517,7 @@ public async Task WaitForSealedSecretSynced_TimesOutWhenStatusNeverMatches_Throw
interval: TimeSpan.FromMilliseconds(1),
getStatus: _ => Task.FromResult(new SealedSecretApplyStep.SealedSecretStatusSnapshot(4, 3, [])),
secretExists: _ => Task.FromResult(false),
timeProvider: TimeProvider.System,
cancellationToken: default));

Assert.Contains("ASPIRERADIUS058", ex.Message);
Expand All @@ -536,45 +543,55 @@ await SealedSecretApplyStep.WaitForSealedSecretSyncedAsync(
5,
[new SealedSecretApplyStep.SealedSecretCondition("Synced", "True", null)])),
secretExists: _ => Task.FromResult(true),
timeProvider: TimeProvider.System,
cancellationToken: default);
}

[Fact]
public async Task WaitForSealedSecretSynced_StatusMatchesButSecretAbsent_KeepsWaitingThenTimesOut()
{
// The budget is driven by a FakeTimeProvider rather than the wall clock. A wall-clock budget
// (previously 10ms, then 250ms) was flaky under CI load: a slow first iteration (cold JIT,
// thread-pool starvation) could consume the whole budget before the second poll, so the
// loop timed out after observing the absent Secret only once. With fake time, the budget only
// elapses when the test advances it, after the loop has demonstrably kept waiting.
var timeProvider = new FakeTimeProvider();
var budget = TimeSpan.FromMilliseconds(250);
var secondProbeStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously);
var secretCalls = 0;
var ex = await Assert.ThrowsAsync<InvalidOperationException>(() =>
SealedSecretApplyStep.WaitForSealedSecretSyncedAsync(
"db-creds", "app", "db-creds", appliedGeneration: 4,
// Use a comfortable timeout (not a tiny wall-clock budget) and drive the timeout
// deterministically from the probe. A previous 10ms budget was flaky under CI load:
// a slow cold-JIT first iteration could consume the whole budget before a second
// poll, so 'secretCalls > 1' occasionally saw only one call. Here the first probe
// reports the Secret absent (so the loop keeps waiting past one iteration), then the
// second probe hangs until the remaining-budget guard cancels it and surfaces the
// ASPIRERADIUS058 timeout — guaranteeing at least two polls regardless of runner speed.
deadline: DateTimeOffset.UtcNow + TimeSpan.FromMilliseconds(250),
timeout: TimeSpan.FromMilliseconds(250),
interval: TimeSpan.FromMilliseconds(1),
getStatus: _ => Task.FromResult(new SealedSecretApplyStep.SealedSecretStatusSnapshot(
4,
4,
[new SealedSecretApplyStep.SealedSecretCondition("Synced", "True", null)])),
secretExists: async ct =>
{
secretCalls++;
if (secretCalls == 1)
{
return false;
}

await Task.Delay(Timeout.Infinite, ct).ConfigureAwait(false);
var waitTask = SealedSecretApplyStep.WaitForSealedSecretSyncedAsync(
"db-creds", "app", "db-creds", appliedGeneration: 4,
deadline: timeProvider.GetUtcNow() + budget,
timeout: budget,
interval: TimeSpan.Zero,
getStatus: _ => Task.FromResult(new SealedSecretApplyStep.SealedSecretStatusSnapshot(
4,
4,
[new SealedSecretApplyStep.SealedSecretCondition("Synced", "True", null)])),
secretExists: async ct =>
{
if (Interlocked.Increment(ref secretCalls) == 1)
{
return false;
},
cancellationToken: default));
}

secondProbeStarted.TrySetResult();
await Task.Delay(Timeout.Infinite, ct).ConfigureAwait(false);
return false;
},
timeProvider: timeProvider,
cancellationToken: default);

await secondProbeStarted.Task.WaitAsync(s_hangGuard);
Assert.False(waitTask.IsCompleted);

// Exhausting the budget cancels the hanging probe and surfaces the ASPIRERADIUS058 timeout.
timeProvider.Advance(budget);

var ex = await Assert.ThrowsAsync<InvalidOperationException>(() => waitTask.WaitAsync(s_hangGuard));
Assert.Contains("ASPIRERADIUS058", ex.Message);
Assert.True(secretCalls > 1);
Assert.Equal(2, Volatile.Read(ref secretCalls));
}

[Fact]
Expand All @@ -593,6 +610,7 @@ public async Task WaitForSealedSecretSynced_HangingProbeTimesOutWith_ASPIRERADIU
return new SealedSecretApplyStep.SealedSecretStatusSnapshot(4, null, []);
},
secretExists: _ => Task.FromResult(false),
timeProvider: TimeProvider.System,
cancellationToken: default));

stopwatch.Stop();
Expand All @@ -614,6 +632,7 @@ await Assert.ThrowsAnyAsync<OperationCanceledException>(() =>
interval: TimeSpan.FromMilliseconds(1),
getStatus: _ => Task.FromResult(new SealedSecretApplyStep.SealedSecretStatusSnapshot(4, null, [])),
secretExists: _ => Task.FromResult(false),
timeProvider: TimeProvider.System,
cancellationToken: cts.Token));
}

Expand Down Expand Up @@ -868,6 +887,7 @@ public async Task InvokeProbeWithRemainingBudget_HangingApply_CancelledWithin_AS
return 0;
},
remaining: TimeSpan.FromMilliseconds(50),
timeProvider: TimeProvider.System,
cancellationToken: default,
createTimeoutException: () => SealedSecretApplyStep.CreateOperationTimeoutException(
"db-creds", "app", "db-creds", "apply", TimeSpan.FromMilliseconds(50))));
Expand All @@ -894,6 +914,7 @@ await Assert.ThrowsAnyAsync<OperationCanceledException>(() =>
return 0;
},
remaining: TimeSpan.FromSeconds(5),
timeProvider: TimeProvider.System,
cancellationToken: cts.Token,
createTimeoutException: () => SealedSecretApplyStep.CreateOperationTimeoutException(
"db-creds", "app", "db-creds", "verify", TimeSpan.FromSeconds(5))));
Expand Down
Loading