Require High CPU to persist: port the built-in gauge alerts onto AlertPersistenceGate (#3282)

Darling/Darling.Tests/AlertReadFailureSurfaceTests.cs

@@ -359,7 +359,7 @@ real failure rather than a matcher that never matches anything. */
     /// </summary>
     private static readonly (string Path, int Counted, int Exempt)[] s_wholeFileScopes =
     {
-        (Path.Combine("PerformanceMonitor.Alerting", "AlertEngine.cs"), 13, 4),
+        (Path.Combine("PerformanceMonitor.Alerting", "AlertEngine.cs"), 13, 5),
         (Path.Combine("Darling", "PerformanceMonitor.Darling.Service", "DarlingSelfAlertEvaluator.cs"), 5, 8),
     };
 
@@ -416,6 +416,7 @@ private static readonly (string Path, int Counted, int Exempt)[] s_wholeFileScop
     {
         ["Could not load incident occurrences"] = "bookkeeping about an alert, not the condition read it is judged on",
         ["Could not persist incident occurrences"] = "a write",
+        ["Could not persist the CPU persistence gate"] = "a write (#3282); the gate has already decided the observation from its in-memory record, so a dropped save costs the streak across a restart and never an alert - the seeding LOAD beside it is the read, and it is counted",
         ["Alert resolution callback failed"] = "the delivery path",
         ["Connection-change self-alert delivery failed"] = "the delivery path",
         ["Store disk-pressure self-alert failed"] = "handed its evidence as parameters; the read is counted in DarlingWorker",
@@ -519,7 +520,7 @@ an off-by-one on a total. */
         /* The whole-tree totals, so a site MOVED between the scoped regions still has to be re-counted by
            a person rather than netting out silently. */
         Assert.Equal(CountedSites, totalCounted);
-        Assert.Equal(19, totalExempt);
+        Assert.Equal(20, totalExempt);
 
         /* Every exemption in the table is actually used. An exemption for a message that no longer exists
            is a hole this pin would otherwise keep open indefinitely — the shape that lets a real new catch

Darling/Darling.Tests/CustomAlertCoreMigrationTests.cs

@@ -17,8 +17,10 @@ namespace Darling.Tests;
 
 /// <summary>
 /// V116 / #3285: the custom-alert core rung (config.custom_alert_rules + config.custom_alert_state). The
-/// "I am the top rung" claims live on <see cref="MuteRuleReloadBeaconTests"/> (V117); this rung is strictly
-/// below the top, and its probe arm has to keep reporting 116 for a store migrated exactly this far.
+/// "I am the top rung" claims live on whichever rung is currently top — <see cref="MuteRuleReloadBeaconTests"/>
+/// took them from here at V117 and <see cref="BuiltinAlertPersistenceRungTests"/> holds them at V118. This
+/// rung is strictly below the top either way, and its probe arm has to keep reporting 116 for a store
+/// migrated exactly this far.
 /// </summary>
 public sealed class CustomAlertCoreMigrationTests
 {

Darling/Darling.Tests/DarlingAlertingTests.cs

@@ -265,7 +265,9 @@ AlertEngine BuildEngine(RecordingDeliverer deliverer, MuteRuleService muteRuleSe
             var snapshot = new AlertServerSnapshot(
                 TestServerKey, TestServerName,
                 IsOnline: true, SqlCpuPercent: null, TotalCpuPercent: null,
-                IsAzureSqlDb: false, Suppressed: false);
+                IsAzureSqlDb: false, Suppressed: false,
+                /* No CPU value at all here, so the #3282 gate never observes anything on this path. */
+                CpuSampleTimeUtc: null);
 
             /* --- first sweep: the deadlock and the poison wait fire, unmuted --- */
             var (deliverer, engine) = await BuildStackAsync();

Darling/Darling.Tests/DarlingSelfAlertTests.cs

@@ -2129,14 +2129,26 @@ await history.RecordAlertAsync(DarlingSelfAlertEvaluator.BuildResolutionRecord(r
             logger: null,
             utcNow: () => now);
 
-        /* Fire: total CPU 90 >= 80. */
-        await engine.EvaluateServerAsync(new AlertServerSnapshot(Key, Name, IsOnline: true, 90, 90, false, false), Ct);
+        /* Fire: total CPU 90 >= 80, held for AlertEngine.CpuBreachSamples distinct samples (#3282 — one
+           sample over the bar is no longer an incident). */
+        var sampleAt = new DateTime(2026, 7, 1, 0, 0, 0, DateTimeKind.Utc);
+        for (var i = 0; i < AlertEngine.CpuBreachSamples; i++)
+        {
+            sampleAt = sampleAt.AddMinutes(1);
+            await engine.EvaluateServerAsync(new AlertServerSnapshot(Key, Name, IsOnline: true, 90, 90, false, false, sampleAt), Ct);
+        }
+
         Assert.Single(deliverer.Outcomes);
         Assert.Empty(history.Records); /* no resolution yet */
 
-        /* Clear: CPU back below threshold => the engine emits a resolution => a history row is written. */
+        /* Clear: CPU back below threshold for AlertEngine.CpuClearSamples distinct samples => the engine
+           emits a resolution => a history row is written. */
         now = now.AddMinutes(1);
-        await engine.EvaluateServerAsync(new AlertServerSnapshot(Key, Name, IsOnline: true, 10, 10, false, false), Ct);
+        for (var i = 0; i < AlertEngine.CpuClearSamples; i++)
+        {
+            sampleAt = sampleAt.AddMinutes(1);
+            await engine.EvaluateServerAsync(new AlertServerSnapshot(Key, Name, IsOnline: true, 10, 10, false, false, sampleAt), Ct);
+        }
         var resolved = Assert.Single(history.Records);
         Assert.Equal("CPU Resolved", resolved.MetricName);
         Assert.Equal(AlertDelivery.ChannelNotApplicable, resolved.NotificationType);
@@ -2194,6 +2206,21 @@ public Task<IReadOnlyDictionary<string, IncidentOccurrenceState>> LoadIncidentOc
                 new Dictionary<string, IncidentOccurrenceState>(StringComparer.Ordinal));
 
         public Task SaveIncidentOccurrencesAsync(string serverKey, string metricName, IReadOnlyDictionary<string, IncidentOccurrenceState> states) => Task.CompletedTask;
+
+        /* #3282: real, for the same reason the other two fakes are. These tests drive the SELF-alert paths
+           and never the CPU check, so nothing here reads it back — but a stub that answered "no memory" to
+           a load and swallowed every save is indistinguishable from the seam being wired wrong, and this
+           class already has one stub-shaped no-op above that had to be justified in a comment. */
+        public Dictionary<(string Key, string Metric), AlertPersistenceRecord> Persistence { get; } = new();
+
+        public Task<AlertPersistenceRecord?> LoadAlertPersistenceAsync(string serverKey, string metricName) =>
+            Task.FromResult(Persistence.TryGetValue((serverKey, metricName), out var r) ? (AlertPersistenceRecord?)r : null);
+
+        public Task SaveAlertPersistenceAsync(string serverKey, string metricName, AlertPersistenceRecord record)
+        {
+            Persistence[(serverKey, metricName)] = record;
+            return Task.CompletedTask;
+        }
     }
 
     /* ---------------- live collection_log reads (gated on DARLING_TEST_PG) ---------------- */

Darling/Darling.Tests/MuteRuleReloadBeaconTests.cs

@@ -28,9 +28,11 @@ namespace Darling.Tests;
 /// confirm it — the rule list, the tool's own success reply — reads the TABLE, so they all agree the mute is
 /// in place while matching alerts keep being delivered.</para>
 ///
-/// <para>This class also carries the "I am the top rung" claims, handed over from
-/// <see cref="CustomAlertCoreMigrationTests"/> (V116) when this rung landed: a fully-migrated store must map
-/// to EXACTLY this version, or the viewer's connect-time gate refuses a store that is current.</para>
+/// <para>The "I am the top rung" claims have moved on to <see cref="BuiltinAlertPersistenceRungTests"/>
+/// (V118), the way this rung took them from <see cref="CustomAlertCoreMigrationTests"/> (V116) — the
+/// documented hand-over when a later rung merges. What stays here is this rung's own identity and its probe
+/// arm, which must keep mapping a store migrated to EXACTLY 117 to 117 rather than letting it fall
+/// through.</para>
 /// </summary>
 public sealed class MuteRuleReloadBeaconTests
 {
@@ -51,7 +53,10 @@ public void TheRungIsRegisteredAtTheTopOfADenseLadder()
 
         Assert.Equal(StorageVersion.SchemaVersion, PgMigrations.Scripts[^1].Version);
         Assert.Equal(StorageVersion.SchemaVersion, versions.Max());
-        Assert.Equal(RungVersion, StorageVersion.SchemaVersion);
+
+        /* V118 (#3282) is the top rung now, so the "== SchemaVersion" claim lives there. Strictly LESS
+           rather than <=, so this cannot silently become the top-rung claim again. */
+        Assert.True(RungVersion < StorageVersion.SchemaVersion);
 
         Assert.Equal(versions.Distinct().OrderBy(v => v), versions);
         var above = versions.Where(v => v > 45).OrderBy(v => v).ToList();
@@ -127,23 +132,26 @@ which it is looking at. */
         Assert.Contains($"reader.GetBoolean({ProbeOrdinal})", viewer, StringComparison.Ordinal);
         Assert.Contains("hasMuteRuleReloadBeacon", viewer, StringComparison.Ordinal);
 
-        Assert.Equal(StorageVersion.SchemaVersion, ViewerDataService.RequiredStoreSchemaVersion);
-
         var method = typeof(ViewerDataService)
             .GetMethod("MapProbedSchemaVersion", BindingFlags.NonPublic | BindingFlags.Static)!;
         var arity = method.GetParameters().Length;
 
-        /* The top rung's sentinel IS the last argument. */
-        Assert.Equal(ProbeOrdinal, arity - 1);
+        /* Not the top rung any more, so this sentinel is not the last argument. */
+        Assert.True(ProbeOrdinal < arity - 1);
+
+        /* A store migrated to exactly V117 — this rung's sentinel true and every LATER one false — maps to
+           117. Everything above this ordinal is turned off, so further rungs do not have to touch this. */
+        var toThisRung = Enumerable.Repeat((object)true, arity).ToArray();
+        for (var i = ProbeOrdinal + 1; i < arity; i++)
+        {
+            toThisRung[i] = false;
+        }
 
-        /* Every sentinel true = a fully-migrated store, which must map to exactly this version. Built by
-           reflection so the arity tracks the signature. */
-        var all = Enumerable.Repeat((object)true, arity).ToArray();
-        Assert.Equal(StorageVersion.SchemaVersion, (int)method.Invoke(null, all)!);
+        Assert.Equal(RungVersion, (int)method.Invoke(null, toThisRung)!);
 
-        /* One rung behind: every sentinel EXCEPT this one reports 116 (the previous top rung). Without this
-           the arm above could be satisfied by an unconditional return and nothing would notice. */
-        var behind = Enumerable.Repeat((object)true, arity).ToArray();
+        /* One rung behind: this sentinel AND every later one false must report 116. Without it the arm
+           above could be satisfied by an unconditional return and nothing would notice. */
+        var behind = (object[])toThisRung.Clone();
         behind[ProbeOrdinal] = false;
         Assert.Equal(PreviousVersion, (int)method.Invoke(null, behind)!);
     }

Darling/Darling.Tests/PgCpuCapacityHeadroomTests.cs

@@ -604,8 +604,15 @@ public void TheAlertTextNamesWhatEachPercentageIsAFractionOf()
         Assert.Contains("of currently allocated capacity", body, StringComparison.Ordinal);
 
         /* The metric NAMES are deliberately unchanged, for the engine-parity reason #2719 recorded: a mute
-           rule or history filter built on these strings keeps working across the fix. */
-        Assert.Contains("\"High CPU\"", body, StringComparison.Ordinal);
+           rule or history filter built on these strings keeps working across the fix.
+
+           #3282 made the fire name a REFERENCE to AlertEngine.CpuPersistenceMetric rather than a second
+           literal, which is a stronger form of the same claim: the two engines now use one string by
+           construction instead of two that happen to match, and the persistence-gate row is the same
+           subject on both. The literal itself is pinned where it now lives, in
+           AlertEngineTests.CpuGateDefaults_AreDerivedFromTheSampleCadence. */
+        Assert.Contains("AlertEngine.CpuPersistenceMetric", body, StringComparison.Ordinal);
+        Assert.Equal("High CPU", PerformanceMonitor.Alerting.AlertEngine.CpuPersistenceMetric);
         Assert.Contains("\"CPU Resolved\"", body, StringComparison.Ordinal);
     }
 
@@ -629,7 +636,7 @@ private static string EvaluatePgCpuBody()
 
         /* The slice has to be a method, not a fragment: an off-by-one on either bound silently shrinks it
            and every Assert.DoesNotContain above starts passing for the wrong reason. */
-        Assert.Contains("var reading = await DarlingPgCpuUtilizationReader.GetLatestAsync", body, StringComparison.Ordinal);
+        Assert.Contains("await DarlingPgCpuUtilizationReader.GetSamplesSinceAsync", body, StringComparison.Ordinal);
         Assert.True(body.Length > 1500, $"the sliced body is only {body.Length} chars, which cannot be this method");
 
         return body;

Darling/Darling.Tests/RepoFileAdoptionTests.cs

@@ -110,6 +110,7 @@ public sealed class RepoFileAdoptionTests
     /// </summary>
     private static readonly string[] s_lfReaders =
     {
+        "BuiltinAlertPersistenceRungTests.cs",
         "ChartWindowDomainTests.cs",
         "DarlingPathFilterGateTests.cs",
         "FleetCardCollectionStaleNamesItsPopulationTests.cs",