diff --git a/services/nvpair-node-info/macmon_supervisor.go b/services/nvpair-node-info/macmon_supervisor.go index 57e193816..cdd735ae7 100644 --- a/services/nvpair-node-info/macmon_supervisor.go +++ b/services/nvpair-node-info/macmon_supervisor.go @@ -22,6 +22,9 @@ type macmonConfig struct { BackoffMin time.Duration BackoffMax time.Duration CheckEvery time.Duration + + // waitBackoff defaults to the context-aware timer in waitMacmon. + waitBackoff func(context.Context, time.Duration) bool } // macmonSupervisor owns one child at a time. Configuration durations must be @@ -41,6 +44,9 @@ type macmonSupervisor struct { func newMacmonSupervisor(cfg macmonConfig) *macmonSupervisor { cfg.Paths = append([]string(nil), cfg.Paths...) + if cfg.waitBackoff == nil { + cfg.waitBackoff = waitMacmon + } return &macmonSupervisor{cfg: cfg, status: MacmonMissing} } @@ -86,7 +92,7 @@ func (s *macmonSupervisor) Run(ctx context.Context) { s.mu.Lock() s.status = MacmonFailed s.mu.Unlock() - if !waitMacmon(ctx, backoff) { + if !s.cfg.waitBackoff(ctx, backoff) { return } if backoff >= s.cfg.BackoffMax-backoff { diff --git a/services/nvpair-node-info/macmon_supervisor_test.go b/services/nvpair-node-info/macmon_supervisor_test.go index acbf0910f..24ec2f84a 100644 --- a/services/nvpair-node-info/macmon_supervisor_test.go +++ b/services/nvpair-node-info/macmon_supervisor_test.go @@ -447,35 +447,44 @@ func TestMacmonSupervisorBackoffDoublesCapsAndResets(t *testing.T) { [ ! -f "$DIR/count" ] || read -r count < "$DIR/count" count=$((count + 1)) printf '%s\n' "$count" > "$DIR/count" -[ "$count" -ne 4 ] || printf '%s\n' "$LINE" -[ "$count" -lt 5 ] || exec sleep 30 +[ "$count" -ne 5 ] || printf '%s\n' "$LINE" exit 1`) cfg := macmonTestConfig(path) + cfg.Interval = func() time.Duration { return time.Hour } cfg.BackoffMin = 50 * time.Millisecond cfg.BackoffMax = 150 * time.Millisecond + want := []time.Duration{ + 50 * time.Millisecond, 100 * time.Millisecond, + 150 * time.Millisecond, 150 * time.Millisecond, + 50 * time.Millisecond, 100 * time.Millisecond, + } + var delays []time.Duration + // Observe the actual scheduled delays, without process startup or runner + // scheduling latency. Stop at the last wait so no extra child can start. + cfg.waitBackoff = func(ctx context.Context, delay time.Duration) bool { + delays = append(delays, delay) + return ctx.Err() == nil && len(delays) < len(want) + } + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() s := newMacmonSupervisor(cfg) - runMacmonSupervisor(t, s) - var starts []time.Time - waitFor(t, func() bool { - count := len(macmonArgs(path)) - if count > len(starts) { - if count != len(starts)+1 { - t.Fatal("missed a child start while observing backoff") - } - starts = append(starts, time.Now()) - } - return len(starts) == 5 - }) - for i, min := range []time.Duration{40 * time.Millisecond, 90 * time.Millisecond, 140 * time.Millisecond, 40 * time.Millisecond} { - if gap := starts[i+1].Sub(starts[i]); gap < min { - t.Errorf("restart gap %d = %v, want >= %v", i+1, gap, min) + s.Run(ctx) + if ctx.Err() != nil { + t.Fatalf("supervisor did not finish: %v", ctx.Err()) + } + if len(delays) != len(want) { + t.Fatalf("scheduled delays = %v, want %v", delays, want) + } + for i, delay := range delays { + if delay != want[i] { + t.Errorf("scheduled backoff %d = %v, want %v", i+1, delay, want[i]) } } - if gap := starts[3].Sub(starts[2]); gap >= 190*time.Millisecond { - t.Errorf("backoff exceeded cap: %v", gap) + if got := len(macmonArgs(path)); got != len(want) { + t.Errorf("child starts = %d, want %d", got, len(want)) } - if gap := starts[4].Sub(starts[3]); gap >= 120*time.Millisecond { - t.Errorf("backoff did not reset after good line: %v", gap) + if _, sample, _ := s.Latest(); sample == nil { + t.Fatal("good line did not publish a sample") } }