From 94d56a484142ddf8f6e83e026b65287c9454d877 Mon Sep 17 00:00:00 2001 From: Baylem Date: Wed, 21 Oct 2026 12:40:51 -0400 Subject: [PATCH 01/18] feat: expose bounded game performance and relay monitoring Signed-off-by: Baylem --- .github/workflows/ci.yaml | 32 +- agent/cmd/main.go | 6 +- agent/internal/caps/caps.go | 18 +- agent/internal/rcon/rcon.go | 46 ++- agent/internal/rcon/rcon_bounds_test.go | 160 ++++++++ agent/internal/status/performance.go | 94 +++++ agent/internal/status/performance_test.go | 134 +++++++ agent/internal/status/status.go | 242 ++++++++---- agent/specs.md | 6 +- .../crds/gameplane.local_backups.yaml | 2 +- .../crds/gameplane.local_backupschedules.yaml | 2 +- .../crds/gameplane.local_clusters.yaml | 2 +- .../crds/gameplane.local_gameservers.yaml | 2 +- .../crds/gameplane.local_gametemplates.yaml | 37 +- .../crds/gameplane.local_modules.yaml | 2 +- .../crds/gameplane.local_modulesources.yaml | 2 +- .../crds/gameplane.local_networkcaptures.yaml | 2 +- .../crds/gameplane.local_restores.yaml | 2 +- .../gameplane.local_backups.yaml | 2 +- .../gameplane.local_backupschedules.yaml | 2 +- .../gameplane.local_clusters.yaml | 2 +- .../gameplane.local_gameservers.yaml | 2 +- .../gameplane.local_gametemplates.yaml | 37 +- .../gameplane.local_modules.yaml | 2 +- .../gameplane.local_modulesources.yaml | 2 +- .../gameplane.local_networkcaptures.yaml | 2 +- .../gameplane.local_restores.yaml | 2 +- .../dashboards/gameplane-workloads.json | 345 ++++++++++++++++++ charts/gameplane/templates/NOTES.txt | 10 + .../templates/grafana-dashboard.yaml | 2 + .../gameplane/templates/monitoring-rules.yaml | 72 ++++ .../templates/monitoring-validation.yaml | 3 + .../gameplane/templates/networkpolicies.yaml | 18 + .../gameplane/templates/servicemonitors.yaml | 28 ++ charts/gameplane/tests/monitoring-rules.sh | 19 + .../tests/monitoring-rules.test.yaml | 146 ++++++++ charts/gameplane/values.yaml | 7 +- deploy/kind/monitoring.sh | 23 ++ .../examples/monitoring/gameplane-values.yaml | 19 + docs/examples/monitoring/prometheus.yaml | 71 ++++ docs/monitoring.md | 142 +++++++ docs/tunnels.md | 3 + operator/api/v1alpha1/gametemplate_types.go | 22 ++ .../api/v1alpha1/zz_generated.deepcopy.go | 29 +- .../crd/gameplane.local_gametemplates.yaml | 35 ++ .../gameserver_relay_health_test.go | 9 + .../internal/controller/gameserver_tunnel.go | 7 + .../gametemplate_performance_envtest_test.go | 46 +++ operator/specs.md | 2 + test/e2e/buckets.sh | 8 +- test/e2e/monitoring_e2e_test.go | 221 +++++++++++ test/e2e/specs.md | 2 + tunnel/relay_health.go | 6 +- tunnel/relay_metrics.go | 44 +++ tunnel/relay_metrics_test.go | 37 ++ tunnel/specs.md | 2 + tunnel/tailscale.go | 6 +- web/src/types.ts | 5 + 58 files changed, 2114 insertions(+), 119 deletions(-) create mode 100644 agent/internal/rcon/rcon_bounds_test.go create mode 100644 agent/internal/status/performance.go create mode 100644 agent/internal/status/performance_test.go create mode 100644 charts/gameplane/dashboards/gameplane-workloads.json create mode 100644 charts/gameplane/templates/monitoring-rules.yaml create mode 100644 charts/gameplane/templates/monitoring-validation.yaml create mode 100644 charts/gameplane/tests/monitoring-rules.sh create mode 100644 charts/gameplane/tests/monitoring-rules.test.yaml create mode 100644 deploy/kind/monitoring.sh create mode 100644 docs/examples/monitoring/gameplane-values.yaml create mode 100644 docs/examples/monitoring/prometheus.yaml create mode 100644 docs/monitoring.md create mode 100644 operator/internal/controller/gametemplate_performance_envtest_test.go create mode 100644 test/e2e/monitoring_e2e_test.go create mode 100644 tunnel/relay_metrics.go create mode 100644 tunnel/relay_metrics_test.go diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 87849a54e..bd2474c2b 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1052,6 +1052,8 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5.0.1 - run: helm lint charts/gameplane + - name: evaluate monitoring rules + run: bash charts/gameplane/tests/monitoring-rules.sh standalone-compose: name: standalone Compose smoke @@ -1203,10 +1205,16 @@ jobs: | grep -q 'app.kubernetes.io/name: gameplane-telemetry-receiver' awk '/kind: Service/,/^---$/' /tmp/telemetry-receiver-scrape.yaml \ | grep -q 'name: dashboard' - # scrapeNamespaceSelector unset (the chart default): neither - # exception renders, matching the pre-fix behavior. + # Missing scrape ingress must fail with actionable guidance. + if helm template gameplane charts/gameplane --set serviceMonitors.enabled=true > /tmp/monitoring-invalid.txt 2>&1; then + echo "::error::monitoring without a scrape selector unexpectedly rendered" + exit 1 + fi + grep -q 'serviceMonitors.scrapeNamespaceSelector' /tmp/monitoring-invalid.txt + # Explicit external policy ownership renders no chart exception. helm template gameplane charts/gameplane \ --set serviceMonitors.enabled=true \ + --set serviceMonitors.externallyManagedPolicy=true \ --show-only templates/networkpolicies.yaml > /tmp/netpol-default.yaml if grep -q 'allow-prometheus-to-agent' /tmp/netpol-default.yaml; then echo "::error::allow-prometheus-to-agent rendered without serviceMonitors.scrapeNamespaceSelector set" @@ -1383,7 +1391,8 @@ jobs: # keeps working without api.telemetry.enabled. helm template gameplane charts/gameplane \ --set api.telemetry.receiver.enabled=true \ - --set serviceMonitors.enabled=true > /tmp/rendered-reuse-values-receiver.yaml + --set serviceMonitors.enabled=true \ + --set serviceMonitors.externallyManagedPolicy=true > /tmp/rendered-reuse-values-receiver.yaml python3 - <<'PY' import yaml docs = [d for d in yaml.safe_load_all(open('/tmp/rendered-reuse-values-receiver.yaml')) if d] @@ -1668,9 +1677,13 @@ jobs: # (operator does zero logins and is wait-dominated, so it runs # wide; the api buckets stay within the login limiter's budget). # The arm64 job below uses the same steps and per-bucket settings. - bucket: [operator, api-auth, api-roles, api-rbac, api-agent, api-mods, telemetry, optional-components, management-placement, network-isolation] + bucket: [operator, api-auth, api-roles, api-rbac, api-agent, api-mods, telemetry, optional-components, management-placement, network-isolation, monitoring] arch: [amd64] include: + - bucket: monitoring + parallel: 1 + test_timeout: 20m + job_timeout: 75 - bucket: network-isolation parallel: 1 test_timeout: 15m @@ -1732,8 +1745,11 @@ jobs: - name: bring up cluster + helm install env: GAMEPLANE_E2E_PROFILE: ${{ matrix.bucket == 'management-placement' && 'management' || 'default' }} - GAMEPLANE_E2E_CNI: ${{ matrix.bucket == 'network-isolation' && 'cilium' || 'kindnet' }} + GAMEPLANE_E2E_CNI: ${{ (matrix.bucket == 'network-isolation' || matrix.bucket == 'monitoring') && 'cilium' || 'kindnet' }} run: ./deploy/kind/e2e.sh up "$CLUSTER" e2e "$GAMEPLANE_E2E_PROFILE" + - name: install real monitoring fixture + if: matrix.bucket == 'monitoring' + run: bash deploy/kind/monitoring.sh - name: run go e2e (${{ matrix.bucket }}) run: | RUN_REGEX="$(./test/e2e/buckets.sh regex '${{ matrix.bucket }}')" @@ -1758,7 +1774,7 @@ jobs: uses: ./.github/actions/dump-cluster-state with: context: ${{ env.CTX }} - namespaces: "gameplane-system,gameplane-games" + namespaces: "${{ matrix.bucket == 'monitoring' && 'gameplane-system,gameplane-games,monitoring' || 'gameplane-system,gameplane-games' }}" include-previous-logs: "true" e2e-go-arm64: @@ -2525,7 +2541,7 @@ jobs: 'go-e2e-unit': (n) => n === 'go (test/e2e)', 'api-postgres': (n) => n === 'api (postgres)', 'e2e-buckets': (n) => n === 'e2e bucket coverage', - 'e2e-go': (n) => /^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components) \//.test(n) && n.includes('/ amd64 '), + 'e2e-go': (n) => /^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components|management-placement|network-isolation|monitoring) \//.test(n) && n.includes('/ amd64 '), 'e2e-go-arm64': (n) => /^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components) \//.test(n) && n.includes('/ arm64 '), 'e2e-multicluster': (n) => n.startsWith('e2e multicluster /') && n.includes('/ amd64 '), 'e2e-multicluster-arm64': (n) => n.startsWith('e2e multicluster /') && n.includes('/ arm64 '), @@ -2678,7 +2694,7 @@ jobs: const bucketSet = new Set(); for (const j of apiJobs) { const n = j.name || ''; - const m = n.match(/^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components) \//); + const m = n.match(/^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components|management-placement|network-isolation|monitoring) \//); if (m) bucketSet.add(m[1]); if (/^e2e multicluster \//.test(n)) bucketSet.add('multicluster'); if (/^e2e upgrade \//.test(n)) bucketSet.add('upgrade'); diff --git a/agent/cmd/main.go b/agent/cmd/main.go index 8ce7f15f8..a27d217ef 100644 --- a/agent/cmd/main.go +++ b/agent/cmd/main.go @@ -16,6 +16,7 @@ import ( "github.com/go-chi/chi/v5" "github.com/go-chi/chi/v5/middleware" + "github.com/prometheus/client_golang/prometheus" "github.com/GameplanePanel/gameplane/agent/internal/actions" "github.com/GameplanePanel/gameplane/agent/internal/auth" @@ -178,6 +179,7 @@ func main() { // and isn't meant to be trusted with the console/files/RCON routes that // cert would also unlock. + statusSource := status.New(rconClient, statusSpec) mountProtected := func(protected chi.Router) { // Console/Logs open long-lived WebSockets and Mods installs can run @@ -198,7 +200,7 @@ func main() { quiesce.Mount(bounded, rconClient, gameName, quiesceSpec) lifecycle.Mount(bounded, rconClient, gameName, lifecycleSpec) actions.Mount(bounded, rconClient, gameName, actionSpecs) - status.Mount(bounded, rconClient, statusSpec) + statusSource.Mount(bounded) }) } r.Group(func(protected chi.Router) { @@ -230,6 +232,8 @@ func main() { // Prometheus metrics for per-server resource usage and player counts. // Registered with the default registry so promhttp.Handler() picks it up. m := metrics.New() + prometheus.MustRegister(statusSource) + go statusSource.Run(ctx) // Resource usage: in proc mode (the operator shares the pod's PID // namespace and sets GAMEPLANE_USAGE_PROC) the agent reports the game diff --git a/agent/internal/caps/caps.go b/agent/internal/caps/caps.go index 495cc2654..448501ff6 100644 --- a/agent/internal/caps/caps.go +++ b/agent/internal/caps/caps.go @@ -83,11 +83,19 @@ type ModInstall struct { // StatusMetric reads one live readout from an RCON command's output via a // named-group regex (group "value"). type StatusMetric struct { - ID string `json:"id"` - DisplayName string `json:"displayName,omitempty"` - Command string `json:"command"` - Regex string `json:"regex"` - Unit string `json:"unit,omitempty"` + ID string `json:"id"` + DisplayName string `json:"displayName,omitempty"` + Command string `json:"command"` + Regex string `json:"regex"` + Unit string `json:"unit,omitempty"` + Performance *PerformanceMetric `json:"performance,omitempty"` +} + +// PerformanceMetric mirrors the optional, explicitly numeric status contract. +type PerformanceMetric struct { + Signal string `json:"signal"` + Unit string `json:"unit"` + DegradedThreshold *string `json:"degradedThreshold,omitempty"` } // PlayerActions maps moderation actions to console command templates diff --git a/agent/internal/rcon/rcon.go b/agent/internal/rcon/rcon.go index bbb0456d2..7dfcaa01e 100644 --- a/agent/internal/rcon/rcon.go +++ b/agent/internal/rcon/rcon.go @@ -56,6 +56,11 @@ const ( // byte length (F-104). minPacketSize = 10 maxPacketSize = 4*4096 + 10 // 16394 + + // sourceMaxResponseBytes permits large administrative listings while + // bounding the aggregate allocation across otherwise valid fragments. + // Status collection applies its separate, smaller 64-KiB acceptance cap. + sourceMaxResponseBytes = 8 * 1024 * 1024 ) // PassFn resolves the RCON password on demand (allowing rotation without restart). @@ -86,6 +91,8 @@ type Client struct { // execDeadline bounds a single command exchange (see Exec). execDeadline time.Duration + // authDeadline separately bounds authentication, including all preamble packets. + authDeadline time.Duration // authFailureCooldown bounds how long ensureLocked refuses to re-dial // after an explicit AUTH_RESPONSE rejection (id=-1). A struct field @@ -114,6 +121,8 @@ type Client struct { // (heartbeat, players, quiesce, console). const defaultExecDeadline = 30 * time.Second +const defaultSourceAuthDeadline = 10 * time.Second + // defaultAuthFailureCooldown bounds how long ensureLocked backs off after // an explicit auth rejection before it will dial again. Mirrors // websocket.go's defaultWebSocketAuthFailureCooldown — same value, same @@ -133,6 +142,7 @@ func New(host string, port int, pw PassFn) *Client { addr: net.JoinHostPort(host, fmt.Sprint(port)), password: pw, execDeadline: defaultExecDeadline, + authDeadline: defaultSourceAuthDeadline, authFailureCooldown: defaultAuthFailureCooldown, } } @@ -154,7 +164,8 @@ func (c *Client) Exec(cmd string) (string, error) { if d <= 0 { d = defaultExecDeadline } - _ = conn.SetDeadline(time.Now().Add(d)) + exchangeDeadline := time.Now().Add(d) + _ = conn.SetDeadline(exchangeDeadline) defer func() { _ = conn.SetDeadline(time.Time{}) }() reqID := c.allocID() @@ -172,12 +183,22 @@ func (c *Client) Exec(cmd string) (string, error) { // reqID) and bound the wait for any trailing fragments with a short grace // window: a single-packet reply returns as soon as the grace // elapses or the server closes; a fragmented Source reply keeps extending - // the window as each fragment arrives. + // the window as each fragment arrives, never beyond exchangeDeadline. var out bytes.Buffer gotResponse := false for { + if !time.Now().Before(exchangeDeadline) { + c.dropLocked() + return "", fmt.Errorf("rcon: command exchange deadline exceeded: %w", os.ErrDeadlineExceeded) + } id, _, body, err := c.readPacket() if err != nil { + if !time.Now().Before(exchangeDeadline) { + // Reaching the hard deadline is not evidence of a complete + // response. Discard its partial text and any queued fragments. + c.dropLocked() + return "", fmt.Errorf("rcon: command exchange deadline exceeded: %w", os.ErrDeadlineExceeded) + } if gotResponse { // We already hold the reply. A timeout means the grace window // elapsed on a still-healthy socket — keep it for reuse. Any @@ -196,9 +217,17 @@ func (c *Client) Exec(cmd string) (string, error) { // Source/Minecraft echo the request id in every reply fragment; ignore // anything else. if id == reqID { + if len(body) > sourceMaxResponseBytes-out.Len() { + c.dropLocked() + return "", fmt.Errorf("rcon: response exceeded limit of %d bytes", sourceMaxResponseBytes) + } out.WriteString(body) gotResponse = true - _ = conn.SetReadDeadline(time.Now().Add(responseGrace)) + readDeadline := time.Now().Add(responseGrace) + if readDeadline.After(exchangeDeadline) { + readDeadline = exchangeDeadline + } + _ = conn.SetReadDeadline(readDeadline) } } } @@ -247,7 +276,12 @@ func (c *Client) ensureLocked() error { return err } c.conn = conn - _ = conn.SetDeadline(time.Now().Add(10 * time.Second)) + authBudget := c.authDeadline + if authBudget <= 0 { + authBudget = defaultSourceAuthDeadline + } + authDeadline := time.Now().Add(authBudget) + _ = conn.SetDeadline(authDeadline) authID := c.allocID() if err := c.writePacket(authID, typeAuth, pw); err != nil { @@ -256,6 +290,10 @@ func (c *Client) ensureLocked() error { } // Server MAY send an empty RESPONSE_VALUE before AUTH_RESPONSE. for { + if !time.Now().Before(authDeadline) { + c.dropLocked() + return fmt.Errorf("rcon: authentication deadline exceeded: %w", os.ErrDeadlineExceeded) + } id, kind, _, err := c.readPacket() if err != nil { c.dropLocked() diff --git a/agent/internal/rcon/rcon_bounds_test.go b/agent/internal/rcon/rcon_bounds_test.go new file mode 100644 index 000000000..bc55fb868 --- /dev/null +++ b/agent/internal/rcon/rcon_bounds_test.go @@ -0,0 +1,160 @@ +package rcon + +import ( + "net" + "strings" + "testing" + "time" +) + +func TestExecContinuousFragmentsRespectAbsoluteDeadline(t *testing.T) { + t.Parallel() + for _, unrelated := range []bool{false, true} { + name := "matching" + if unrelated { + name = "unrelated-after-response" + } + t.Run(name, func(t *testing.T) { + t.Parallel() + clientConn, serverConn := net.Pipe() + defer clientConn.Close() + defer serverConn.Close() + client := &Client{conn: clientConn, execDeadline: 100 * time.Millisecond} + server := &Client{conn: serverConn} + done := make(chan struct{}) + go func() { + defer close(done) + id, _, _, err := server.readPacket() + if err != nil { + return + } + if server.writePacket(id, typeRespValue, "partial") != nil { + return + } + if unrelated { + id++ + } + // Empty fragments cannot trigger the aggregate byte cap. A + // continuously readable socket must still reach its hard deadline. + for server.writePacket(id, typeRespValue, "") == nil { + } + }() + finished := make(chan struct{}) + var output string + var execErr error + go func() { + output, execErr = client.Exec("status") + close(finished) + }() + select { + case <-finished: + case <-time.After(2 * time.Second): + // Unblock both goroutines even when testing the old read loop. + _ = serverConn.Close() + <-finished + t.Fatal("continuous fragments extended the absolute command deadline") + } + if execErr == nil || !isTimeout(execErr) || output != "" { + t.Fatalf("deadline must discard incomplete output: bytes=%d err=%v", len(output), execErr) + } + if client.conn != nil { + t.Fatal("deadline must drop the connection to discard trailing fragments") + } + <-done + }) + } +} + +func TestExecAggregateResponseBound(t *testing.T) { + t.Parallel() + for _, extra := range []int{0, 1} { + name := "exact-limit" + if extra > 0 { + name = "over-limit" + } + t.Run(name, func(t *testing.T) { + t.Parallel() + clientConn, serverConn := net.Pipe() + defer clientConn.Close() + defer serverConn.Close() + client := &Client{conn: clientConn, execDeadline: 10 * time.Second} + server := &Client{conn: serverConn} + done := make(chan struct{}) + go func() { + defer close(done) + defer serverConn.Close() + id, _, _, err := server.readPacket() + if err != nil { + return + } + fragment := strings.Repeat("x", 4000) + remaining := sourceMaxResponseBytes + extra + for remaining > 0 { + length := min(remaining, len(fragment)) + if server.writePacket(id, typeRespValue, fragment[:length]) != nil { + return + } + remaining -= length + } + }() + output, err := client.Exec("large-list") + <-done + if extra == 0 { + if err != nil || len(output) != sourceMaxResponseBytes || strings.Trim(output, "x") != "" { + t.Fatalf("legitimate response at the generous cap must succeed: bytes=%d err=%v", len(output), err) + } + return + } + if err == nil || !strings.Contains(err.Error(), "response exceeded limit") || output != "" { + t.Fatalf("oversized response must fail without a truncated result: bytes=%d err=%v", len(output), err) + } + if client.conn != nil { + t.Fatal("oversized response must drop the connection") + } + }) + } +} + +func TestAuthContinuousPreambleRespectsAbsoluteDeadline(t *testing.T) { + t.Parallel() + lc := &net.ListenConfig{} + listener, err := lc.Listen(t.Context(), "tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer listener.Close() + done := make(chan struct{}) + go func() { + defer close(done) + conn, err := listener.Accept() + if err != nil { + return + } + defer conn.Close() + // Bound the fixture itself even if the client's deadline regresses. + _ = conn.SetDeadline(time.Now().Add(3 * time.Second)) + server := &Client{conn: conn} + id, _, _, err := server.readPacket() + if err != nil { + return + } + for server.writePacket(id, typeRespValue, "") == nil { + } + }() + host, port, err := net.SplitHostPort(listener.Addr().String()) + if err != nil { + t.Fatal(err) + } + client := New(host, mustAtoi(port), func() (string, error) { return "pw", nil }) + client.authDeadline = 100 * time.Millisecond + defer client.Close() + started := time.Now() + output, err := client.Exec("status") + if err == nil || !isTimeout(err) || output != "" || time.Since(started) > 2*time.Second { + t.Fatalf("continuous auth preamble must time out without output: bytes=%d err=%v elapsed=%v", len(output), err, time.Since(started)) + } + if client.conn != nil { + t.Fatal("authentication deadline must drop the connection") + } + <-done +} diff --git a/agent/internal/status/performance.go b/agent/internal/status/performance.go new file mode 100644 index 000000000..0dbda1979 --- /dev/null +++ b/agent/internal/status/performance.go @@ -0,0 +1,94 @@ +package status + +import ( + "math" + "regexp" + "strconv" + "time" + + "github.com/GameplanePanel/gameplane/agent/internal/caps" + "github.com/prometheus/client_golang/prometheus" +) + +type performanceSample struct { + value float64 + observed time.Time +} + +var decimalThreshold = regexp.MustCompile(`^(0|[1-9][0-9]{0,8})(\.[0-9]{1,6})?$`) + +func validPerformance(p *caps.PerformanceMetric) bool { + if p == nil { + return false + } + if p.DegradedThreshold != nil && !decimalThreshold.MatchString(*p.DegradedThreshold) { + return false + } + switch p.Signal { + case "tick_rate", "frame_rate": + return p.Unit == "per_second" + case "tick_duration": + return p.Unit == "seconds" || p.Unit == "milliseconds" + default: + return false + } +} + +func finite(n float64) bool { return !math.IsNaN(n) && !math.IsInf(n, 0) } +func baseValue(n float64, p *caps.PerformanceMetric) float64 { + if p.Unit == "milliseconds" { + return n / 1000 + } + return n +} +func parsePerformance(raw string, p *caps.PerformanceMetric) (performanceSample, bool) { + n, err := strconv.ParseFloat(raw, 64) + if err != nil || !finite(n) || n < 0 { + return performanceSample{}, false + } + return performanceSample{value: baseValue(n, p)}, true +} + +var performanceLabels = []string{"metric", "signal", "unit"} +var performanceValue = prometheus.NewDesc("gameplane_agent_performance_value", "Finite game performance measurement in the declared base unit; absent when unknown or stale.", performanceLabels, nil) +var performanceAvailable = prometheus.NewDesc("gameplane_agent_performance_available", "Whether the declared performance signal has a fresh valid sample.", performanceLabels, nil) +var performanceTimestamp = prometheus.NewDesc("gameplane_agent_performance_sample_timestamp_seconds", "Unix collection timestamp for a fresh valid game performance sample.", performanceLabels, nil) +var performanceThreshold = prometheus.NewDesc("gameplane_agent_performance_degraded_threshold", "Module-declared degradation threshold in the same base unit as the measurement.", append(append([]string{}, performanceLabels...), "direction"), nil) + +// Describe publishes the fixed descriptors for explicitly declared performance signals. +func (s *Source) Describe(ch chan<- *prometheus.Desc) { + ch <- performanceValue + ch <- performanceAvailable + ch <- performanceTimestamp + ch <- performanceThreshold +} + +// Collect exports cached performance samples and availability without issuing commands. +func (s *Source) Collect(ch chan<- prometheus.Metric) { + s.mu.Lock() + defer s.mu.Unlock() + for _, m := range s.metrics { + p := m.performance + if p == nil { + continue + } + unit, direction := "hertz", "below" + if p.Signal == "tick_duration" { + unit, direction = "seconds", "above" + } + labels := []string{m.id, p.Signal, unit} + sample, ok := s.samples[m.id] + fresh := !sample.observed.IsZero() && time.Since(sample.observed) < performanceFreshness + available := 0.0 + if fresh && ok { + available = 1 + ch <- prometheus.MustNewConstMetric(performanceValue, prometheus.GaugeValue, sample.value, labels...) + ch <- prometheus.MustNewConstMetric(performanceTimestamp, prometheus.GaugeValue, float64(sample.observed.UnixNano())/1e9, labels...) + } + ch <- prometheus.MustNewConstMetric(performanceAvailable, prometheus.GaugeValue, available, labels...) + if p.DegradedThreshold != nil { + threshold, _ := strconv.ParseFloat(*p.DegradedThreshold, 64) // validated bounded decimal + ch <- prometheus.MustNewConstMetric(performanceThreshold, prometheus.GaugeValue, baseValue(threshold, p), append(labels, direction)...) + } + } +} diff --git a/agent/internal/status/performance_test.go b/agent/internal/status/performance_test.go new file mode 100644 index 000000000..2155bb998 --- /dev/null +++ b/agent/internal/status/performance_test.go @@ -0,0 +1,134 @@ +package status + +import ( + "context" + "net/http/httptest" + "sync/atomic" + "testing" + "time" + + "github.com/GameplanePanel/gameplane/agent/internal/caps" + "github.com/go-chi/chi/v5" + "github.com/prometheus/client_golang/prometheus" +) + +func performanceSpec() *caps.Status { + threshold := "18" + return &caps.Status{Metrics: []caps.StatusMetric{{ID: "tps", Command: "stats", Regex: `TPS: (?P\S+)`, Performance: &caps.PerformanceMetric{Signal: "tick_rate", Unit: "per_second", DegradedThreshold: &threshold}}, {ID: "tick", Command: "stats", Regex: `Tick: (?P\S+)`, Performance: &caps.PerformanceMetric{Signal: "tick_duration", Unit: "milliseconds"}}}} +} + +func TestPerformanceCacheZeroFreshnessAndSharedCommand(t *testing.T) { + t.Parallel() + rc := &fakeRcon{respond: func(string) (string, error) { return "TPS: 0 Tick: 50", nil }} + s := New(rc, performanceSpec()) + s.refresh(t.Context()) + if rc.count() != 1 { + t.Fatalf("shared command executed %d times", rc.count()) + } + reg := prometheus.NewRegistry() + reg.MustRegister(s) + router := chi.NewRouter() + router.Route("/legacy", s.Mount) + router.Route("/versioned", s.Mount) + for _, route := range []string{"/legacy/status", "/versioned/status"} { + response := httptest.NewRecorder() + router.ServeHTTP(response, httptest.NewRequest("GET", route, nil)) + if response.Code != 200 { + t.Fatal("shared status cache unavailable") + } + } + for range 20 { + if _, err := reg.Gather(); err != nil { + t.Fatal(err) + } + } + if rc.count() != 1 { + t.Fatal("scrape executed RCON") + } + s.mu.Lock() + if s.samples["tps"].value != 0 || s.samples["tick"].value != .05 { + t.Fatal("zero or unit conversion lost") + } + s.lastFetch = time.Now().Add(-performanceFreshness - time.Second) + for id, sample := range s.samples { + sample.observed = s.lastFetch + s.samples[id] = sample + } + s.mu.Unlock() + families, err := reg.Gather() + if err != nil { + t.Fatal(err) + } + for _, family := range families { + if family.GetName() == "gameplane_agent_performance_value" { + t.Fatal("stale value emitted") + } + } +} + +func TestPerformanceRejectsNonFiniteAndInvalidDeclarations(t *testing.T) { + t.Parallel() + for _, value := range []string{"NaN", "+Inf", "-1", "no-value"} { + s := New(&fakeRcon{respond: func(string) (string, error) { return "TPS: " + value, nil }}, performanceSpec()) + s.refresh(t.Context()) + if len(s.samples) != 0 { + t.Fatalf("invalid %q emitted", value) + } + } + inf := "+Inf" + for _, p := range []*caps.PerformanceMetric{{Signal: "seed", Unit: "per_second"}, {Signal: "tick_rate", Unit: "milliseconds"}, {Signal: "tick_duration", Unit: "seconds", DegradedThreshold: &inf}} { + if validPerformance(p) { + t.Fatalf("accepted %+v", p) + } + } +} + +func TestPerformanceTimeoutBoundsOutstandingWorkAndDiscardsLateResult(t *testing.T) { + t.Parallel() + release := make(chan struct{}) + var calls atomic.Int32 + rc := &fakeRcon{respond: func(string) (string, error) { calls.Add(1); <-release; return "TPS: 20", nil }} + s := New(rc, performanceSpec()) + s.commandTimeout = 10 * time.Millisecond + s.refresh(t.Context()) + for range 10 { + s.refresh(t.Context()) + } + if calls.Load() != 1 { + t.Fatalf("unbounded workers: %d", calls.Load()) + } + close(release) + deadline := time.After(time.Second) + for len(s.execSlot) != 0 { + select { + case <-deadline: + t.Fatal("worker failed to finish") + default: + time.Sleep(time.Millisecond) + } + } + if len(s.samples) != 0 { + t.Fatal("late result became a fresh sample") + } + ctx, cancel := context.WithCancel(t.Context()) + cancel() + s.Run(ctx) + if calls.Load() != 1 { + t.Fatal("cancelled collector executed a command") + } +} + +func TestPerformanceBoundsDirectCapabilitiesAndDeduplicatesIDs(t *testing.T) { + t.Parallel() + spec := performanceSpec() + for range 30 { + spec.Metrics = append(spec.Metrics, spec.Metrics[0]) + } + if compiled := compile(spec.Metrics); len(compiled) != 2 { + t.Fatalf("duplicate IDs expanded cardinality: %d", len(compiled)) + } + spec.Metrics[0].Command = string(make([]byte, maxCommandBytes+1)) + if len(compile(spec.Metrics[:1])) != 0 { + t.Fatal("oversized command accepted") + } +} diff --git a/agent/internal/status/status.go b/agent/internal/status/status.go index 7ddc82bc8..1304d37ba 100644 --- a/agent/internal/status/status.go +++ b/agent/internal/status/status.go @@ -1,45 +1,45 @@ -// Package status reads module-declared live metrics over RCON for the -// dashboard's Overview tab. -// -// A GameTemplate declares metrics in spec.capabilities.status.metrics[]; -// the operator serializes them into GAMEPLANE_CAPABILITIES and the agent -// interprets them here. Each metric runs an RCON command and extracts a -// value via a named-group regex (group "value"), so modules surface -// game-specific readouts (TPS, world time, …) without an agent change. +// Package status shares module-declared RCON readouts between the dashboard +// and explicitly opted-in numeric performance monitoring. package status import ( + "context" "errors" - "log/slog" "net/http" "regexp" "strings" "sync" "time" - "github.com/go-chi/chi/v5" - "github.com/GameplanePanel/gameplane/agent/internal/caps" "github.com/GameplanePanel/gameplane/agent/internal/httpjson" "github.com/GameplanePanel/gameplane/agent/internal/rcon" + "github.com/go-chi/chi/v5" ) -// Rcon is the slice of *rcon.Client we use. An interface so tests can -// swap in a recording fake. +// Rcon executes module-declared commands through the configured game transport. type Rcon interface { Exec(cmd string) (string, error) } -// metric is one declared readout whose regex compiled at mount time. +const ( + cacheTTL = 5 * time.Second + performanceInterval = 30 * time.Second + performanceFreshness = 75 * time.Second + cycleTimeout = 10 * time.Second + maxCommandBytes = 1024 + maxRegexBytes = 4096 + maxResponseBytes = 64 * 1024 +) + type metric struct { id, displayName, command, unit string re *regexp.Regexp valueIdx int + performance *caps.PerformanceMetric } -// Result is one metric's current value (empty when the command did not -// match this cycle). DisplayName/Unit are echoed so the dashboard needs -// no second lookup. +// Result is one module-declared readout returned by the dashboard status route. type Result struct { ID string `json:"id"` DisplayName string `json:"displayName,omitempty"` @@ -47,85 +47,183 @@ type Result struct { Unit string `json:"unit,omitempty"` } -type handler struct { - rc Rcon - metrics []metric - - mu sync.Mutex - lastFetch time.Time - cached []Result +// Source is shared by both protected route mounts and the metrics collector. +// At most one legacy context-free Exec can remain in flight after a timeout. +type Source struct { + rc Rcon + metrics []metric + automatic bool + mu sync.Mutex + refreshMu sync.Mutex + lastFetch time.Time + cached []Result + samples map[string]performanceSample + execSlot chan struct{} + commandTimeout time.Duration } -const cacheTTL = 5 * time.Second - -// Mount registers GET /status. spec is the module's declared metrics -// (nil when the template declares none — the endpoint then returns []). -func Mount(r chi.Router, rc Rcon, spec *caps.Status) { - h := &handler{rc: rc} +// New validates the declared readouts and creates their shared cache and collector. +// Call Run to collect opted-in performance readings independently of requests. +func New(rc Rcon, spec *caps.Status) *Source { + s := &Source{rc: rc, execSlot: make(chan struct{}, 1), commandTimeout: 3 * time.Second, cached: []Result{}} if spec != nil { - h.metrics = compile(spec.Metrics) + s.metrics = compile(spec.Metrics) + } + for _, m := range s.metrics { + s.automatic = s.automatic || m.performance != nil } - r.Get("/status", h.serve) + return s } -// compile builds the metric list, dropping any whose regex is invalid or -// lacks the required (?P…) group (logged) so one bad declaration -// can't disable the whole panel. +// Mount retains the standalone handler API. Production constructs one Source. +func Mount(r chi.Router, rc Rcon, spec *caps.Status) { New(rc, spec).Mount(r) } + +// Mount registers the status route using this source's shared readout cache. +func (s *Source) Mount(r chi.Router) { r.Get("/status", s.serve) } + +var metricID = regexp.MustCompile(`^[a-z0-9]([a-z0-9-]*[a-z0-9])?$`) + func compile(specs []caps.StatusMetric) []metric { - out := make([]metric, 0, len(specs)) - for _, s := range specs { - if s.ID == "" || s.Command == "" || s.Regex == "" { + out := make([]metric, 0, min(len(specs), 16)) + seen := map[string]bool{} + for _, s := range specs[:min(len(specs), 16)] { + if seen[s.ID] || len(s.ID) > 63 || !metricID.MatchString(s.ID) || s.Command == "" || len(s.Command) > maxCommandBytes || s.Regex == "" || len(s.Regex) > maxRegexBytes { continue } re, err := regexp.Compile(s.Regex) - if err != nil { - slog.Warn("invalid status metric regex; metric disabled", "metric", s.ID, "err", err) + if err != nil || re.SubexpIndex("value") < 0 { continue } - idx := re.SubexpIndex("value") - if idx < 0 { - slog.Warn("status metric regex has no (?P…) group; metric disabled", "metric", s.ID) - continue + m := metric{id: s.ID, displayName: s.DisplayName, command: s.Command, unit: s.Unit, re: re, valueIdx: re.SubexpIndex("value")} + if validPerformance(s.Performance) { + p := *s.Performance + if p.DegradedThreshold != nil { + value := *p.DegradedThreshold + p.DegradedThreshold = &value + } + m.performance = &p } - out = append(out, metric{ - id: s.ID, displayName: s.DisplayName, command: s.Command, - unit: s.Unit, re: re, valueIdx: idx, - }) + seen[s.ID] = true + out = append(out, m) } return out } -func (h *handler) serve(w http.ResponseWriter, _ *http.Request) { - h.mu.Lock() - fresh := !h.lastFetch.IsZero() && time.Since(h.lastFetch) < cacheTTL - cached := h.cached - h.mu.Unlock() - if fresh { - httpjson.Write(w, http.StatusOK, cached) +// Run refreshes opted-in performance readouts until ctx is canceled. +// Sources without performance declarations retain on-demand collection. +func (s *Source) Run(ctx context.Context) { + if !s.automatic { return } - - results := make([]Result, 0, len(h.metrics)) - for _, m := range h.metrics { - raw, err := h.rc.Exec(m.command) - if errors.Is(err, rcon.ErrDisabled) { - // No RCON: no live metrics. Empty array so the dashboard can - // render the panel uniformly for every game. - httpjson.Write(w, http.StatusOK, []Result{}) + for ctx.Err() == nil { + s.refresh(ctx) + timer := time.NewTimer(performanceInterval) + select { + case <-ctx.Done(): + timer.Stop() return + case <-timer.C: + } + } +} + +type commandResult struct { + raw string + err error + completed time.Time +} + +var errUnavailable = errors.New("status source unavailable") + +func (s *Source) execute(ctx context.Context, command string) commandResult { + if ctx.Err() != nil { + return commandResult{err: ctx.Err()} + } + select { + case s.execSlot <- struct{}{}: + default: + return commandResult{err: errUnavailable} + } + result := make(chan commandResult, 1) + go func() { + raw, err := s.rc.Exec(command) + if len(raw) > maxResponseBytes { + raw, err = "", errUnavailable + } + <-s.execSlot + result <- commandResult{raw: raw, err: err, completed: time.Now()} + }() + queryCtx, cancel := context.WithTimeout(ctx, s.commandTimeout) + defer cancel() + select { + case <-queryCtx.Done(): + return commandResult{err: queryCtx.Err()} + case reply := <-result: + if queryCtx.Err() != nil { + return commandResult{err: queryCtx.Err()} + } + return reply + } +} + +func (s *Source) refresh(ctx context.Context) { + s.refreshMu.Lock() + defer s.refreshMu.Unlock() + s.mu.Lock() + fresh := !s.automatic && !s.lastFetch.IsZero() && time.Since(s.lastFetch) < cacheTTL + s.mu.Unlock() + if fresh { + return + } + ctx, cancel := context.WithTimeout(ctx, cycleTimeout) + defer cancel() + results := make([]Result, 0, len(s.metrics)) + samples := map[string]performanceSample{} + commands := map[string]commandResult{} + for _, m := range s.metrics { + reply, ok := commands[m.command] + if !ok { + reply = s.execute(ctx, m.command) + commands[m.command] = reply + } + if errors.Is(reply.err, rcon.ErrDisabled) { + results = []Result{} + samples = map[string]performanceSample{} + break } res := Result{ID: m.id, DisplayName: m.displayName, Unit: m.unit} - if err != nil { - slog.Warn("status metric rcon", "metric", m.id, "err", err) - } else if sub := m.re.FindStringSubmatch(strings.ReplaceAll(raw, "\r", "")); sub != nil { - res.Value = strings.TrimSpace(sub[m.valueIdx]) + if reply.err == nil { + if sub := m.re.FindStringSubmatch(strings.ReplaceAll(reply.raw, "\r", "")); sub != nil { + res.Value = strings.TrimSpace(sub[m.valueIdx]) + } + } + if m.performance != nil { + if sample, valid := parsePerformance(res.Value, m.performance); valid { + sample.observed = reply.completed + samples[m.id] = sample + } } results = append(results, res) } + s.mu.Lock() + s.lastFetch, s.cached, s.samples = time.Now(), results, samples + s.mu.Unlock() +} - h.mu.Lock() - h.lastFetch = time.Now() - h.cached = results - h.mu.Unlock() - httpjson.Write(w, http.StatusOK, results) +func (s *Source) serve(w http.ResponseWriter, r *http.Request) { + s.mu.Lock() + fresh := !s.lastFetch.IsZero() && time.Since(s.lastFetch) < cacheTTL + s.mu.Unlock() + if !s.automatic && !fresh { + s.refresh(r.Context()) + } + s.mu.Lock() + cached := append([]Result{}, s.cached...) + if s.automatic && time.Since(s.lastFetch) >= performanceFreshness { + for i := range cached { + cached[i].Value = "" + } + } + s.mu.Unlock() + httpjson.Write(w, http.StatusOK, cached) } diff --git a/agent/specs.md b/agent/specs.md index af1f57529..76e343a15 100644 --- a/agent/specs.md +++ b/agent/specs.md @@ -21,7 +21,7 @@ The agent is a per-pod HTTP/HTTPS sidecar that runs inside every game pod to exp - **Quiesce**: Pause auto-saves and flush in-flight state before snapshots; run module-declared sequences over RCON and handle games that don't support it gracefully. - **Lifecycle**: Run module-declared stop sequences over RCON before the operator scales the server to zero. - **Actions**: Execute module-declared operator actions (templated RCON commands with user parameters) without agent code changes. -- **Status metrics**: Query live game metrics (TPS, world time, etc.) via RCON and regex extraction; module-declared in `capabilities.status.metrics[]`. +- **Status metrics**: Query live game metrics (TPS, world time, etc.) via RCON and regex extraction; module-declared in `capabilities.status.metrics[]`. Optional `performance` declarations add closed numeric signal/unit semantics and decimal degradation thresholds. One shared source serves both protected route mounts and Prometheus: opted-in templates use a 30-second interval cache, 3-second command/10-second cycle budgets, one outstanding legacy RCON call, and 75-second sample freshness. Scrapes issue no commands; invalid or stale values are absent and valid zero is retained. See `docs/monitoring.md` for bounds and metric families. - **Mods**: List, install, and uninstall game mods from a registry; track per-volume metadata; guard downloads with strict SSRF egress validation. - **Heartbeat**: Periodically patch the owning GameServer's status with lastHeartbeat, playersOnline, playersMax, gameVersion, and resource usage (CPU, memory, disk). @@ -86,7 +86,7 @@ Per-package roles: - **`mods`**: Tracks installed mods in a per-volume manifest (`.gameplane-mods.json`); downloads from registry with strict egress validation via `netguard.IsPublic`. - **`players`**: Queries player count, names, ban lists, and runs moderation actions over RCON; a single template-driven `commander` renders each moderation command from the module's declared `capabilities.players` templates (no per-game Go implementations), and reports capabilities from what the template declares. - **`quiesce`**: Runs module-declared sequences (e.g., Minecraft's `save-off` + `save-all flush`) over RCON; responds `quiesced: false` + reason when unsupported (not an error). -- **`rcon`**: Factory pattern for wire-protocol clients (`Valve/Source`, `Telnet`, `WebSocket`, `BattlEye`, `Satisfactory`, `Palworld`, `NuclearOption`, `REST`, `CLI`, `Disabled`); `Exec(cmd) (string, error)` interface. RCON reply packets are accepted up to 16394 bytes (a 4096-character Minecraft chunk at its worst-case UTF-8 byte length, plus the 10-byte packet header) and rejected as malformed outside `[10, 16394]` (F-104). +- **`rcon`**: Factory pattern for wire-protocol clients (`Valve/Source`, `Telnet`, `WebSocket`, `BattlEye`, `Satisfactory`, `Palworld`, `NuclearOption`, `REST`, `CLI`, `Disabled`); `Exec(cmd) (string, error)` interface. RCON reply packets are accepted up to 16394 bytes (a 4096-character Minecraft chunk at its worst-case UTF-8 byte length, plus the 10-byte packet header) and rejected as malformed outside `[10, 16394]` (F-104). Source RCON also caps aggregate response text at 8 MiB and enforces an absolute 30-second command exchange deadline after connection/authentication; each 400-ms fragment idle window is clamped to that deadline. Exceeding the aggregate cap or hard deadline drops the connection and returns an error without partial output. The status collector separately accepts at most 64 KiB. - **`status`**: Runs module-declared metrics queries over RCON; each metric specifies a command and a regex with named group `"value"` for extraction. - **`usage`**: Reads CPU/memory from `/proc` (proc mode, default) or cgroup v2; disk via `statfs`; exposes `Sample` with `Known` flags so callers distinguish "unknown" from "zero". @@ -210,7 +210,7 @@ All endpoints on the `--addr` control mux, except `/healthz`, return `401 Unauth - **Resource usage is in-pod**: The `usage` package reads from `/proc` or cgroups; no external metrics pipeline required. Cgroup mode is a fallback for older clusters; proc mode (default in production) requires the operator to set `ShareProcessNamespace: true`. - **Module capabilities drive behavior**: Every game-specific handler (players, quiesce, lifecycle, status, actions) reads its config from `--capabilities` (JSON unmarshaled into `caps.Spec`). New games require no agent code change. - **RCON connection errors are graceful**: A lost RCON connection does not crash the agent. `console`, `players`, `quiesce`, and `lifecycle` handlers catch connection errors and return appropriate HTTP status (e.g., `502 Bad Gateway`). -- **Source RCON sends complete frames**: Each AUTH or command frame is submitted in one socket write, with its existing byte-size limit, request ID, type, body, and two NUL terminators preserved. This avoids separate header-only writes rejected by vanilla Minecraft's RCON reader. A short write is an error and drops the connection without replaying the command; response grace and healthy connection reuse remain unchanged. +- **Source RCON sends complete frames**: Each AUTH or command frame is submitted in one socket write, with its existing byte-size limit, request ID, type, body, and two NUL terminators preserved. This avoids separate header-only writes rejected by vanilla Minecraft's RCON reader. A short write is an error and drops the connection without replaying the command. Dialing has a five-second limit; authentication has an absolute ten-second limit including continuous preamble packets. Healthy idle completion permits connection reuse; aggregate response overflow or absolute command deadline exhaustion discards partial output and drops the connection. - **Log streams are tail-only**: The `logs` package does not support random-access reads. It streams from the current end (live mode) or from file start (backlog mode); clients must handle partial output and reconnection. ## Dependencies diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backups.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backups.yaml index dbbcda875..448222049 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backups.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backups.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: ceab4a3a14adff11367ab42c914aaf44a34965e6bea1fb30ecf7ea2fadec09a1 + gameplane.local/crd-bundle-sha256: 1dc8f563ef15434a03e2f9c2f61c92576d3a3483dd19c787434ed8b4b743cacd controller-gen.kubebuilder.io/version: v0.20.1 name: backups.gameplane.local spec: diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backupschedules.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backupschedules.yaml index 9aa2aa7b8..cae7c3a68 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backupschedules.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backupschedules.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: ceab4a3a14adff11367ab42c914aaf44a34965e6bea1fb30ecf7ea2fadec09a1 + gameplane.local/crd-bundle-sha256: 1dc8f563ef15434a03e2f9c2f61c92576d3a3483dd19c787434ed8b4b743cacd controller-gen.kubebuilder.io/version: v0.20.1 name: backupschedules.gameplane.local spec: diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_clusters.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_clusters.yaml index 436cc742e..0dff5ac61 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_clusters.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_clusters.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: ceab4a3a14adff11367ab42c914aaf44a34965e6bea1fb30ecf7ea2fadec09a1 + gameplane.local/crd-bundle-sha256: 1dc8f563ef15434a03e2f9c2f61c92576d3a3483dd19c787434ed8b4b743cacd controller-gen.kubebuilder.io/version: v0.20.1 name: clusters.gameplane.local spec: diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gameservers.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gameservers.yaml index dea6133a3..bf338daa1 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gameservers.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gameservers.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: ceab4a3a14adff11367ab42c914aaf44a34965e6bea1fb30ecf7ea2fadec09a1 + gameplane.local/crd-bundle-sha256: 1dc8f563ef15434a03e2f9c2f61c92576d3a3483dd19c787434ed8b4b743cacd controller-gen.kubebuilder.io/version: v0.20.1 name: gameservers.gameplane.local spec: diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gametemplates.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gametemplates.yaml index db8b1f189..a9b33a3f8 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gametemplates.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gametemplates.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: ceab4a3a14adff11367ab42c914aaf44a34965e6bea1fb30ecf7ea2fadec09a1 + gameplane.local/crd-bundle-sha256: 1dc8f563ef15434a03e2f9c2f61c92576d3a3483dd19c787434ed8b4b743cacd controller-gen.kubebuilder.io/version: v0.20.1 name: gametemplates.gameplane.local spec: @@ -961,6 +961,41 @@ spec: maxLength: 63 pattern: ^[a-z0-9]([a-z0-9-]*[a-z0-9])?$ type: string + performance: + description: |- + Performance opts this existing readout into cached numeric monitoring. + Display-only strings are never inferred to be performance measurements. + properties: + degradedThreshold: + description: |- + DegradedThreshold is a finite nonnegative decimal string in the input + unit. Rates below it, or durations above it, are degraded. Omit to + disable alerts. A string keeps JSON/schema handling portable. + maxLength: 16 + pattern: ^(0|[1-9][0-9]{0,8})(\.[0-9]{1,6})?$ + type: string + signal: + enum: + - tick_rate + - tick_duration + - frame_rate + type: string + unit: + enum: + - per_second + - seconds + - milliseconds + type: string + required: + - signal + - unit + type: object + x-kubernetes-validations: + - message: duration requires seconds/milliseconds; rates + require per_second + rule: 'self.signal == ''tick_duration'' ? self.unit + in [''seconds'', ''milliseconds''] : self.unit == + ''per_second''' regex: description: |- Regex extracts the value via the named group "value", e.g. diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modules.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modules.yaml index 63a2264c3..e5d1f6c77 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modules.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modules.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: ceab4a3a14adff11367ab42c914aaf44a34965e6bea1fb30ecf7ea2fadec09a1 + gameplane.local/crd-bundle-sha256: 1dc8f563ef15434a03e2f9c2f61c92576d3a3483dd19c787434ed8b4b743cacd controller-gen.kubebuilder.io/version: v0.20.1 name: modules.gameplane.local spec: diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modulesources.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modulesources.yaml index b21bf973b..d618481a4 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modulesources.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modulesources.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: ceab4a3a14adff11367ab42c914aaf44a34965e6bea1fb30ecf7ea2fadec09a1 + gameplane.local/crd-bundle-sha256: 1dc8f563ef15434a03e2f9c2f61c92576d3a3483dd19c787434ed8b4b743cacd controller-gen.kubebuilder.io/version: v0.20.1 name: modulesources.gameplane.local spec: diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_networkcaptures.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_networkcaptures.yaml index 9f9c0da36..4d7d7c25f 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_networkcaptures.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_networkcaptures.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: ceab4a3a14adff11367ab42c914aaf44a34965e6bea1fb30ecf7ea2fadec09a1 + gameplane.local/crd-bundle-sha256: 1dc8f563ef15434a03e2f9c2f61c92576d3a3483dd19c787434ed8b4b743cacd controller-gen.kubebuilder.io/version: v0.20.1 name: networkcaptures.gameplane.local spec: diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_restores.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_restores.yaml index fe4e219df..f5a47f635 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_restores.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_restores.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: ceab4a3a14adff11367ab42c914aaf44a34965e6bea1fb30ecf7ea2fadec09a1 + gameplane.local/crd-bundle-sha256: 1dc8f563ef15434a03e2f9c2f61c92576d3a3483dd19c787434ed8b4b743cacd controller-gen.kubebuilder.io/version: v0.20.1 name: restores.gameplane.local spec: diff --git a/charts/gameplane/crd-manifests/gameplane.local_backups.yaml b/charts/gameplane/crd-manifests/gameplane.local_backups.yaml index dbbcda875..448222049 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_backups.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_backups.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: ceab4a3a14adff11367ab42c914aaf44a34965e6bea1fb30ecf7ea2fadec09a1 + gameplane.local/crd-bundle-sha256: 1dc8f563ef15434a03e2f9c2f61c92576d3a3483dd19c787434ed8b4b743cacd controller-gen.kubebuilder.io/version: v0.20.1 name: backups.gameplane.local spec: diff --git a/charts/gameplane/crd-manifests/gameplane.local_backupschedules.yaml b/charts/gameplane/crd-manifests/gameplane.local_backupschedules.yaml index 9aa2aa7b8..cae7c3a68 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_backupschedules.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_backupschedules.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: ceab4a3a14adff11367ab42c914aaf44a34965e6bea1fb30ecf7ea2fadec09a1 + gameplane.local/crd-bundle-sha256: 1dc8f563ef15434a03e2f9c2f61c92576d3a3483dd19c787434ed8b4b743cacd controller-gen.kubebuilder.io/version: v0.20.1 name: backupschedules.gameplane.local spec: diff --git a/charts/gameplane/crd-manifests/gameplane.local_clusters.yaml b/charts/gameplane/crd-manifests/gameplane.local_clusters.yaml index 436cc742e..0dff5ac61 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_clusters.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_clusters.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: ceab4a3a14adff11367ab42c914aaf44a34965e6bea1fb30ecf7ea2fadec09a1 + gameplane.local/crd-bundle-sha256: 1dc8f563ef15434a03e2f9c2f61c92576d3a3483dd19c787434ed8b4b743cacd controller-gen.kubebuilder.io/version: v0.20.1 name: clusters.gameplane.local spec: diff --git a/charts/gameplane/crd-manifests/gameplane.local_gameservers.yaml b/charts/gameplane/crd-manifests/gameplane.local_gameservers.yaml index dea6133a3..bf338daa1 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_gameservers.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_gameservers.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: ceab4a3a14adff11367ab42c914aaf44a34965e6bea1fb30ecf7ea2fadec09a1 + gameplane.local/crd-bundle-sha256: 1dc8f563ef15434a03e2f9c2f61c92576d3a3483dd19c787434ed8b4b743cacd controller-gen.kubebuilder.io/version: v0.20.1 name: gameservers.gameplane.local spec: diff --git a/charts/gameplane/crd-manifests/gameplane.local_gametemplates.yaml b/charts/gameplane/crd-manifests/gameplane.local_gametemplates.yaml index db8b1f189..a9b33a3f8 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_gametemplates.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_gametemplates.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: ceab4a3a14adff11367ab42c914aaf44a34965e6bea1fb30ecf7ea2fadec09a1 + gameplane.local/crd-bundle-sha256: 1dc8f563ef15434a03e2f9c2f61c92576d3a3483dd19c787434ed8b4b743cacd controller-gen.kubebuilder.io/version: v0.20.1 name: gametemplates.gameplane.local spec: @@ -961,6 +961,41 @@ spec: maxLength: 63 pattern: ^[a-z0-9]([a-z0-9-]*[a-z0-9])?$ type: string + performance: + description: |- + Performance opts this existing readout into cached numeric monitoring. + Display-only strings are never inferred to be performance measurements. + properties: + degradedThreshold: + description: |- + DegradedThreshold is a finite nonnegative decimal string in the input + unit. Rates below it, or durations above it, are degraded. Omit to + disable alerts. A string keeps JSON/schema handling portable. + maxLength: 16 + pattern: ^(0|[1-9][0-9]{0,8})(\.[0-9]{1,6})?$ + type: string + signal: + enum: + - tick_rate + - tick_duration + - frame_rate + type: string + unit: + enum: + - per_second + - seconds + - milliseconds + type: string + required: + - signal + - unit + type: object + x-kubernetes-validations: + - message: duration requires seconds/milliseconds; rates + require per_second + rule: 'self.signal == ''tick_duration'' ? self.unit + in [''seconds'', ''milliseconds''] : self.unit == + ''per_second''' regex: description: |- Regex extracts the value via the named group "value", e.g. diff --git a/charts/gameplane/crd-manifests/gameplane.local_modules.yaml b/charts/gameplane/crd-manifests/gameplane.local_modules.yaml index 63a2264c3..e5d1f6c77 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_modules.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_modules.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: ceab4a3a14adff11367ab42c914aaf44a34965e6bea1fb30ecf7ea2fadec09a1 + gameplane.local/crd-bundle-sha256: 1dc8f563ef15434a03e2f9c2f61c92576d3a3483dd19c787434ed8b4b743cacd controller-gen.kubebuilder.io/version: v0.20.1 name: modules.gameplane.local spec: diff --git a/charts/gameplane/crd-manifests/gameplane.local_modulesources.yaml b/charts/gameplane/crd-manifests/gameplane.local_modulesources.yaml index b21bf973b..d618481a4 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_modulesources.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_modulesources.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: ceab4a3a14adff11367ab42c914aaf44a34965e6bea1fb30ecf7ea2fadec09a1 + gameplane.local/crd-bundle-sha256: 1dc8f563ef15434a03e2f9c2f61c92576d3a3483dd19c787434ed8b4b743cacd controller-gen.kubebuilder.io/version: v0.20.1 name: modulesources.gameplane.local spec: diff --git a/charts/gameplane/crd-manifests/gameplane.local_networkcaptures.yaml b/charts/gameplane/crd-manifests/gameplane.local_networkcaptures.yaml index 9f9c0da36..4d7d7c25f 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_networkcaptures.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_networkcaptures.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: ceab4a3a14adff11367ab42c914aaf44a34965e6bea1fb30ecf7ea2fadec09a1 + gameplane.local/crd-bundle-sha256: 1dc8f563ef15434a03e2f9c2f61c92576d3a3483dd19c787434ed8b4b743cacd controller-gen.kubebuilder.io/version: v0.20.1 name: networkcaptures.gameplane.local spec: diff --git a/charts/gameplane/crd-manifests/gameplane.local_restores.yaml b/charts/gameplane/crd-manifests/gameplane.local_restores.yaml index fe4e219df..f5a47f635 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_restores.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_restores.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: ceab4a3a14adff11367ab42c914aaf44a34965e6bea1fb30ecf7ea2fadec09a1 + gameplane.local/crd-bundle-sha256: 1dc8f563ef15434a03e2f9c2f61c92576d3a3483dd19c787434ed8b4b743cacd controller-gen.kubebuilder.io/version: v0.20.1 name: restores.gameplane.local spec: diff --git a/charts/gameplane/dashboards/gameplane-workloads.json b/charts/gameplane/dashboards/gameplane-workloads.json new file mode 100644 index 000000000..f0a978b26 --- /dev/null +++ b/charts/gameplane/dashboards/gameplane-workloads.json @@ -0,0 +1,345 @@ +{ + "title": "Gameplane Workloads", + "uid": "gameplane-workloads", + "tags": [ + "gameplane" + ], + "schemaVersion": 39, + "version": 1, + "editable": true, + "refresh": "30s", + "time": { + "from": "now-6h", + "to": "now" + }, + "templating": { + "list": [ + { + "name": "datasource", + "type": "datasource", + "query": "prometheus" + }, + { + "name": "namespace", + "type": "textbox", + "label": "Games namespace", + "current": { + "value": "gameplane-games", + "text": "gameplane-games" + } + } + ] + }, + "panels": [ + { + "id": 1, + "title": "Scrape health", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "noValue": "Unknown", + "custom": { + "spanNulls": false + } + } + }, + "targets": [ + { + "refId": "A", + "expr": "max by (namespace,pod,gameplane_component) (up{gameplane_component=~\"agent|tunnel\",namespace=~\"$namespace\"})", + "legendFormat": "{{namespace}} / {{pod}} {{metric}}" + } + ] + }, + { + "id": 2, + "title": "Relay readiness", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "noValue": "Unknown", + "custom": { + "spanNulls": false + } + } + }, + "targets": [ + { + "refId": "A", + "expr": "max by (namespace,pod) (gameplane_tunnel_ready{namespace=~\"$namespace\"})", + "legendFormat": "{{namespace}} / {{pod}} {{metric}}" + } + ] + }, + { + "id": 3, + "title": "Relay health known", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "noValue": "Unknown", + "custom": { + "spanNulls": false + } + } + }, + "targets": [ + { + "refId": "A", + "expr": "max by (namespace,pod) (gameplane_tunnel_connection_known{namespace=~\"$namespace\"})", + "legendFormat": "{{namespace}} / {{pod}} {{metric}}" + } + ] + }, + { + "id": 4, + "title": "Game rates (declared signals)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "unit": "hertz", + "noValue": "Unknown", + "custom": { + "spanNulls": false + } + } + }, + "targets": [ + { + "refId": "A", + "expr": "gameplane_agent_performance_value{namespace=~\"$namespace\",unit=\"hertz\"}", + "legendFormat": "{{namespace}} / {{pod}} {{metric}}" + } + ] + }, + { + "id": 5, + "title": "Game tick duration", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "noValue": "Unknown", + "custom": { + "spanNulls": false + } + } + }, + "targets": [ + { + "refId": "A", + "expr": "gameplane_agent_performance_value{namespace=~\"$namespace\",unit=\"seconds\"}", + "legendFormat": "{{namespace}} / {{pod}} {{metric}}" + } + ] + }, + { + "id": 6, + "title": "Performance available", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "noValue": "Unknown", + "custom": { + "spanNulls": false + } + } + }, + "targets": [ + { + "refId": "A", + "expr": "gameplane_agent_performance_available{namespace=~\"$namespace\"}", + "legendFormat": "{{namespace}} / {{pod}} {{metric}}" + } + ] + }, + { + "id": 7, + "title": "Container CPU throttled periods", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "noValue": "Unknown", + "custom": { + "spanNulls": false + } + } + }, + "targets": [ + { + "refId": "A", + "expr": "max by (namespace,pod) (rate(container_cpu_cfs_throttled_periods_total{namespace=~\"$namespace\",container=\"game\"}[5m]) / rate(container_cpu_cfs_periods_total{namespace=~\"$namespace\",container=\"game\"}[5m]))", + "legendFormat": "{{namespace}} / {{pod}} {{metric}}" + } + ] + }, + { + "id": 8, + "title": "Container working set / limit", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "noValue": "Unknown", + "custom": { + "spanNulls": false + } + } + }, + "targets": [ + { + "refId": "A", + "expr": "max by (namespace,pod) (container_memory_working_set_bytes{namespace=~\"$namespace\",container=\"game\"} / (container_spec_memory_limit_bytes{namespace=~\"$namespace\",container=\"game\"} > 0))", + "legendFormat": "{{namespace}} / {{pod}} {{metric}}" + } + ] + }, + { + "id": 9, + "title": "Container restarts", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 32 + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "noValue": "Unknown", + "custom": { + "spanNulls": false + } + } + }, + "targets": [ + { + "refId": "A", + "expr": "max by (namespace,pod,container) (increase(kube_pod_container_status_restarts_total{namespace=~\"$namespace\"}[15m]))", + "legendFormat": "{{namespace}} / {{pod}} {{metric}}" + } + ] + }, + { + "id": 10, + "title": "Pending Pods", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 32 + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "noValue": "Unknown", + "custom": { + "spanNulls": false + } + } + }, + "targets": [ + { + "refId": "A", + "expr": "max by (namespace,pod) (kube_pod_status_phase{namespace=~\"$namespace\",phase=\"Pending\"})", + "legendFormat": "{{namespace}} / {{pod}} {{metric}}" + } + ] + } + ] +} diff --git a/charts/gameplane/templates/NOTES.txt b/charts/gameplane/templates/NOTES.txt index 6f5d24139..e33acd801 100644 --- a/charts/gameplane/templates/NOTES.txt +++ b/charts/gameplane/templates/NOTES.txt @@ -4,6 +4,16 @@ persistence and dashboard blocks are absent. */ -}} {{- $r := .Values.api.telemetry.receiver }} +{{- if .Values.serviceMonitors.enabled }} + +Monitoring: configure Prometheus monitor label selectors AND discovery of the +release and games namespaces. Verify agent and tunnel targets have up=1. +See docs/monitoring.md and docs/examples/monitoring for a complete configuration. +{{- if .Values.serviceMonitors.externallyManagedPolicy }} +External scrape policies are selected: this setting opens no network access. +Ensure your policies allow the scraper to agent TCP 9090 and tunnel TCP 8091. +{{- end }} +{{- end }} {{- if .Values.api.standalone }} Standalone central panel: no local game cluster or Kubernetes API credentials. diff --git a/charts/gameplane/templates/grafana-dashboard.yaml b/charts/gameplane/templates/grafana-dashboard.yaml index de1052252..9c9664fb2 100644 --- a/charts/gameplane/templates/grafana-dashboard.yaml +++ b/charts/gameplane/templates/grafana-dashboard.yaml @@ -15,6 +15,8 @@ metadata: {{- toYaml . | nindent 4 }} {{- end }} data: + gameplane-workloads.json: |- + {{- .Files.Get "dashboards/gameplane-workloads.json" | nindent 4 }} gameplane-operator.json: |- {{- .Files.Get "dashboards/gameplane-operator.json" | nindent 4 }} {{- end }} diff --git a/charts/gameplane/templates/monitoring-rules.yaml b/charts/gameplane/templates/monitoring-rules.yaml new file mode 100644 index 000000000..120c5c9a6 --- /dev/null +++ b/charts/gameplane/templates/monitoring-rules.yaml @@ -0,0 +1,72 @@ +{{- if and .Values.prometheusRules.enabled (eq (include "gameplane.operatorEnabled" .) "true") }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: gameplane-workloads + namespace: {{ .Release.Namespace }} + labels: + {{- include "gameplane.labels" . | nindent 4 }} + {{- with .Values.prometheusRules.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + groups: + - name: gameplane.workloads + rules: + - alert: GameplaneScrapeUnavailable + expr: max by (namespace, pod, gameplane_component) (up{gameplane_component=~"agent|tunnel",namespace="{{ .Values.gamesNamespace }}"}) == 0 + for: 5m + labels: { severity: warning } + annotations: { summary: "Gameplane metrics target cannot be scraped" } + - alert: GameplaneRelayUnavailable + expr: (max by (namespace, pod) (gameplane_tunnel_ready{namespace="{{ .Values.gamesNamespace }}"}) == 0) and on (namespace, pod) (max by (namespace, pod) (gameplane_tunnel_connection_known{namespace="{{ .Values.gamesNamespace }}"}) == 1) + for: 2m + labels: { severity: warning } + annotations: { summary: "Gameplane relay connection or forwards are unavailable" } + - alert: GameplaneRelayHealthUnknown + expr: max by (namespace, pod) (gameplane_tunnel_connection_known{namespace="{{ .Values.gamesNamespace }}"}) == 0 + for: 5m + labels: { severity: info } + annotations: { summary: "Gameplane provider cannot confirm live relay health" } + - alert: GameplanePerformanceUnknown + expr: max by (namespace, pod, metric, signal, unit) (gameplane_agent_performance_available{namespace="{{ .Values.gamesNamespace }}"}) == 0 + for: 5m + labels: { severity: warning } + annotations: { summary: "Declared game performance measurement is unavailable" } + - alert: GameplanePerformanceDegraded + expr: | + (max by (namespace, pod, metric, signal, unit) (gameplane_agent_performance_value{namespace="{{ .Values.gamesNamespace }}"}) + < on (namespace, pod, metric, signal, unit) + max by (namespace, pod, metric, signal, unit) (gameplane_agent_performance_degraded_threshold{namespace="{{ .Values.gamesNamespace }}",direction="below"})) + or + (max by (namespace, pod, metric, signal, unit) (gameplane_agent_performance_value{namespace="{{ .Values.gamesNamespace }}"}) + > on (namespace, pod, metric, signal, unit) + max by (namespace, pod, metric, signal, unit) (gameplane_agent_performance_degraded_threshold{namespace="{{ .Values.gamesNamespace }}",direction="above"})) + for: 5m + labels: { severity: warning } + annotations: { summary: "Game performance crossed its module-declared threshold" } + - alert: GameplaneCPUThrottled + expr: max by (namespace, pod, container) (rate(container_cpu_cfs_throttled_periods_total{namespace="{{ .Values.gamesNamespace }}",container="game"}[5m]) / rate(container_cpu_cfs_periods_total{namespace="{{ .Values.gamesNamespace }}",container="game"}[5m])) > 0.25 + for: 10m + labels: { severity: warning } + annotations: { summary: "Game container CPU is throttled in more than 25% of periods" } + - alert: GameplaneMemoryPressure + expr: max by (namespace, pod, container) (container_memory_working_set_bytes{namespace="{{ .Values.gamesNamespace }}",container="game"} / (container_spec_memory_limit_bytes{namespace="{{ .Values.gamesNamespace }}",container="game"} > 0)) > 0.9 + for: 10m + labels: { severity: warning } + annotations: { summary: "Game container working set exceeds 90% of its memory limit" } + - alert: GameplaneContainerOOM + expr: (max by (namespace, pod, container) (increase(kube_pod_container_status_restarts_total{namespace="{{ .Values.gamesNamespace }}"}[10m])) > 0) and on (namespace, pod, container) (max by (namespace, pod, container) (kube_pod_container_status_last_terminated_reason{namespace="{{ .Values.gamesNamespace }}",reason="OOMKilled"}) == 1) + labels: { severity: warning } + annotations: { summary: "Gameplane container restarted after an OOM kill" } + - alert: GameplanePodPending + expr: max by (namespace, pod) (kube_pod_status_phase{namespace="{{ .Values.gamesNamespace }}",phase="Pending"}) == 1 + for: 10m + labels: { severity: warning } + annotations: { summary: "Gameplane Pod has remained Pending for ten minutes" } + - alert: GameplaneNodePressure + expr: (max by (node, condition) (kube_node_status_condition{condition=~"MemoryPressure|DiskPressure|PIDPressure",status="true"}) == 1) and on (node) max by (node) (kube_pod_info{namespace="{{ .Values.gamesNamespace }}"}) + for: 5m + labels: { severity: warning } + annotations: { summary: "A node hosting Gameplane Pods reports resource pressure" } +{{- end }} diff --git a/charts/gameplane/templates/monitoring-validation.yaml b/charts/gameplane/templates/monitoring-validation.yaml new file mode 100644 index 000000000..2e4dc0d7c --- /dev/null +++ b/charts/gameplane/templates/monitoring-validation.yaml @@ -0,0 +1,3 @@ +{{- if and .Values.serviceMonitors.enabled .Values.networkPolicies.enabled (not .Values.serviceMonitors.scrapeNamespaceSelector) (not .Values.serviceMonitors.externallyManagedPolicy) }} +{{- fail "serviceMonitors.enabled with networkPolicies.enabled requires serviceMonitors.scrapeNamespaceSelector (for example matchLabels: {kubernetes.io/metadata.name: monitoring}); alternatively set serviceMonitors.externallyManagedPolicy=true and supply your own scrape ingress policies" }} +{{- end }} diff --git a/charts/gameplane/templates/networkpolicies.yaml b/charts/gameplane/templates/networkpolicies.yaml index 612fc5071..d652bd58b 100644 --- a/charts/gameplane/templates/networkpolicies.yaml +++ b/charts/gameplane/templates/networkpolicies.yaml @@ -163,6 +163,24 @@ spec: {{- toYaml . | nindent 12 }} ports: - { protocol: TCP, port: 9090 } +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-prometheus-to-tunnel + namespace: {{ $.Values.gamesNamespace }} + labels: {{- include "gameplane.labels" $ | nindent 4 }} +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: gameplane-tunnel + policyTypes: [Ingress] + ingress: + - from: + - namespaceSelector: + {{- toYaml . | nindent 12 }} + ports: + - { protocol: TCP, port: 8091 } {{- end }} {{- end }} --- diff --git a/charts/gameplane/templates/servicemonitors.yaml b/charts/gameplane/templates/servicemonitors.yaml index 4521db18e..2b5b6834b 100644 --- a/charts/gameplane/templates/servicemonitors.yaml +++ b/charts/gameplane/templates/servicemonitors.yaml @@ -107,6 +107,34 @@ spec: - port: metrics interval: 30s path: /metrics + relabelings: + - targetLabel: gameplane_component + replacement: agent + - sourceLabels: [__meta_kubernetes_pod_label_app_kubernetes_io_instance] + targetLabel: server +--- +apiVersion: monitoring.coreos.com/v1 +kind: PodMonitor +metadata: + name: gameplane-tunnel + namespace: {{ .Values.gamesNamespace }} + labels: + {{- include "gameplane.labels" . | nindent 4 }} + {{- with .Values.serviceMonitors.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + selector: + matchLabels: { app.kubernetes.io/name: gameplane-tunnel } + podMetricsEndpoints: + - port: relay-metrics + interval: 15s + path: /metrics + relabelings: + - targetLabel: gameplane_component + replacement: tunnel + - sourceLabels: [__meta_kubernetes_pod_label_app_kubernetes_io_instance] + targetLabel: server {{- end }} {{- end }} diff --git a/charts/gameplane/tests/monitoring-rules.sh b/charts/gameplane/tests/monitoring-rules.sh new file mode 100644 index 000000000..6b754c69b --- /dev/null +++ b/charts/gameplane/tests/monitoring-rules.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# Render and evaluate shipped rules in CI, using the same Prometheus as E2E. +set -euo pipefail +cd "$(dirname "$0")/../../.." +scratch="$(mktemp -d)" +trap 'rm -rf "$scratch"' EXIT +helm template gameplane charts/gameplane -f docs/examples/monitoring/gameplane-values.yaml > "$scratch/rendered.yaml" +python3 - "$scratch" <<'PY' +import pathlib,sys,yaml +root=pathlib.Path(sys.argv[1]) +groups=[] +for document in yaml.safe_load_all((root/'rendered.yaml').read_text()): + if document and document.get('kind')=='PrometheusRule': + groups.extend(document['spec']['groups']) +(root/'rules.yaml').write_text(yaml.safe_dump({'groups':groups})) +PY +cp charts/gameplane/tests/monitoring-rules.test.yaml "$scratch/tests.yaml" +docker run --rm --volume "$scratch:/work:ro" --workdir /work --entrypoint /bin/promtool \ + prom/prometheus:v3.13.4 test rules tests.yaml diff --git a/charts/gameplane/tests/monitoring-rules.test.yaml b/charts/gameplane/tests/monitoring-rules.test.yaml new file mode 100644 index 000000000..3bd63c84f --- /dev/null +++ b/charts/gameplane/tests/monitoring-rules.test.yaml @@ -0,0 +1,146 @@ +rule_files: [rules.yaml] +evaluation_interval: 1m +tests: + - interval: 1m + input_series: + - series: 'gameplane_agent_performance_value{namespace="gameplane-games",pod="duration-0",metric="tick",signal="tick_duration",unit="seconds"}' + values: '0.06x10 0.04x10' + - series: 'gameplane_agent_performance_degraded_threshold{namespace="gameplane-games",pod="duration-0",metric="tick",signal="tick_duration",unit="seconds",direction="above"}' + values: '0.05x20' + - series: 'container_memory_working_set_bytes{namespace="gameplane-games",pod="unlimited",container="game"}' + values: '1000x20' + - series: 'container_spec_memory_limit_bytes{namespace="gameplane-games",pod="unlimited",container="game"}' + values: '0x20' + alert_rule_test: + - eval_time: 6m + alertname: GameplanePerformanceDegraded + exp_alerts: + - exp_labels: {namespace: gameplane-games, pod: duration-0, metric: tick, signal: tick_duration, unit: seconds, severity: warning} + exp_annotations: {summary: "Game performance crossed its module-declared threshold"} + - eval_time: 12m + alertname: GameplanePerformanceDegraded + exp_alerts: [] + - eval_time: 16m + alertname: GameplaneMemoryPressure + exp_alerts: [] + - eval_time: 16m + alertname: GameplanePerformanceUnknown + exp_alerts: [] + - interval: 1m + input_series: + - series: 'up{namespace="gameplane-games",pod="game-0",gameplane_component="agent"}' + values: '0x10 1x10' + - series: 'gameplane_tunnel_ready{namespace="gameplane-games",pod="relay"}' + values: '0x10 1x10' + - series: 'gameplane_tunnel_connection_known{namespace="gameplane-games",pod="relay"}' + values: '1x20' + alert_rule_test: + - eval_time: 6m + alertname: GameplaneScrapeUnavailable + exp_alerts: + - exp_labels: {namespace: gameplane-games, pod: game-0, gameplane_component: agent, severity: warning} + exp_annotations: {summary: "Gameplane metrics target cannot be scraped"} + - eval_time: 6m + alertname: GameplaneRelayUnavailable + exp_alerts: + - exp_labels: {namespace: gameplane-games, pod: relay, severity: warning} + exp_annotations: {summary: "Gameplane relay connection or forwards are unavailable"} + - eval_time: 12m + alertname: GameplaneScrapeUnavailable + exp_alerts: [] + - eval_time: 12m + alertname: GameplaneRelayUnavailable + exp_alerts: [] + - interval: 1m + input_series: + - series: 'gameplane_agent_performance_value{namespace="gameplane-games",pod="game-0",metric="tps",signal="tick_rate",unit="hertz"}' + values: '0x10 stale _x10' + - series: 'gameplane_agent_performance_degraded_threshold{namespace="gameplane-games",pod="game-0",metric="tps",signal="tick_rate",unit="hertz",direction="below"}' + values: '18x20' + - series: 'gameplane_agent_performance_available{namespace="gameplane-games",pod="game-0",metric="tps",signal="tick_rate",unit="hertz"}' + values: '1x10 0x10' + - series: 'gameplane_tunnel_connection_known{namespace="gameplane-games",pod="playit"}' + values: '0x10 1x10' + - series: 'gameplane_tunnel_ready{namespace="gameplane-games",pod="playit"}' + values: '0x10 1x10' + alert_rule_test: + - eval_time: 6m + alertname: GameplanePerformanceDegraded + exp_alerts: + - exp_labels: {namespace: gameplane-games, pod: game-0, metric: tps, signal: tick_rate, unit: hertz, severity: warning} + exp_annotations: {summary: "Game performance crossed its module-declared threshold"} + - eval_time: 12m + alertname: GameplanePerformanceDegraded + exp_alerts: [] + - eval_time: 17m + alertname: GameplanePerformanceUnknown + exp_alerts: + - exp_labels: {namespace: gameplane-games, pod: game-0, metric: tps, signal: tick_rate, unit: hertz, severity: warning} + exp_annotations: {summary: "Declared game performance measurement is unavailable"} + - eval_time: 6m + alertname: GameplaneRelayHealthUnknown + exp_alerts: + - exp_labels: {namespace: gameplane-games, pod: playit, severity: info} + exp_annotations: {summary: "Gameplane provider cannot confirm live relay health"} + - eval_time: 6m + alertname: GameplaneRelayUnavailable + exp_alerts: [] + - eval_time: 12m + alertname: GameplaneRelayHealthUnknown + exp_alerts: [] + - interval: 1m + input_series: + - series: 'container_cpu_cfs_throttled_periods_total{namespace="gameplane-games",pod="game-0",container="game"}' + values: '0+50x25' + - series: 'container_cpu_cfs_periods_total{namespace="gameplane-games",pod="game-0",container="game"}' + values: '0+100x25' + - series: 'container_memory_working_set_bytes{namespace="gameplane-games",pod="game-0",container="game"}' + values: '95x25' + - series: 'container_spec_memory_limit_bytes{namespace="gameplane-games",pod="game-0",container="game"}' + values: '100x25' + - series: 'kube_pod_status_phase{namespace="gameplane-games",pod="pending",phase="Pending"}' + values: '1x25' + - series: 'kube_pod_container_status_restarts_total{namespace="gameplane-games",pod="game-0",container="game"}' + values: '0+1x25' + - series: 'kube_pod_container_status_last_terminated_reason{namespace="gameplane-games",pod="game-0",container="game",reason="OOMKilled"}' + values: '1x25' + - series: 'kube_node_status_condition{node="worker",condition="MemoryPressure",status="true"}' + values: '1x25' + - series: 'kube_pod_info{namespace="gameplane-games",pod="game-0",node="worker"}' + values: '1x25' + - series: 'gameplane_gameservers{phase="Failed",instance="operator-a"}' + values: '1x25' + - series: 'gameplane_gameservers{phase="Failed",instance="operator-b"}' + values: '1x25' + alert_rule_test: + - eval_time: 16m + alertname: GameplaneCPUThrottled + exp_alerts: + - exp_labels: {namespace: gameplane-games, pod: game-0, container: game, severity: warning} + exp_annotations: {summary: "Game container CPU is throttled in more than 25% of periods"} + - eval_time: 16m + alertname: GameplaneMemoryPressure + exp_alerts: + - exp_labels: {namespace: gameplane-games, pod: game-0, container: game, severity: warning} + exp_annotations: {summary: "Game container working set exceeds 90% of its memory limit"} + - eval_time: 16m + alertname: GameplanePodPending + exp_alerts: + - exp_labels: {namespace: gameplane-games, pod: pending, severity: warning} + exp_annotations: {summary: "Gameplane Pod has remained Pending for ten minutes"} + - eval_time: 16m + alertname: GameplaneContainerOOM + exp_alerts: + - exp_labels: {namespace: gameplane-games, pod: game-0, container: game, severity: warning} + exp_annotations: {summary: "Gameplane container restarted after an OOM kill"} + - eval_time: 16m + alertname: GameplaneNodePressure + exp_alerts: + - exp_labels: {node: worker, condition: MemoryPressure, severity: warning} + exp_annotations: {summary: "A node hosting Gameplane Pods reports resource pressure"} + promql_expr_test: + - expr: 'max(gameplane_gameservers{phase="Failed"})' + eval_time: 16m + exp_samples: + - labels: '{}' + value: 1 diff --git a/charts/gameplane/values.yaml b/charts/gameplane/values.yaml index bb6268f16..9f6f530f7 100644 --- a/charts/gameplane/values.yaml +++ b/charts/gameplane/values.yaml @@ -676,6 +676,9 @@ updates: channel: "" serviceMonitors: + # Explicit escape for independently managed scrape ingress policies. This + # does not open any ports; you must install and verify those policies. + externallyManagedPolicy: false enabled: false # when true, creates ServiceMonitors for operator/api/telemetry-receiver and a PodMonitor for agent sidecars on their separate plain metrics port 9090 (the control port 8090 stays mTLS-only) labels: {} # additional labels applied to ServiceMonitor/PodMonitor objects for Prometheus selector matching # Namespace selector (standard Kubernetes namespaceSelector label match @@ -688,8 +691,8 @@ serviceMonitors: # api.telemetry.receiver.dashboard.tokenSecretRef.name) when # networkPolicies.enabled is also true. Required # alongside enabled: true whenever networkPolicies.enabled is also true; - # left empty, both stay unreachable to Prometheus even though their - # ServiceMonitor/PodMonitor objects are rendered. Example: + # left empty, chart validation fails unless externallyManagedPolicy is true. + # Also permits tunnel health/metrics on port 8091. Example: # `{matchLabels: {kubernetes.io/metadata.name: monitoring}}`. scrapeNamespaceSelector: {} diff --git a/deploy/kind/monitoring.sh b/deploy/kind/monitoring.sh new file mode 100644 index 000000000..1bbc20063 --- /dev/null +++ b/deploy/kind/monitoring.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# CI-only real scrape fixture; never invoke as a local validation shortcut. +set -euo pipefail +cd "$(dirname "$0")/../.." +context="${CTX:-kind-gameplane-e2e}" +kubectl() { command kubectl --context "$context" "$@"; } +scratch="$(mktemp -d)" +trap 'rm -rf "$scratch"' EXIT +curl --fail --silent --show-error --location \ + https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.94.1/bundle.yaml \ + -o "$scratch/bundle.yaml" +echo "7ab177610b271c70c57c7d6fc550a1e934eff4f707a5d8b61dfaa3845d52deea $scratch/bundle.yaml" | sha256sum --check +kubectl create namespace monitoring --dry-run=client -o yaml | kubectl apply -f - +# The official bundle's namespaced objects and binding subjects use default. +sed 's/namespace: default/namespace: monitoring/g' "$scratch/bundle.yaml" > "$scratch/monitoring.yaml" +kubectl apply --server-side -f "$scratch/monitoring.yaml" +kubectl wait --for=condition=Established --timeout=120s crd/prometheuses.monitoring.coreos.com crd/podmonitors.monitoring.coreos.com crd/servicemonitors.monitoring.coreos.com crd/prometheusrules.monitoring.coreos.com +kubectl rollout status -n monitoring deployment/prometheus-operator --timeout=180s +helm --kube-context "$context" upgrade gameplane charts/gameplane --namespace gameplane-system --reuse-values \ + -f docs/examples/monitoring/gameplane-values.yaml --wait --timeout 5m +kubectl apply -f docs/examples/monitoring/prometheus.yaml +kubectl wait -n monitoring --for=create statefulset/prometheus-gameplane --timeout=120s +kubectl rollout status -n monitoring statefulset/prometheus-gameplane --timeout=180s diff --git a/docs/examples/monitoring/gameplane-values.yaml b/docs/examples/monitoring/gameplane-values.yaml new file mode 100644 index 000000000..8b51fe3cb --- /dev/null +++ b/docs/examples/monitoring/gameplane-values.yaml @@ -0,0 +1,19 @@ +# Apply to the Gameplane Helm release. See docs/monitoring.md for node-specific +# strict NetworkPolicy settings and the companion Prometheus resource. +networkPolicies: + enabled: true +serviceMonitors: + enabled: true + labels: + monitoring: gameplane + scrapeNamespaceSelector: + matchLabels: + kubernetes.io/metadata.name: monitoring +prometheusRules: + enabled: true + labels: + monitoring: gameplane +grafanaDashboards: + enabled: true + labels: + grafana_dashboard: "1" diff --git a/docs/examples/monitoring/prometheus.yaml b/docs/examples/monitoring/prometheus.yaml new file mode 100644 index 000000000..11793ace4 --- /dev/null +++ b/docs/examples/monitoring/prometheus.yaml @@ -0,0 +1,71 @@ +# Requires the Prometheus Operator and its CRDs. The monitoring namespace must +# match serviceMonitors.scrapeNamespaceSelector in gameplane-values.yaml. +apiVersion: v1 +kind: Namespace +metadata: + name: monitoring +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: gameplane-prometheus + namespace: monitoring +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: gameplane-prometheus-discovery +rules: + - apiGroups: [""] + resources: [nodes, nodes/metrics, services, endpoints, pods] + verbs: [get, list, watch] + - apiGroups: [discovery.k8s.io] + resources: [endpointslices] + verbs: [get, list, watch] + - nonResourceURLs: [/metrics] + verbs: [get] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: gameplane-prometheus-discovery +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: gameplane-prometheus-discovery +subjects: + - kind: ServiceAccount + name: gameplane-prometheus + namespace: monitoring +--- +apiVersion: monitoring.coreos.com/v1 +kind: Prometheus +metadata: + name: gameplane + namespace: monitoring +spec: + version: v3.13.4 + serviceAccountName: gameplane-prometheus + replicas: 1 + retention: 24h + resources: + requests: { cpu: 100m, memory: 256Mi } + limits: { memory: 1Gi } + serviceMonitorSelector: + matchLabels: { monitoring: gameplane } + serviceMonitorNamespaceSelector: + matchExpressions: + - key: kubernetes.io/metadata.name + operator: In + values: [gameplane-system, gameplane-games] + podMonitorSelector: + matchLabels: { monitoring: gameplane } + podMonitorNamespaceSelector: + matchExpressions: + - key: kubernetes.io/metadata.name + operator: In + values: [gameplane-system, gameplane-games] + ruleSelector: + matchLabels: { monitoring: gameplane } + ruleNamespaceSelector: + matchLabels: { kubernetes.io/metadata.name: gameplane-system } diff --git a/docs/monitoring.md b/docs/monitoring.md new file mode 100644 index 000000000..3ec5f53db --- /dev/null +++ b/docs/monitoring.md @@ -0,0 +1,142 @@ +# Monitoring games and relays + +Gameplane exports the existing agent CPU, process memory, disk and player gauges, +optional game performance measurements, and relay health. Prometheus and Grafana +are optional. A monitor resource alone does not prove discovery or network access. + +## Complete setup + +Install the Prometheus Operator and its CRDs first. Apply +[`examples/monitoring/gameplane-values.yaml`](examples/monitoring/gameplane-values.yaml) +to the Gameplane release and deploy +[`examples/monitoring/prometheus.yaml`](examples/monitoring/prometheus.yaml), or +copy that resource's selectors into your existing Prometheus configuration. + +The examples agree on the `monitoring: gameplane` monitor/rule label, discover +both `gameplane-system` and `gameplane-games`, and place Prometheus in `monitoring`. +Change all three files/settings together when renaming namespaces. With +kube-prometheus-stack, configure `prometheus.prometheusSpec.serviceMonitorSelector`, +`podMonitorSelector`, their namespace selectors and `ruleSelector` to match the +example. Do not leave the chart's default release-label selector pointing at a +different Helm release. The Grafana sidecar must watch ConfigMaps labeled +`grafana_dashboard: "1"` in the Gameplane release namespace. + +The chart enables rules and both Grafana dashboards in this example. Agent +metrics use plain HTTP port 9090; tunnel metrics share the health listener on +8091. Neither monitor uses the agent's mTLS control port or a client certificate. +Check Prometheus **Status → Targets** for agent and tunnel targets with `up=1`, +then query `gameplane_agent_players_online` and `gameplane_tunnel_ready`. +Unknown player counts are absent even when scraping works. + +With NetworkPolicies enabled, the chart requires a nonempty +`serviceMonitors.scrapeNamespaceSelector`. Missing configuration fails with a +message naming the setting. If another system owns the policies, set +`serviceMonitors.externallyManagedPolicy: true`; that flag opens no traffic. +When upgrading an installation that already enabled monitors without a selector, +set the selector or the explicit external-policy option in the same Helm upgrade. +Supply your own ingress policies selecting `app.kubernetes.io/name: gameplane-game` +on TCP 9090 and `app.kubernetes.io/name: gameplane-tunnel` on TCP 8091, with a +namespace selector for the scraper. Restricted Prometheus egress policies must +also allow those destinations, DNS and API discovery. + +For enforced isolation, configure `networkPolicies.strict: true`, actual +`kubeletCIDRs` and `apiServerCIDRs`, and the game modules' exact `probePorts`. +The legacy broad kubelet allowance can otherwise admit other private-address +sources on metrics ports. Keep metrics ports out of the game probe allowance; +the dedicated scrape policies grant their access. The CI monitoring bucket +uses Cilium and actual node/API addresses, tests a scraper in a separate +namespace, revokes/restores the scrape policy, and checks an unapproved namespace +cannot reach agent metrics. + +## Declaring numeric game measurements + +Reuse a module's existing `capabilities.status.metrics[]` entry. This opt-in +example describes an output such as `TPS: 19.5`: + +```yaml +capabilities: + status: + metrics: + - id: tps + displayName: TPS + command: stats + regex: 'TPS: (?P[0-9.]+)' + unit: TPS # presentation only + performance: + signal: tick_rate + unit: per_second # machine-readable input unit + degradedThreshold: "18" +``` + +`signal` is one of `tick_rate`, `tick_duration`, or `frame_rate`. Rates require +`per_second`; durations accept `seconds` or `milliseconds`. Values and thresholds +are converted to hertz or seconds. The optional threshold is a finite nonnegative +decimal string, up to nine integer and six fractional digits, for portable CRD +handling. Rates below the threshold or durations above it trigger the degradation +rule after five minutes. No default threshold or game-specific tick rate is +invented. Modules must choose a read-only command and a meaningful threshold. +No external module is opted in by this change. + +The source polls immediately and then 30 seconds after each completed cycle. +There are at most 16 declared entries, 16 distinct commands per cycle, 1024 bytes +per command, 4096 bytes per regex and 64 KiB of accepted response text. Identical +commands are executed once and parsed for each reading. Each command waits at +most three seconds and the whole cycle at most ten seconds. The legacy RCON +interface cannot cancel every protocol operation: one outstanding call may +finish later, but it blocks further background calls until it returns, its late +result is discarded, and no queue or growing set of workers is created. +Underlying transports retain their own deadlines and response limits. Source +RCON bounds dialing to five seconds, authentication to ten seconds (including +all preamble packets), and a command exchange to 30 seconds after authentication, +including continuously arriving fragments, and accepts at most 8 MiB of +aggregate response text for large administrative commands. Its 400-ms fragment +idle window never extends the absolute exchange deadline. Exceeding either +bound discards the incomplete response and closes the connection. This transport +cap is separate from the status collector's smaller 64-KiB acceptance limit. + +Both protected `/status` routes and Prometheus share the same source. Templates +with valid performance declarations use the interval cache for all their status +readouts; templates without them retain the five-second on-demand cache. Scrapes +never execute RCON or wait on an RCON command. Samples expire 75 seconds after +the command completed. Failed queries, invalid/negative/nonfinite values and +stale samples have no numeric value series. A genuine zero is retained. + +The metric families are `gameplane_agent_performance_value`, +`gameplane_agent_performance_available`, +`gameplane_agent_performance_sample_timestamp_seconds`, and optional +`gameplane_agent_performance_degraded_threshold`. Labels are the bounded metric +ID, closed signal and base unit; threshold adds `direction=below|above`. +PodMonitor adds namespace, Pod, server and component identity. Display strings, +commands, responses and errors never become labels. Unsupported declarations +produce no performance series. Failed supported declarations emit availability +zero; value and timestamp remain absent. + +## Relays and infrastructure + +Tunnel gauges read the same snapshot as readiness: `gameplane_tunnel_ready`, +`gameplane_tunnel_connection_known`, `gameplane_tunnel_connected`, +`gameplane_tunnel_forwards_registered`, `gameplane_tunnel_endpoint_assigned`, +and `gameplane_tunnel_observation_timestamp_seconds`. `connected` and observation +timestamp are absent when their evidence is unknown or stale. Readiness and +registration flags mean *confirmed*, so zero does not invent a disconnected +provider. The rules distinguish an unavailable relay from unknown health. +Pinned Playit cannot confirm live connectivity and therefore raises the +informational unknown-health alert; see [relay-health.md](relay-health.md). +Tailscale, frp and Playit all use the same metrics path. Metrics contain no +credentials, login URLs, proxy names or provider error messages. + +Container CPU throttling and working-set pressure require kubelet/cAdvisor +scrapes. OOM/restart, Pending Pod and node pressure rules require kube-state-metrics. +Install/configure those collectors (kube-prometheus-stack normally provides +them); the minimal Prometheus example demonstrates Gameplane discovery and +does not deploy them or Grafana. A missing upstream collector leaves those +series absent; it does not mean zero pressure. Agent process RSS is not total +container memory. These rules filter the games namespace and use `max` to +deduplicate replicated sources. The existing fleet rules likewise collapse +duplicate operator replicas instead of adding their cache-derived counts. + +The Workloads dashboard shows scrape health, relay readiness/known state, +supported game measurements, container pressure, restarts and Pending Pods. +Missing measurements remain gaps. CI renders and evaluates the shipped alert +rules with promtool, including outage/recovery, supported zero versus unknown, +pressure, throttling, OOM, Pending/node pressure and duplicate operator readings. diff --git a/docs/tunnels.md b/docs/tunnels.md index 504d98d96..1ba4c9a57 100644 --- a/docs/tunnels.md +++ b/docs/tunnels.md @@ -1,5 +1,8 @@ # Tunnels +For Prometheus discovery, relay health alerts and the complete scrape-policy +configuration, see [Monitoring games and relays](monitoring.md). + A GameServer can optionally route players through an external relay — the server dials outbound to the relay and players connect to the relay's address. This exists for installs with no public IP and no ability to port-forward (homelab k3s behind diff --git a/operator/api/v1alpha1/gametemplate_types.go b/operator/api/v1alpha1/gametemplate_types.go index 38a906125..092b48acf 100644 --- a/operator/api/v1alpha1/gametemplate_types.go +++ b/operator/api/v1alpha1/gametemplate_types.go @@ -741,6 +741,28 @@ type StatusMetricSpec struct { // Unit is an optional suffix shown after the value (e.g. "ms", "TPS"). // +optional Unit string `json:"unit,omitempty"` + + // Performance opts this existing readout into cached numeric monitoring. + // Display-only strings are never inferred to be performance measurements. + // +optional + Performance *PerformanceMetricSpec `json:"performance,omitempty"` +} + +// PerformanceMetricSpec gives a status readout explicit numeric semantics. +// Rate signals use per_second; duration accepts seconds or milliseconds. +// +kubebuilder:validation:XValidation:rule="self.signal == 'tick_duration' ? self.unit in ['seconds', 'milliseconds'] : self.unit == 'per_second'",message="duration requires seconds/milliseconds; rates require per_second" +type PerformanceMetricSpec struct { + // +kubebuilder:validation:Enum=tick_rate;tick_duration;frame_rate + Signal string `json:"signal"` + // +kubebuilder:validation:Enum=per_second;seconds;milliseconds + Unit string `json:"unit"` + // DegradedThreshold is a finite nonnegative decimal string in the input + // unit. Rates below it, or durations above it, are degraded. Omit to + // disable alerts. A string keeps JSON/schema handling portable. + // +kubebuilder:validation:Pattern=`^(0|[1-9][0-9]{0,8})(\.[0-9]{1,6})?$` + // +kubebuilder:validation:MaxLength=16 + // +optional + DegradedThreshold *string `json:"degradedThreshold,omitempty"` } // PlayerActionsSpec maps moderation actions to console commands. Each diff --git a/operator/api/v1alpha1/zz_generated.deepcopy.go b/operator/api/v1alpha1/zz_generated.deepcopy.go index 05926fd58..20421d118 100644 --- a/operator/api/v1alpha1/zz_generated.deepcopy.go +++ b/operator/api/v1alpha1/zz_generated.deepcopy.go @@ -2215,6 +2215,26 @@ func (in *OCISourceSpec) DeepCopy() *OCISourceSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PerformanceMetricSpec) DeepCopyInto(out *PerformanceMetricSpec) { + *out = *in + if in.DegradedThreshold != nil { + in, out := &in.DegradedThreshold, &out.DegradedThreshold + *out = new(string) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PerformanceMetricSpec. +func (in *PerformanceMetricSpec) DeepCopy() *PerformanceMetricSpec { + if in == nil { + return nil + } + out := new(PerformanceMetricSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *PlayerActionsSpec) DeepCopyInto(out *PlayerActionsSpec) { *out = *in @@ -2521,6 +2541,11 @@ func (in *ServerActionSpec) DeepCopy() *ServerActionSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *StatusMetricSpec) DeepCopyInto(out *StatusMetricSpec) { *out = *in + if in.Performance != nil { + in, out := &in.Performance, &out.Performance + *out = new(PerformanceMetricSpec) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StatusMetricSpec. @@ -2539,7 +2564,9 @@ func (in *StatusSpec) DeepCopyInto(out *StatusSpec) { if in.Metrics != nil { in, out := &in.Metrics, &out.Metrics *out = make([]StatusMetricSpec, len(*in)) - copy(*out, *in) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } } } diff --git a/operator/config/crd/gameplane.local_gametemplates.yaml b/operator/config/crd/gameplane.local_gametemplates.yaml index 0b9cf37a0..f749bccaa 100644 --- a/operator/config/crd/gameplane.local_gametemplates.yaml +++ b/operator/config/crd/gameplane.local_gametemplates.yaml @@ -960,6 +960,41 @@ spec: maxLength: 63 pattern: ^[a-z0-9]([a-z0-9-]*[a-z0-9])?$ type: string + performance: + description: |- + Performance opts this existing readout into cached numeric monitoring. + Display-only strings are never inferred to be performance measurements. + properties: + degradedThreshold: + description: |- + DegradedThreshold is a finite nonnegative decimal string in the input + unit. Rates below it, or durations above it, are degraded. Omit to + disable alerts. A string keeps JSON/schema handling portable. + maxLength: 16 + pattern: ^(0|[1-9][0-9]{0,8})(\.[0-9]{1,6})?$ + type: string + signal: + enum: + - tick_rate + - tick_duration + - frame_rate + type: string + unit: + enum: + - per_second + - seconds + - milliseconds + type: string + required: + - signal + - unit + type: object + x-kubernetes-validations: + - message: duration requires seconds/milliseconds; rates + require per_second + rule: 'self.signal == ''tick_duration'' ? self.unit + in [''seconds'', ''milliseconds''] : self.unit == + ''per_second''' regex: description: |- Regex extracts the value via the named group "value", e.g. diff --git a/operator/internal/controller/gameserver_relay_health_test.go b/operator/internal/controller/gameserver_relay_health_test.go index 2f3597791..1ae0379f6 100644 --- a/operator/internal/controller/gameserver_relay_health_test.go +++ b/operator/internal/controller/gameserver_relay_health_test.go @@ -36,6 +36,15 @@ func TestRelayDeploymentIdentityProbesAndResourceRemoval(t *testing.T) { t.Fatal("relay permits overlapping identities") } container := dep.Spec.Template.Spec.Containers[0] + metricsPorts := 0 + for _, port := range container.Ports { + if port.Name == "relay-metrics" && port.ContainerPort == 8091 { + metricsPorts++ + } + } + if metricsPorts != 1 { + t.Fatal("tunnel metrics port missing or duplicated") + } if container.StartupProbe == nil || container.StartupProbe.HTTPGet == nil || container.StartupProbe.HTTPGet.Path != "/healthz" || !hasRelayReadinessProbe(&dep) || container.LivenessProbe != nil { t.Fatal("provider outage must affect readiness without restarting the supervisor") } diff --git a/operator/internal/controller/gameserver_tunnel.go b/operator/internal/controller/gameserver_tunnel.go index 03785c236..8a6c10aed 100644 --- a/operator/internal/controller/gameserver_tunnel.go +++ b/operator/internal/controller/gameserver_tunnel.go @@ -235,6 +235,12 @@ func (r *GameServerReconciler) reconcileTunnel( ContainerPort: p.ContainerPort, Protocol: p.Protocol, } + // The generated health port owns this discovery name. Forwarded + // port metadata is optional; the provider still uses the unchanged + // template/API name in its actual mapping. + if cp.Name == "relay-metrics" { + cp.Name = "" + } ports = append(ports, cp) } @@ -408,6 +414,7 @@ func (r *GameServerReconciler) reconcileTunnel( readiness = &corev1.Probe{ProbeHandler: corev1.ProbeHandler{HTTPGet: &corev1.HTTPGetAction{Path: "/readyz", Port: intstr.FromInt(8091)}}, PeriodSeconds: 5, TimeoutSeconds: 2, FailureThreshold: 1} } dep.Spec.Template.Spec.Volumes = volumes + ports = append(ports, corev1.ContainerPort{Name: "relay-metrics", ContainerPort: 8091, Protocol: corev1.ProtocolTCP}) dep.Spec.Template.Spec.Containers = []corev1.Container{{ Name: "tunnel", Image: image, diff --git a/operator/internal/controller/gametemplate_performance_envtest_test.go b/operator/internal/controller/gametemplate_performance_envtest_test.go new file mode 100644 index 000000000..ee0a0c96a --- /dev/null +++ b/operator/internal/controller/gametemplate_performance_envtest_test.go @@ -0,0 +1,46 @@ +//go:build envtest + +package controller + +import ( + "testing" + + gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" + apierrors "k8s.io/apimachinery/pkg/api/errors" +) + +func TestTemplatePerformanceSchema(t *testing.T) { + for _, tc := range []struct { + signal, unit, threshold string + valid bool + }{ + {"tick_rate", "per_second", "18", true}, + {"tick_duration", "milliseconds", "50.5", true}, + {"frame_rate", "per_second", "0", true}, + {"seed", "per_second", "1", false}, + {"tick_rate", "seconds", "1", false}, + {"tick_duration", "per_second", "1", false}, + {"tick_rate", "per_second", "NaN", false}, + {"tick_rate", "per_second", "-1", false}, + {"tick_rate", "per_second", "1e999", false}, + } { + t.Run(tc.signal+"-"+tc.unit+"-"+tc.threshold, func(t *testing.T) { + tmpl := buildGameTemplate(uniqueName("performance")) + tmpl.Spec.RCON = &gameplanev1alpha1.RCONSpec{Protocol: "source", Port: 25575} + if tmpl.Spec.Capabilities == nil { + tmpl.Spec.Capabilities = &gameplanev1alpha1.CapabilitiesSpec{} + } + threshold := tc.threshold + tmpl.Spec.Capabilities.Status = &gameplanev1alpha1.StatusSpec{Metrics: []gameplanev1alpha1.StatusMetricSpec{{ID: "performance", DisplayName: "Performance", Command: "stats", Regex: `(?P\S+)`, Performance: &gameplanev1alpha1.PerformanceMetricSpec{Signal: tc.signal, Unit: tc.unit, DegradedThreshold: &threshold}}}} + err := k8sClient.Create(t.Context(), tmpl) + if tc.valid { + if err != nil { + t.Fatal(err) + } + deleteCleanup(t, tmpl) + } else if !apierrors.IsInvalid(err) { + t.Fatalf("wanted schema refusal, got %v", err) + } + }) + } +} diff --git a/operator/specs.md b/operator/specs.md index 96763e7b2..dede0e559 100644 --- a/operator/specs.md +++ b/operator/specs.md @@ -10,6 +10,8 @@ The operator is a Kubernetes controller-runtime-based process that reconciles Ga ## Responsibilities +- Preserve explicit optional numeric performance declarations in template capabilities passed to the agent; CRD validation restricts signal/unit pairs and finite decimal thresholds. Tunnel containers name their existing 8091 listener `relay-metrics` for PodMonitor discovery; health, resources and provider identity policies remain unchanged. + - **Relay health:** frp identities use GameServer UID without changing advertised port names. frp/Playit Deployments use one replica, Recreate, a wrapper startup probe and a provider readiness probe on TCP 8091. `TunnelReady` requires the current observed generation and updated/ready replicas with the new probe; legacy unprobed Pods cannot report success. Playit v1.0.10's cached IPC assignment cannot confirm a relay connection, so it reports unknown/not ready rather than optimistic success. Preserve `TunnelResources` defaults and override removal. See `docs/relay-health.md` for health bounds, migration and the provider limitation. - **GameServer lifecycle:** reconcile GameServer CRs into StatefulSets, Services, PVCs, and pod-injected agent sidecars; manage pod affinity, RCON/console integration, file I/O, and graceful stop sequences. diff --git a/test/e2e/buckets.sh b/test/e2e/buckets.sh index 0bdd4071c..1ad462f1e 100755 --- a/test/e2e/buckets.sh +++ b/test/e2e/buckets.sh @@ -399,10 +399,15 @@ EOF unbucketed() { :; } +bucket_monitoring() { cat <<'EOF' +TestMonitoring_DiscoveryPerformanceAndPolicy +EOF +} + bucket_network_isolation() { echo TestNetworkIsolation_EnforcedAndUpgrade; } bucket_names() { - printf '%s\n' operator api-auth api-roles api-rbac api-agent api-mods ratelimit bot-fast bot-heavy multicluster upgrade telemetry capture-overhead optional-components management-placement network-isolation + printf '%s\n' operator api-auth api-roles api-rbac api-agent api-mods ratelimit bot-fast bot-heavy multicluster upgrade telemetry capture-overhead optional-components management-placement network-isolation monitoring } list_bucket() { @@ -423,6 +428,7 @@ list_bucket() { optional-components) bucket_optional_components ;; management-placement) bucket_management_placement ;; network-isolation) bucket_network_isolation ;; + monitoring) bucket_monitoring ;; *) echo "unknown bucket: $1 (known: $(bucket_names | tr '\n' ' '))" >&2 exit 2 diff --git a/test/e2e/monitoring_e2e_test.go b/test/e2e/monitoring_e2e_test.go new file mode 100644 index 000000000..00bdb02ae --- /dev/null +++ b/test/e2e/monitoring_e2e_test.go @@ -0,0 +1,221 @@ +//go:build e2e + +package e2e + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "strconv" + "strings" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/types" +) + +// Dedicated serial bucket: real Prometheus Operator discovery and Cilium policy +// enforcement, with deterministic game readouts instead of an external module. +func TestMonitoring_DiscoveryPerformanceAndPolicy(t *testing.T) { + ctx := t.Context() + if _, err := envInstance.K8s.AppsV1().DaemonSets("kube-system").Get(ctx, "cilium", metav1.GetOptions{}); err != nil { + t.Fatalf("monitoring requires enforcing Cilium: %v", err) + } + port, stop := envInstance.PortForward(t, "monitoring", "svc/prometheus-operated", 9090) + defer stop() + client := &http.Client{Timeout: 10 * time.Second} + query := func(expression string) (string, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, fmt.Sprintf("http://127.0.0.1:%d/api/v1/query?query=%s", port, url.QueryEscape(expression)), nil) + if err != nil { + return "", err + } + response, err := client.Do(req) + if err != nil { + return "", err + } + defer response.Body.Close() + var result struct { + Status string `json:"status"` + Data struct { + Result []struct { + Value []json.RawMessage `json:"value"` + } `json:"result"` + } `json:"data"` + } + if err := json.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(&result); err != nil { + return "", err + } + if response.StatusCode != 200 || result.Status != "success" || len(result.Data.Result) != 1 || len(result.Data.Result[0].Value) != 2 { + return "", fmt.Errorf("query %q returned status %s and %d results", expression, result.Status, len(result.Data.Result)) + } + var value string + err = json.Unmarshal(result.Data.Result[0].Value[1], &value) + return value, err + } + waitQuery := func(expression, want string) { + envInstance.Eventually(t, 3*time.Minute, func() (bool, string) { + got, err := query(expression) + return err == nil && got == want, fmt.Sprintf("%s: got %s want %s (%v)", expression, got, want, err) + }) + } + envInstance.ApplyYAML(t, "frps.yaml") + stamp := time.Now().UnixNano() + name := fmt.Sprintf("monitoring-%d", stamp) + const gameCode = `from http.server import BaseHTTPRequestHandler,ThreadingHTTPServer +import json,pathlib,threading +value=pathlib.Path('/tmp/performance') +count=pathlib.Path('/tmp/performance-queries') +value.write_text('20') +count.write_text('0') +class Handler(BaseHTTPRequestHandler): + def log_message(self,*args): pass + def do_GET(self): + self.send_response(200); self.end_headers(); self.wfile.write(b'ok') + def do_POST(self): + command=json.loads(self.rfile.read(int(self.headers.get('Content-Length','0')))).get('command','') + if command=='stats': + count.write_text(str(int(count.read_text())+1)) + output='TPS: '+value.read_text()+' Tick: 50' + else: output='0 players online' + self.send_response(200); self.end_headers(); self.wfile.write(json.dumps({'output':output}).encode()) +threading.Thread(target=ThreadingHTTPServer(('127.0.0.1',25575),Handler).serve_forever,daemon=True).start() +ThreadingHTTPServer(('0.0.0.0',25566),Handler).serve_forever() +` + tmpl := &unstructured.Unstructured{Object: map[string]any{"apiVersion": "gameplane.local/v1alpha1", "kind": "GameTemplate", "metadata": map[string]any{"name": name}, "spec": map[string]any{ + "displayName": "Monitoring fixture", "game": "noop", "version": "1", "image": "python:3.14-alpine", "consoleMode": "rcon", "command": []any{"python", "-u", "-c"}, "args": []any{gameCode}, "sentinelIsolation": true, + "ports": []any{map[string]any{"name": "game", "containerPort": int64(25566), "protocol": "TCP", "advertise": true}}, "rcon": map[string]any{"protocol": "rest", "port": int64(25575)}, + "capabilities": map[string]any{"status": map[string]any{"metrics": []any{ + map[string]any{"id": "tps", "displayName": "TPS", "command": "stats", "regex": `TPS: (?P\S+)`, "performance": map[string]any{"signal": "tick_rate", "unit": "per_second", "degradedThreshold": "18"}}, + map[string]any{"id": "tick", "displayName": "Tick", "command": "stats", "regex": `Tick: (?P\S+)`, "performance": map[string]any{"signal": "tick_duration", "unit": "milliseconds"}}, + map[string]any{"id": "display", "displayName": "Display only", "command": "stats", "regex": `TPS: (?P\S+)`}, + }}}, + }}} + if _, err := envInstance.Dyn.Resource(gameTemplateGVR).Create(ctx, tmpl, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + _ = envInstance.Dyn.Resource(gameTemplateGVR).Delete(context.Background(), name, metav1.DeleteOptions{}) + }) + gs := &unstructured.Unstructured{Object: map[string]any{"apiVersion": "gameplane.local/v1alpha1", "kind": "GameServer", "metadata": map[string]any{"name": name, "namespace": ocGamesNS}, "spec": map[string]any{"templateRef": map[string]any{"name": name}, "networking": map[string]any{"expose": "ClusterIP", "tunnel": map[string]any{"enabled": false, "provider": "frp", "frp": map[string]any{"serverAddr": ocFrpsAddr, "remotePorts": []any{map[string]any{"name": "game", "remotePort": int64(30020)}}}}}}}} + gs, err := envInstance.Dyn.Resource(gameServerGVR).Namespace(ocGamesNS).Create(ctx, gs, metav1.CreateOptions{}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + _ = envInstance.Dyn.Resource(gameServerGVR).Namespace(ocGamesNS).Delete(context.Background(), name, metav1.DeleteOptions{}) + }) + secret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: name + "-auth", Namespace: ocGamesNS, OwnerReferences: []metav1.OwnerReference{{APIVersion: "gameplane.local/v1alpha1", Kind: "GameServer", Name: name, UID: gs.GetUID()}}}, StringData: map[string]string{"token": ocFrpsToken}} + if _, err := envInstance.K8s.CoreV1().Secrets(ocGamesNS).Create(ctx, secret, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + patch, _ := json.Marshal(map[string]any{"spec": map[string]any{"networking": map[string]any{"tunnel": map[string]any{"enabled": true, "credentialsSecretRef": map[string]any{"name": secret.Name}}}}}) + if _, err := envInstance.Dyn.Resource(gameServerGVR).Namespace(ocGamesNS).Patch(ctx, name, types.MergePatchType, patch, metav1.PatchOptions{}); err != nil { + t.Fatal(err) + } + selector := fmt.Sprintf(`namespace="gameplane-games",server="%s"`, name) + waitQuery(`max(up{gameplane_component="agent",`+selector+`})`, "1") + waitQuery(`max(up{gameplane_component="tunnel",`+selector+`})`, "1") + waitQuery(`max(gameplane_tunnel_ready{`+selector+`})`, "1") + waitQuery(`max(gameplane_agent_performance_value{metric="tps",`+selector+`})`, "20") + waitQuery(`max(gameplane_agent_performance_value{metric="tick",`+selector+`})`, "0.05") + waitQuery(`count(gameplane_agent_performance_value{metric="display",`+selector+`}) or vector(0)`, "0") + readCount := func() int { + out, err := envInstance.Kubectl(ctx, "exec", "-n", ocGamesNS, name+"-0", "-c", "game", "--", "python", "-c", `from pathlib import Path; print(Path('/tmp/performance-queries').read_text())`) + if err != nil { + t.Fatal(err) + } + n, err := strconv.Atoi(strings.TrimSpace(out)) + if err != nil { + t.Fatal(err) + } + return n + } + pod, err := envInstance.K8s.CoreV1().Pods(ocGamesNS).Get(ctx, name+"-0", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + approvedProbe := name + "-approved" + monitoringProbe(t, "monitoring", approvedProbe) + before := readCount() + scrapeStart := time.Now() + const scrapeMany = `import urllib.request,sys +for _ in range(20): + with urllib.request.urlopen(sys.argv[1],timeout=2) as response: + assert b'gameplane_agent_performance_value' in response.read() +` + if out, err := envInstance.Kubectl(ctx, "exec", "-n", "monitoring", approvedProbe, "--", "python", "-c", scrapeMany, "http://"+pod.Status.PodIP+":9090/metrics"); err != nil { + t.Fatalf("repeated direct scrape failed: %v %s", err, out) + } + if after := readCount(); after-before > int(time.Since(scrapeStart)/(30*time.Second))+1 { + t.Fatalf("scrapes triggered RCON: %d -> %d", before, after) + } + for _, value := range []string{"0", "NaN", "20"} { + if out, err := envInstance.Kubectl(ctx, "exec", "-n", ocGamesNS, name+"-0", "-c", "game", "--", "python", "-c", `from pathlib import Path; import sys; Path('/tmp/performance').write_text(sys.argv[1])`, value); err != nil { + t.Fatalf("set measurement: %v %s", err, out) + } + if value == "NaN" { + waitQuery(`max(gameplane_agent_performance_available{metric="tps",`+selector+`})`, "0") + waitQuery(`count(gameplane_agent_performance_value{metric="tps",`+selector+`}) or vector(0)`, "0") + } else { + waitQuery(`max(gameplane_agent_performance_value{metric="tps",`+selector+`})`, value) + } + } + // A successful Prometheus query is not proof of policy enforcement. Revoke + // exactly the chart's scrape exception, observe scrape loss, then restore. + policy, err := envInstance.K8s.NetworkingV1().NetworkPolicies(ocGamesNS).Get(ctx, "allow-prometheus-to-agent", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if err := envInstance.K8s.NetworkingV1().NetworkPolicies(ocGamesNS).Delete(ctx, policy.Name, metav1.DeleteOptions{}); err != nil { + t.Fatal(err) + } + policy.ResourceVersion = "" + policy.UID = "" + t.Cleanup(func() { + _, _ = envInstance.K8s.NetworkingV1().NetworkPolicies(ocGamesNS).Create(context.Background(), policy, metav1.CreateOptions{}) + }) + waitQuery(`max(up{gameplane_component="agent",`+selector+`})`, "0") + if _, err := envInstance.K8s.NetworkingV1().NetworkPolicies(ocGamesNS).Create(ctx, policy, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + waitQuery(`max(up{gameplane_component="agent",`+selector+`})`, "1") + // The system namespace is deliberately outside the monitoring selector. + probeName := name + "-probe" + monitoringProbe(t, ocNS, probeName) + const denied = `import urllib.request,sys +try: + urllib.request.urlopen(sys.argv[1],timeout=2).read() + print('ALLOWED') +except (OSError,TimeoutError): + print('DENIED') +` + out, err := envInstance.Kubectl(ctx, "exec", "-n", ocNS, probeName, "--", "python", "-c", denied, "http://"+pod.Status.PodIP+":9090/metrics") + if err != nil || strings.TrimSpace(out) != "DENIED" { + t.Fatalf("unapproved namespace reached metrics, or probe failed: %v %s", err, out) + } +} + +func monitoringProbe(t *testing.T, namespace, name string) { + t.Helper() + probe := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace}, Spec: corev1.PodSpec{RestartPolicy: corev1.RestartPolicyNever, Containers: []corev1.Container{{Name: "probe", Image: "python:3.14-alpine", Command: []string{"python", "-c", "import time; time.sleep(900)"}}}}} + nonRoot, no, uid := true, false, int64(65532) + probe.Spec.AutomountServiceAccountToken = &no + probe.Spec.SecurityContext = &corev1.PodSecurityContext{RunAsNonRoot: &nonRoot, RunAsUser: &uid, SeccompProfile: &corev1.SeccompProfile{Type: corev1.SeccompProfileTypeRuntimeDefault}} + probe.Spec.Containers[0].SecurityContext = &corev1.SecurityContext{AllowPrivilegeEscalation: &no, Capabilities: &corev1.Capabilities{Drop: []corev1.Capability{"ALL"}}} + if _, err := envInstance.K8s.CoreV1().Pods(namespace).Create(t.Context(), probe, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + _ = envInstance.K8s.CoreV1().Pods(namespace).Delete(context.Background(), name, metav1.DeleteOptions{}) + }) + envInstance.Eventually(t, time.Minute, func() (bool, string) { + ready, err := envInstance.PodIsReady(t.Context(), namespace, name) + return ready, fmt.Sprint(err) + }) +} diff --git a/test/e2e/specs.md b/test/e2e/specs.md index a62289eab..5da77d74d 100644 --- a/test/e2e/specs.md +++ b/test/e2e/specs.md @@ -19,6 +19,8 @@ It was one of three `go.work` modules without CI lint coverage until this featur ## Responsibilities +- The serial `monitoring` bucket installs a checksum-pinned Prometheus Operator and Prometheus in a distinct namespace on the Cilium profile. It discovers real agent/tunnel targets through the documented chart values, verifies cached numeric samples, valid zero, invalid-value omission, no scrape-driven commands, policy revocation/recovery and refusal from an unapproved namespace. It performs no API login and does not opt in external modules. + - The optional-components lifecycle's `frp-identity-health` subtest runs two same-template TCP/UDP echo GameServers through one real frps. It asserts backend identity, registration rejection, replacement isolation, invalid credentials, relay outage/recovery and secret-free logs. It shares the lifecycle's controlled fixture and serial bucket; no extra top-level test or login budget is introduced. - Boot and tear down (or reuse) a kind cluster, install the working-tree Helm chart, and wait for every component to come up healthy. diff --git a/tunnel/relay_health.go b/tunnel/relay_health.go index f095810e9..290d51b58 100644 --- a/tunnel/relay_health.go +++ b/tunnel/relay_health.go @@ -34,7 +34,8 @@ func (o relayObservation) ready() bool { type relaySnapshot struct { relayObservation - Ready bool `json:"ready"` + Ready bool `json:"ready"` + checked time.Time } type relayHealth struct { @@ -76,11 +77,12 @@ func (h *relayHealth) snapshot(now time.Time) relaySnapshot { if h.checked.IsZero() || now.Sub(h.checked) >= healthStaleAfter { return relaySnapshot{relayObservation: relayObservation{Connection: "unknown"}} } - return relaySnapshot{relayObservation: h.observation, Ready: h.observation.ready()} + return relaySnapshot{relayObservation: h.observation, Ready: h.observation.ready(), checked: h.checked} } func relayHealthHandler(h *relayHealth) http.Handler { mux := http.NewServeMux() + mux.Handle("/metrics", relayMetrics(h)) mux.HandleFunc("/healthz", func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) }) mux.HandleFunc("/readyz", func(w http.ResponseWriter, _ *http.Request) { if !h.snapshot(time.Now()).Ready { diff --git a/tunnel/relay_metrics.go b/tunnel/relay_metrics.go new file mode 100644 index 000000000..f671a3ba2 --- /dev/null +++ b/tunnel/relay_metrics.go @@ -0,0 +1,44 @@ +package main + +import ( + "fmt" + "net/http" + "time" +) + +// Fixed metric names and a closed state enum make the small exposition bounded. +// Scraping never polls a provider. Unknown connection is not encoded as down. +func relayMetrics(h *relayHealth) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "text/plain; version=0.0.4; charset=utf-8") + now := time.Now() + snapshot := h.snapshot(now) + checked := snapshot.checked + known := snapshot.Connection == "connected" || snapshot.Connection == "unavailable" + gauges := []struct { + name, help string + value bool + }{ + {"gameplane_tunnel_ready", "Whether the relay and all configured forwards are confirmed ready.", snapshot.Ready}, + {"gameplane_tunnel_connection_known", "Whether the provider exposes a fresh confirmed connection state.", known}, + {"gameplane_tunnel_forwards_registered", "Whether every configured forward is confirmed registered.", snapshot.Forwards}, + {"gameplane_tunnel_endpoint_assigned", "Whether the configured endpoint is assigned.", snapshot.Assigned}, + } + if known { + gauges = append(gauges, struct { + name, help string + value bool + }{"gameplane_tunnel_connected", "Whether the relay is connected; absent when unknown.", snapshot.Connection == "connected"}) + } + for _, g := range gauges { + n := 0 + if g.value { + n = 1 + } + _, _ = fmt.Fprintf(w, "# HELP %s %s\n# TYPE %s gauge\n%s %d\n", g.name, g.help, g.name, g.name, n) + } + if !checked.IsZero() && now.Sub(checked) < healthStaleAfter { + _, _ = fmt.Fprintf(w, "# HELP gameplane_tunnel_observation_timestamp_seconds Unix time of the latest fresh provider observation.\n# TYPE gameplane_tunnel_observation_timestamp_seconds gauge\ngameplane_tunnel_observation_timestamp_seconds %.3f\n", float64(checked.UnixNano())/1e9) + } + }) +} diff --git a/tunnel/relay_metrics_test.go b/tunnel/relay_metrics_test.go new file mode 100644 index 000000000..52001d16f --- /dev/null +++ b/tunnel/relay_metrics_test.go @@ -0,0 +1,37 @@ +package main + +import ( + "net/http/httptest" + "strings" + "testing" + "time" +) + +func TestRelayMetricsUnknownStaleAndRecovery(t *testing.T) { + h := &relayHealth{} + for _, tc := range []struct { + state relayObservation + age time.Duration + connected, ready string + }{ + {relayObservation{Connection: "unknown", Assigned: true}, 0, "", "0"}, + {relayObservation{Connection: "connected", Assigned: true, Forwards: true}, 0, "1", "1"}, + {relayObservation{Connection: "connected", Assigned: true, Forwards: true}, healthStaleAfter + time.Second, "", "0"}, + {relayObservation{Connection: "unavailable"}, 0, "0", "0"}, + } { + h.observe(time.Now().Add(-tc.age), tc.state, nil) + w := httptest.NewRecorder() + relayHealthHandler(h).ServeHTTP(w, httptest.NewRequest("GET", "/metrics", nil)) + body := w.Body.String() + if !strings.Contains(body, "gameplane_tunnel_ready "+tc.ready+"\n") { + t.Fatal(body) + } + if tc.connected == "" { + if strings.Contains(body, "gameplane_tunnel_connected ") { + t.Fatal("unknown fabricated as connected/down") + } + } else if !strings.Contains(body, "gameplane_tunnel_connected "+tc.connected+"\n") { + t.Fatal(body) + } + } +} diff --git a/tunnel/specs.md b/tunnel/specs.md index 1ad4ebef5..9344719f3 100644 --- a/tunnel/specs.md +++ b/tunnel/specs.md @@ -10,6 +10,8 @@ Relay client supervisor that configures and supervises frp/playit child processe ## Responsibilities +- Expose cached provider evidence as bounded Prometheus gauges on `/metrics` at the existing health port 8091, including known/unknown connection, confirmed forwards/assignment and observation timestamp. Scraping never queries the provider; unknown or stale connection evidence has no `connected` value. Tailscale uses the same snapshot for readiness and metrics. See `docs/monitoring.md`. + 1. Parse environment variables to determine the tunnel provider (frp, tailscale, or playit), backing service address, and provider-specific config. 2. Read provider-specific credentials from the mounted Secret at `/etc/gameplane/tunnel-auth` (read-only). 3. Render frp TOML or playit credentials at fixed paths, with secrets excluded from argv. diff --git a/tunnel/tailscale.go b/tunnel/tailscale.go index 4d4e12e74..3cbb609b9 100644 --- a/tunnel/tailscale.go +++ b/tunnel/tailscale.go @@ -80,11 +80,12 @@ func runTailscale(ctx context.Context, cfg Config, authKey string) error { return err } var ready atomic.Bool + healthHistory := &relayHealth{} listener, err := (&net.ListenConfig{}).Listen(ctx, "tcp", ":8091") if err != nil { return fmt.Errorf("Tailscale health listener: %w", err) } - health := &http.Server{Handler: tailscaleHealth(&ready), ReadHeaderTimeout: 2 * time.Second, ReadTimeout: 5 * time.Second, WriteTimeout: 5 * time.Second, IdleTimeout: 10 * time.Second} + health := &http.Server{Handler: relayHealthHandler(healthHistory), ReadHeaderTimeout: 2 * time.Second, ReadTimeout: 5 * time.Second, WriteTimeout: 5 * time.Second, IdleTimeout: 10 * time.Second} healthDone := make(chan error, 1) go func() { healthDone <- health.Serve(listener) }() defer func() { _ = health.Close() }() @@ -94,7 +95,7 @@ func runTailscale(ctx context.Context, cfg Config, authKey string) error { if err != nil { return err } - healthHistory := &relayHealth{} + healthHistory.clear() err = serveTailnetObserved(ctx, embeddedTailnet{server}, tags, ports, &ready, 2*time.Second, func(healthy bool) { observation := relayObservation{Connection: "unknown"} if healthy { @@ -103,6 +104,7 @@ func runTailscale(ctx context.Context, cfg Config, authKey string) error { healthHistory.observe(time.Now(), observation, backoff) }) ready.Store(false) + healthHistory.clear() _ = server.Close() if ctx.Err() != nil { return nil diff --git a/web/src/types.ts b/web/src/types.ts index 880c2b72c..0c81ab229 100644 --- a/web/src/types.ts +++ b/web/src/types.ts @@ -47,6 +47,11 @@ export interface StatusMetricDecl { command?: string; regex?: string; unit?: string; + performance?: { + signal: "tick_rate" | "tick_duration" | "frame_rate"; + unit: "per_second" | "seconds" | "milliseconds"; + degradedThreshold?: string; + }; } // Install policy for the mods capability (spec.capabilities.mods.install). From 0c5fdf57d0d6a9ae6c62eee6690f66da030dec09 Mon Sep 17 00:00:00 2001 From: Baylem Date: Wed, 21 Oct 2026 17:15:41 -0400 Subject: [PATCH 02/18] fix(agent): bound NuclearOption request writes Signed-off-by: Baylem --- agent/internal/rcon/nuclearoption.go | 28 ++++++----- .../rcon/nuclearoption_deadline_test.go | 46 +++++++++++++++++++ agent/internal/rcon/nuclearoption_test.go | 2 +- agent/specs.md | 2 +- 4 files changed, 64 insertions(+), 14 deletions(-) create mode 100644 agent/internal/rcon/nuclearoption_deadline_test.go diff --git a/agent/internal/rcon/nuclearoption.go b/agent/internal/rcon/nuclearoption.go index 7b8f2b1f2..001135354 100644 --- a/agent/internal/rcon/nuclearoption.go +++ b/agent/internal/rcon/nuclearoption.go @@ -112,14 +112,21 @@ func (c *NuclearOption) Close() error { // If the response has no body (body length = 0), an empty string is returned. // Non-2000 status codes produce an error that includes the status code and, // if present, the error detail from the response body. +// One deadline bounds the complete exchange, including dialing and request writes. func (c *NuclearOption) Exec(cmd string) (string, error) { c.mu.Lock() defer c.mu.Unlock() + d := c.execDeadline + if d <= 0 { + d = defaultNuclearOptionExecDeadline + } + exchangeDeadline := time.Now().Add(d) + // Ensure connection is open. if c.conn == nil { d := net.Dialer{Timeout: c.dialTimeout} - ctx, cancel := context.WithTimeout(context.Background(), c.dialTimeout) + ctx, cancel := context.WithDeadline(context.Background(), exchangeDeadline) defer cancel() conn, err := d.DialContext(ctx, "tcp", net.JoinHostPort(c.host, fmt.Sprint(c.port))) if err != nil { @@ -144,6 +151,14 @@ func (c *NuclearOption) Exec(cmd string) (string, error) { return "", fmt.Errorf("nuclearoption rcon: marshal request: %w", err) } + // Bound writes as well as reads. Arm this before sending anything: a + // server that stops reading must not retain the shared client forever. + if err := c.conn.SetDeadline(exchangeDeadline); err != nil { + _ = c.conn.Close() + c.conn = nil + return "", fmt.Errorf("nuclearoption rcon: set deadline: %w", err) + } + // Write request: 4-byte LE length + body. reqLen := len(reqBody) if reqLen < 0 || reqLen > math.MaxUint32 { @@ -162,17 +177,6 @@ func (c *NuclearOption) Exec(cmd string) (string, error) { return "", fmt.Errorf("nuclearoption rcon: write body: %w", err) } - // Set deadline for the entire response read. - d := c.execDeadline - if d <= 0 { - d = defaultNuclearOptionExecDeadline - } - if err := c.conn.SetDeadline(time.Now().Add(d)); err != nil { - _ = c.conn.Close() - c.conn = nil - return "", fmt.Errorf("nuclearoption rcon: set deadline: %w", err) - } - // Read response: 4-byte status + 4-byte body length + body. statusBuf := make([]byte, 4) if _, err := io.ReadFull(c.conn, statusBuf); err != nil { diff --git a/agent/internal/rcon/nuclearoption_deadline_test.go b/agent/internal/rcon/nuclearoption_deadline_test.go new file mode 100644 index 000000000..9cae0e073 --- /dev/null +++ b/agent/internal/rcon/nuclearoption_deadline_test.go @@ -0,0 +1,46 @@ +package rcon + +import ( + "io" + "net" + "strings" + "testing" + "time" +) + +func TestNuclearOptionDeadlineIncludesRequestWrites(t *testing.T) { + t.Parallel() + for _, readHeader := range []bool{false, true} { + stage := "write length" + if readHeader { + stage = "write body" + } + t.Run(stage, func(t *testing.T) { + t.Parallel() + local, remote := net.Pipe() + defer local.Close() + defer remote.Close() + client := NewNuclearOption("ignored", 1, nil) + client.conn = local + client.execDeadline = 100 * time.Millisecond + if readHeader { + go func() { _, _ = io.CopyN(io.Discard, remote, 4) }() + } + done := make(chan error, 1) + go func() { _, err := client.Exec("get-server-id"); done <- err }() + select { + case err := <-done: + if err == nil || !isTimeout(err) || !strings.Contains(err.Error(), stage) { + t.Fatalf("stalled %s must time out: %v", stage, err) + } + case <-time.After(2 * time.Second): + _ = remote.Close() + <-done + t.Fatalf("stalled %s exceeded the command budget", stage) + } + if client.conn != nil { + t.Fatal("timed-out partial request must discard the connection") + } + }) + } +} diff --git a/agent/internal/rcon/nuclearoption_test.go b/agent/internal/rcon/nuclearoption_test.go index 248c6a7fb..cb46a4e21 100644 --- a/agent/internal/rcon/nuclearoption_test.go +++ b/agent/internal/rcon/nuclearoption_test.go @@ -740,7 +740,7 @@ func TestNuclearOptionWriteBodyError(t *testing.T) { } // TestNuclearOptionSetDeadlineError verifies that a failure setting the -// response read deadline (after the request was fully written) is reported +// exchange deadline (before sending the request) is reported // and the connection is reset. func TestNuclearOptionSetDeadlineError(t *testing.T) { local, remote := net.Pipe() diff --git a/agent/specs.md b/agent/specs.md index 76e343a15..6806637ae 100644 --- a/agent/specs.md +++ b/agent/specs.md @@ -244,7 +244,7 @@ The agent and operator modules use `k8s.io/apimachinery`/`k8s.io/client-go` v0.3 - `battleye`: UDP-based CRC32 packet framing. - `satisfactory`: HTTPS JSON function-call API with session token. - `palworld`: HTTP Basic REST admin API. - - `nuclearoption`: Dedicated TCP stream for Nuclear Option remote command protocol. + - `nuclearoption`: Dedicated TCP stream for Nuclear Option remote command protocol. One 30-second deadline bounds dialing, request writes, and response reads; responses are capped at 10 MiB. A timed-out partial exchange closes the connection before the next command. - `rest`: Generic HTTP/JSON REST console client with lazy auth (`PassFn`), 1 MiB bounded body read, configurable timeouts, auth failure cooldown, and loopback TLS protection. Provides adapters for FiveM (txAdmin: `POST /fxserver/commands`, bearer/X-TxAdmin-Token, `{"action":"console","parameter":""}`), Farming Simulator 25 (`POST /api/console`, Basic auth, `{"command":""}`), and generic REST (`POST /api/command`). - `cli`: Container stdin/PTY / local execution client (Option A in OPEN-DECISIONS.md). Drives commands via named FIFO pipe (`GAMEPLANE_CLI_PIPE`) or local process shell execution without requiring remote TCP networking or password secrets, while web console continues to attach via pod-attach PTY. From 6cd3f48a95f647c271a8e808c57d284d3b9be9bf Mon Sep 17 00:00:00 2001 From: Baylem Date: Wed, 21 Oct 2026 17:16:54 -0400 Subject: [PATCH 03/18] fix: align monitoring validation and relay regression fixtures Signed-off-by: Baylem --- agent/internal/rcon/rcon_bounds_test.go | 10 ++++++++-- agent/internal/status/performance_test.go | 2 +- charts/gameplane/templates/monitoring-validation.yaml | 2 +- charts/gameplane/tests/monitoring-rules.sh | 4 ++++ charts/gameplane/values.yaml | 2 +- charts/tests/check_gateway.py | 4 +++- charts/tests/check_standalone.py | 2 ++ docs/monitoring.md | 2 +- go.work.sum | 1 + hack/sync-chart-crds.sh | 8 +++++--- .../controller/gameserver_relay_health_test.go | 11 +++++++++++ tunnel/relay_health.go | 6 ++++-- tunnel/relay_health_test.go | 2 +- tunnel/relay_metrics_test.go | 2 +- tunnel/tailscale.go | 8 ++++---- tunnel/tailscale_forward.go | 6 +++--- 16 files changed, 51 insertions(+), 21 deletions(-) diff --git a/agent/internal/rcon/rcon_bounds_test.go b/agent/internal/rcon/rcon_bounds_test.go index bc55fb868..39038bd81 100644 --- a/agent/internal/rcon/rcon_bounds_test.go +++ b/agent/internal/rcon/rcon_bounds_test.go @@ -36,7 +36,10 @@ func TestExecContinuousFragmentsRespectAbsoluteDeadline(t *testing.T) { } // Empty fragments cannot trigger the aggregate byte cap. A // continuously readable socket must still reach its hard deadline. - for server.writePacket(id, typeRespValue, "") == nil { + for { + if server.writePacket(id, typeRespValue, "") != nil { + return + } } }() finished := make(chan struct{}) @@ -138,7 +141,10 @@ func TestAuthContinuousPreambleRespectsAbsoluteDeadline(t *testing.T) { if err != nil { return } - for server.writePacket(id, typeRespValue, "") == nil { + for { + if server.writePacket(id, typeRespValue, "") != nil { + return + } } }() host, port, err := net.SplitHostPort(listener.Addr().String()) diff --git a/agent/internal/status/performance_test.go b/agent/internal/status/performance_test.go index 2155bb998..86e934242 100644 --- a/agent/internal/status/performance_test.go +++ b/agent/internal/status/performance_test.go @@ -32,7 +32,7 @@ func TestPerformanceCacheZeroFreshnessAndSharedCommand(t *testing.T) { router.Route("/versioned", s.Mount) for _, route := range []string{"/legacy/status", "/versioned/status"} { response := httptest.NewRecorder() - router.ServeHTTP(response, httptest.NewRequest("GET", route, nil)) + router.ServeHTTP(response, httptest.NewRequestWithContext(t.Context(), "GET", route, nil)) if response.Code != 200 { t.Fatal("shared status cache unavailable") } diff --git a/charts/gameplane/templates/monitoring-validation.yaml b/charts/gameplane/templates/monitoring-validation.yaml index 2e4dc0d7c..15fcacc60 100644 --- a/charts/gameplane/templates/monitoring-validation.yaml +++ b/charts/gameplane/templates/monitoring-validation.yaml @@ -1,3 +1,3 @@ -{{- if and .Values.serviceMonitors.enabled .Values.networkPolicies.enabled (not .Values.serviceMonitors.scrapeNamespaceSelector) (not .Values.serviceMonitors.externallyManagedPolicy) }} +{{- if and (eq (include "gameplane.operatorEnabled" .) "true") .Values.serviceMonitors.enabled .Values.networkPolicies.enabled (not .Values.serviceMonitors.scrapeNamespaceSelector) (not .Values.serviceMonitors.externallyManagedPolicy) }} {{- fail "serviceMonitors.enabled with networkPolicies.enabled requires serviceMonitors.scrapeNamespaceSelector (for example matchLabels: {kubernetes.io/metadata.name: monitoring}); alternatively set serviceMonitors.externallyManagedPolicy=true and supply your own scrape ingress policies" }} {{- end }} diff --git a/charts/gameplane/tests/monitoring-rules.sh b/charts/gameplane/tests/monitoring-rules.sh index 6b754c69b..6e39efd8e 100644 --- a/charts/gameplane/tests/monitoring-rules.sh +++ b/charts/gameplane/tests/monitoring-rules.sh @@ -15,5 +15,9 @@ for document in yaml.safe_load_all((root/'rendered.yaml').read_text()): (root/'rules.yaml').write_text(yaml.safe_dump({'groups':groups})) PY cp charts/gameplane/tests/monitoring-rules.test.yaml "$scratch/tests.yaml" +# mktemp creates a private directory, but promtool runs as the image's nobody +# user. These generated fixtures contain no credentials and are mounted read-only. +chmod 755 "$scratch" +chmod 644 "$scratch/rules.yaml" "$scratch/tests.yaml" docker run --rm --volume "$scratch:/work:ro" --workdir /work --entrypoint /bin/promtool \ prom/prometheus:v3.13.4 test rules tests.yaml diff --git a/charts/gameplane/values.yaml b/charts/gameplane/values.yaml index 9f6f530f7..4c93637ae 100644 --- a/charts/gameplane/values.yaml +++ b/charts/gameplane/values.yaml @@ -690,7 +690,7 @@ serviceMonitors: # so the receiver ServiceMonitor also needs # api.telemetry.receiver.dashboard.tokenSecretRef.name) when # networkPolicies.enabled is also true. Required - # alongside enabled: true whenever networkPolicies.enabled is also true; + # alongside enabled: true when the operator and networkPolicies are enabled; # left empty, chart validation fails unless externallyManagedPolicy is true. # Also permits tunnel health/metrics on port 8091. Example: # `{matchLabels: {kubernetes.io/metadata.name: monitoring}}`. diff --git a/charts/tests/check_gateway.py b/charts/tests/check_gateway.py index 83f174ed6..819e46979 100644 --- a/charts/tests/check_gateway.py +++ b/charts/tests/check_gateway.py @@ -81,7 +81,9 @@ def main(): gateway["namespaces"] = ["games-one", "games-two"] objects = render({ "gateway": gateway, "api": {"enabled": False, "telemetry": {"receiver": {"enabled": True}}}, - "serviceMonitors": {"enabled": True}, + "serviceMonitors": {"enabled": True, "scrapeNamespaceSelector": { + "matchLabels": {"kubernetes.io/metadata.name": "monitoring"}, + }}, "clusterOps": {"enabled": True}, }, upgrade=True) assert not any(obj["metadata"]["name"] in {"gameplane-api", "gameplane-api-data", "gameplane-web", "gameplane-strategy-migrate", "gameplane-telemetry-receiver"} for obj in objects) diff --git a/charts/tests/check_standalone.py b/charts/tests/check_standalone.py index 4caea9cc6..e6d43eb82 100644 --- a/charts/tests/check_standalone.py +++ b/charts/tests/check_standalone.py @@ -53,6 +53,8 @@ def main(): assert any(rule.get("resources") == ["clusters"] and "update" in rule["verbs"] for rule in api_rules) assert "--standalone" not in one(defaults, "Deployment", "gameplane-api")["spec"]["template"]["spec"]["containers"][0]["args"] + # With the operator disabled no chart NetworkPolicies render, so monitors + # must remain valid without a game-namespace scrape selector. values = { "operator": {"enabled": False}, "api": {"standalone": True}, "serviceMonitors": {"enabled": True}, "prometheusRules": {"enabled": True}, diff --git a/docs/monitoring.md b/docs/monitoring.md index 3ec5f53db..34c6346ec 100644 --- a/docs/monitoring.md +++ b/docs/monitoring.md @@ -28,7 +28,7 @@ Check Prometheus **Status → Targets** for agent and tunnel targets with `up=1` then query `gameplane_agent_players_online` and `gameplane_tunnel_ready`. Unknown player counts are absent even when scraping works. -With NetworkPolicies enabled, the chart requires a nonempty +With the operator and its NetworkPolicies enabled, the chart requires a nonempty `serviceMonitors.scrapeNamespaceSelector`. Missing configuration fails with a message naming the setting. If another system owns the policies, set `serviceMonitors.externallyManagedPolicy: true`; that flag opens no traffic. diff --git a/go.work.sum b/go.work.sum index 1a92b2fe8..2d4f7c3f7 100644 --- a/go.work.sum +++ b/go.work.sum @@ -407,6 +407,7 @@ github.com/maratori/testableexamples v1.0.1/go.mod h1:XE2F/nQs7B9N08JgyRmdGjYVGq github.com/maratori/testpackage v1.1.2/go.mod h1:8F24GdVDFW5Ew43Et02jamrVMNXLUNaOynhDssITGfc= github.com/maruel/natural v1.1.1/go.mod h1:v+Rfd79xlw1AgVBjbO0BEQmptqb5HvL/k9GRHB7ZKEg= github.com/matoous/godox v1.1.0/go.mod h1:jgE/3fUXiTurkdHOLT5WEkThTSuE7yxHv5iWPa80afs= +github.com/mattn/go-colorable v0.1.15 h1:+u9SLTRGnXv73cEsnsmoZBom+dMU88B2M0aDcWy0/jY= github.com/mattn/go-colorable v0.1.15/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.16/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= diff --git a/hack/sync-chart-crds.sh b/hack/sync-chart-crds.sh index 98ba9ff18..a86fe3f5a 100755 --- a/hack/sync-chart-crds.sh +++ b/hack/sync-chart-crds.sh @@ -48,19 +48,21 @@ if command -v sha256sum >/dev/null 2>&1; then else sha256() { shasum -a 256; } fi -hash=$(cat "${files[@]}" | sha256 | cut -d' ' -f1) +# Git may check out these text files with CRLF on Windows. Hash and copy the +# canonical LF representation so the stamp matches Linux code generation. +hash=$(sed $'s/\r$//' "${files[@]}" | sha256 | cut -d' ' -f1) for f in "${files[@]}"; do # controller-gen always emits exactly one top-level metadata.annotations # block (two-space indent); schema-level `annotations:` keys sit deeper. - if [ "$(grep -c '^ annotations:$' "$f")" != 1 ]; then + if [ "$(sed $'s/\r$//' "$f" | grep -c '^ annotations:$')" != 1 ]; then echo "sync-chart-crds: $f has no single metadata.annotations block to stamp" >&2 exit 1 fi base=$(basename "$f") for dir in charts/gameplane/charts/gameplane-crds/crds charts/gameplane/crd-manifests; do awk -v line=" ${annotation}: ${hash}" ' - { print } + { sub(/\r$/, ""); print } /^ annotations:$/ && !done { print line; done = 1 } ' "$f" > "$dir/$base" done diff --git a/operator/internal/controller/gameserver_relay_health_test.go b/operator/internal/controller/gameserver_relay_health_test.go index 1ae0379f6..edbce76a0 100644 --- a/operator/internal/controller/gameserver_relay_health_test.go +++ b/operator/internal/controller/gameserver_relay_health_test.go @@ -10,6 +10,7 @@ import ( "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" @@ -101,6 +102,16 @@ func TestPlayitAssignmentDoesNotImplyRelayReady(t *testing.T) { t.Parallel() gs := &gameplanev1alpha1.GameServer{Spec: gameplanev1alpha1.GameServerSpec{Networking: gameplanev1alpha1.GameServerNetworking{Tunnel: &gameplanev1alpha1.GameServerTunnel{Provider: "playit", Playit: &gameplanev1alpha1.PlayitTunnelSpec{}, CredentialsSecretRef: &gameplanev1alpha1.SecretNameRef{Name: "relay-credentials"}}}}} dep := tunnelReadyDeployment() + // A legacy deployment without the provider probe remains unknown even + // when its Pod is Ready; assignment alone cannot bypass that requirement. + legacy := meta.FindStatusCondition(computeTunnelConditions(gs, tunnelPlan{wantTunnel: true}, dep), "TunnelReady") + if legacy == nil || legacy.Status != metav1.ConditionUnknown || legacy.Reason != "RelayConnectionUnknown" { + t.Fatalf("legacy playit readiness=%+v, want unknown connection", legacy) + } + dep.Generation = 1 + dep.Status.ObservedGeneration = 1 + dep.Status.UpdatedReplicas = 1 + dep.Spec.Template.Spec.Containers = []corev1.Container{{Name: "tunnel", ReadinessProbe: &corev1.Probe{ProbeHandler: corev1.ProbeHandler{HTTPGet: &corev1.HTTPGetAction{Path: "/readyz", Port: intstr.FromInt32(8091)}}}}} for _, endpoints := range [][]gameplanev1alpha1.GameServerEndpoint{nil, {{Name: "game", Host: "example.playit.gg", Port: 30000}}} { gs.Status.TunnelEndpoints = endpoints cond := meta.FindStatusCondition(computeTunnelConditions(gs, tunnelPlan{wantTunnel: true}, dep), "TunnelReady") diff --git a/tunnel/relay_health.go b/tunnel/relay_health.go index 290d51b58..290efb25b 100644 --- a/tunnel/relay_health.go +++ b/tunnel/relay_health.go @@ -154,8 +154,10 @@ func superviseRelay(ctx context.Context, cfg Config, h *relayHealth, check relay cancel() h.clear() <-done - if ctx.Err() != nil { + select { + case <-ctx.Done(): return nil + default: } log.Print("relay process exited; readiness cleared") if isUnrecoverable(cfg, err) { @@ -266,7 +268,7 @@ func validFrpIdentity(uid string) bool { return false } for _, r := range uid { - if !(r >= 'a' && r <= 'z' || r >= 'A' && r <= 'Z' || r >= '0' && r <= '9' || r == '-') { + if (r < 'a' || r > 'z') && (r < 'A' || r > 'Z') && (r < '0' || r > '9') && r != '-' { return false } } diff --git a/tunnel/relay_health_test.go b/tunnel/relay_health_test.go index d4b1274ab..b6b363225 100644 --- a/tunnel/relay_health_test.go +++ b/tunnel/relay_health_test.go @@ -50,7 +50,7 @@ func TestRelayHealthHTTPSeparatesLivenessAndReadiness(t *testing.T) { handler := relayHealthHandler(h) for _, path := range []string{"/healthz", "/readyz", "/status"} { w := httptest.NewRecorder() - handler.ServeHTTP(w, httptest.NewRequest(http.MethodGet, path, nil)) + handler.ServeHTTP(w, httptest.NewRequestWithContext(t.Context(), http.MethodGet, path, nil)) want := http.StatusOK if path == "/readyz" { want = http.StatusServiceUnavailable diff --git a/tunnel/relay_metrics_test.go b/tunnel/relay_metrics_test.go index 52001d16f..90f73ff48 100644 --- a/tunnel/relay_metrics_test.go +++ b/tunnel/relay_metrics_test.go @@ -21,7 +21,7 @@ func TestRelayMetricsUnknownStaleAndRecovery(t *testing.T) { } { h.observe(time.Now().Add(-tc.age), tc.state, nil) w := httptest.NewRecorder() - relayHealthHandler(h).ServeHTTP(w, httptest.NewRequest("GET", "/metrics", nil)) + relayHealthHandler(h).ServeHTTP(w, httptest.NewRequestWithContext(t.Context(), "GET", "/metrics", nil)) body := w.Body.String() if !strings.Contains(body, "gameplane_tunnel_ready "+tc.ready+"\n") { t.Fatal(body) diff --git a/tunnel/tailscale.go b/tunnel/tailscale.go index 3cbb609b9..93a42b989 100644 --- a/tunnel/tailscale.go +++ b/tunnel/tailscale.go @@ -62,10 +62,10 @@ func checkTailnetHostname(status *ipnstate.Status, hostname string) (*ipnstate.S func tailscaleServer(cfg Config, authKey string) (*tsnet.Server, []string, error) { tags, err := parseTailscaleTags(cfg.TailscaleTags) if err != nil { - return nil, nil, fmt.Errorf("Tailscale tags: %w", err) + return nil, nil, fmt.Errorf("tailscale tags: %w", err) } if cfg.TailscaleHostname == "" || authKey == "" { - return nil, nil, errors.New("Tailscale hostname and authKey are required") + return nil, nil, errors.New("tailscale hostname and authKey are required") } return &tsnet.Server{Hostname: cfg.TailscaleHostname, AuthKey: authKey, AdvertiseTags: tags, Dir: tailscaleStateDir, // Neither verbose daemon logs nor interactive login URLs belong in pod @@ -83,7 +83,7 @@ func runTailscale(ctx context.Context, cfg Config, authKey string) error { healthHistory := &relayHealth{} listener, err := (&net.ListenConfig{}).Listen(ctx, "tcp", ":8091") if err != nil { - return fmt.Errorf("Tailscale health listener: %w", err) + return fmt.Errorf("tailscale health listener: %w", err) } health := &http.Server{Handler: relayHealthHandler(healthHistory), ReadHeaderTimeout: 2 * time.Second, ReadTimeout: 5 * time.Second, WriteTimeout: 5 * time.Second, IdleTimeout: 10 * time.Second} healthDone := make(chan error, 1) @@ -118,7 +118,7 @@ func runTailscale(ctx context.Context, cfg Config, authKey string) error { case <-ctx.Done(): return nil case err := <-healthDone: - return fmt.Errorf("Tailscale health server: %w", err) + return fmt.Errorf("tailscale health server: %w", err) case <-time.After(backoff.next()): } } diff --git a/tunnel/tailscale_forward.go b/tunnel/tailscale_forward.go index d55b6bc06..d74eda22b 100644 --- a/tunnel/tailscale_forward.go +++ b/tunnel/tailscale_forward.go @@ -18,18 +18,18 @@ type forwardPort struct{ network, port, target string } // packet can choose a destination, and this node never advertises subnet routes. func tailscaleForwardPlan(cfg Config) ([]forwardPort, error) { if cfg.GameServerName == "" || cfg.GameServerNamespace == "" || cfg.BackingServiceDNS != cfg.GameServerName+"."+cfg.GameServerNamespace+".svc" { - return nil, errors.New("Tailscale target must be the GameServer's backing Service DNS") + return nil, errors.New("tailscale target must be the GameServer's backing Service DNS") } entries := strings.Split(cfg.BackingServicePorts, ",") if len(entries) > 32 { - return nil, errors.New("Tailscale supports at most 32 advertised ports") + return nil, errors.New("tailscale supports at most 32 advertised ports") } ports := make([]forwardPort, 0, len(entries)) seen := map[string]bool{} for _, entry := range entries { parts := strings.Split(entry, ":") if len(parts) != 3 || parts[0] == "" { - return nil, fmt.Errorf("Tailscale port %q must be name:port:tcp or name:port:udp; update the operator and tunnel images together", entry) + return nil, fmt.Errorf("tailscale port %q must be name:port:tcp or name:port:udp; update the operator and tunnel images together", entry) } port, err := strconv.Atoi(parts[1]) if err != nil || port < 1 || port > 65535 { From 55ddd2d3193d9b16b0422ad3cfdb202dc302e081 Mon Sep 17 00:00:00 2001 From: Baylem Date: Wed, 21 Oct 2026 22:49:08 -0400 Subject: [PATCH 04/18] fix(e2e): read metrics probes from completed pod logs Signed-off-by: Baylem --- test/e2e/helm_install_e2e_test.go | 65 +++++++++++++++++++++++++------ 1 file changed, 53 insertions(+), 12 deletions(-) diff --git a/test/e2e/helm_install_e2e_test.go b/test/e2e/helm_install_e2e_test.go index ffa62e6aa..4c6ba7b7e 100644 --- a/test/e2e/helm_install_e2e_test.go +++ b/test/e2e/helm_install_e2e_test.go @@ -17,6 +17,7 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/wait" "sigs.k8s.io/yaml" ) @@ -221,12 +222,58 @@ func TestHelmInstall_APIHealthz(t *testing.T) { // metricsProbeScript runs in a transient curl pod. It prints the public // API port's status code for /metrics, flags a Prometheus body there, and // reports whether the dedicated metrics port serves Prometheus text. -const metricsProbeScript = `code=$(curl -s -o /dev/null -w '%{http_code}' --max-time 5 http://gameplane-api/metrics) +const metricsProbeScript = `code=$(curl -sS -o /tmp/public-metrics -w '%{http_code}' --max-time 5 http://gameplane-api/metrics) || exit 1 echo "public-status=$code" -if curl -s --max-time 5 http://gameplane-api/metrics | grep -q '^# HELP'; then echo "public-body=metrics"; fi -if curl -fsS --max-time 5 http://gameplane-api:9090/metrics | grep -q '^# HELP go_goroutines'; then echo "metrics-port=ok"; fi +if grep -q '^# HELP' /tmp/public-metrics; then echo "public-body=metrics"; fi +if curl -fsS -o /tmp/dedicated-metrics --max-time 5 http://gameplane-api:9090/metrics; then + if grep -q '^# HELP go_goroutines' /tmp/dedicated-metrics; then + echo "metrics-port=ok" + else + echo "metrics-port=unexpected-body" + head -n 10 /tmp/dedicated-metrics + fi +fi exit 0` +// Read completed Pod logs instead of attaching after its command has started. +// A fast probe can print its verdict before kubectl run establishes the stream. +func metricsProbeOutput(ctx context.Context, name string) (string, error) { + const ns = "gameplane-system" + noToken := false + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: ns}, Spec: corev1.PodSpec{ + RestartPolicy: corev1.RestartPolicyNever, AutomountServiceAccountToken: &noToken, + Containers: []corev1.Container{{Name: "probe", Image: "curlimages/curl:8.10.1", Command: []string{"sh", "-c", metricsProbeScript}}}, + }} + pods := envInstance.K8s.CoreV1().Pods(ns) + if _, err := pods.Create(ctx, pod, metav1.CreateOptions{}); err != nil { + return "", fmt.Errorf("create metrics probe: %w", err) + } + defer func() { + cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 5*time.Second) + defer cancel() + _ = pods.Delete(cleanupCtx, name, metav1.DeleteOptions{}) + }() + var phase corev1.PodPhase + if err := wait.PollUntilContextCancel(ctx, 250*time.Millisecond, true, func(ctx context.Context) (bool, error) { + current, err := pods.Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return false, err + } + phase = current.Status.Phase + return phase == corev1.PodSucceeded || phase == corev1.PodFailed, nil + }); err != nil { + return "", fmt.Errorf("wait for metrics probe (phase %s): %w", phase, err) + } + out, err := pods.GetLogs(name, &corev1.PodLogOptions{Container: "probe"}).DoRaw(ctx) + if err != nil { + return "", fmt.Errorf("read metrics probe logs: %w", err) + } + if phase != corev1.PodSucceeded { + return string(out), fmt.Errorf("metrics probe pod finished in phase %s", phase) + } + return string(out), nil +} + // TestHelmInstall_MetricsNotOnPublicPort — the API's public port (the // Service port the Ingress and the web front end route to) does not serve // Prometheus metrics, and the dedicated metrics port that the ServiceMonitor @@ -240,15 +287,9 @@ func TestHelmInstall_MetricsNotOnPublicPort(t *testing.T) { // Random suffix so an Eventually retry doesn't collide with a // not-yet-cleaned-up pod from the previous tick. name := fmt.Sprintf("metrics-probe-%d", time.Now().UnixNano()) - o, err := envInstance.Kubectl( - t.Context(), - "run", "-n", "gameplane-system", - "--rm", "--restart=Never", "--attach", - "--image=curlimages/curl:8.10.1", - name, - "--command", "--", - "sh", "-c", metricsProbeScript, - ) + ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) + defer cancel() + o, err := metricsProbeOutput(ctx, name) if err != nil { return false, fmt.Sprintf("metrics probe pod failed: %v\n%s", err, o) } From 7b794f257763d83c28a9f60a2cd4b587a3bef81a Mon Sep 17 00:00:00 2001 From: Baylem Date: Wed, 21 Oct 2026 22:49:07 -0400 Subject: [PATCH 05/18] fix(e2e): distinguish policy verdicts from probe diagnostics Signed-off-by: Baylem --- .github/workflows/ci.yaml | 3 +- test/e2e/buckets.sh | 4 ++- test/e2e/network_isolation_e2e_test.go | 41 ++++++++++++++++++++++++-- 3 files changed, 44 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index bd2474c2b..02121e0ba 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -372,7 +372,8 @@ jobs: needs: [changes] if: needs.changes.outputs.images == 'true' runs-on: ubuntu-latest - timeout-minutes: 15 + # Include all image builds, cache exports, artifact upload and cleanup. + timeout-minutes: 30 permissions: contents: read steps: diff --git a/test/e2e/buckets.sh b/test/e2e/buckets.sh index 1ad462f1e..8e6645b95 100755 --- a/test/e2e/buckets.sh +++ b/test/e2e/buckets.sh @@ -404,7 +404,9 @@ TestMonitoring_DiscoveryPerformanceAndPolicy EOF } -bucket_network_isolation() { echo TestNetworkIsolation_EnforcedAndUpgrade; } +bucket_network_isolation() { + printf '%s\n' TestNetworkIsolation_EnforcedAndUpgrade TestNetworkIsolation_ProbeOutput +} bucket_names() { printf '%s\n' operator api-auth api-roles api-rbac api-agent api-mods ratelimit bot-fast bot-heavy multicluster upgrade telemetry capture-overhead optional-components management-placement network-isolation monitoring diff --git a/test/e2e/network_isolation_e2e_test.go b/test/e2e/network_isolation_e2e_test.go index 436a192ee..412b9b6ab 100644 --- a/test/e2e/network_isolation_e2e_test.go +++ b/test/e2e/network_isolation_e2e_test.go @@ -183,11 +183,48 @@ func isolationHTTP(t *testing.T, source, ip string, port int, allowed bool) { if err != nil { return false, "probe execution failed: " + err.Error() + out } - got := strings.TrimSpace(out) + got := isolationProbeVerdict(out) want := "DENIED" if allowed { want = "ALLOWED" } - return got == want, fmt.Sprintf("%s -> %s: got %q, want %s", source, target, got, want) + return got == want, fmt.Sprintf("%s -> %s: got %q, want %s; output: %q", source, target, got, want, out) }) } + +// Kubectl combines stdout and stderr, so wget's expected timeout diagnostic +// can precede the result. Require one exact marker; never accept missing or +// contradictory results, and retain the full output in the failure message. +func isolationProbeVerdict(out string) string { + var verdict string + for line := range strings.SplitSeq(out, "\n") { + line = strings.TrimSpace(line) + if line != "ALLOWED" && line != "DENIED" { + continue + } + if verdict != "" { + return "" + } + verdict = line + } + return verdict +} + +func TestNetworkIsolation_ProbeOutput(t *testing.T) { + t.Parallel() + for _, tc := range []struct{ name, output, want string }{ + {"allowed", "ALLOWED\n", "ALLOWED"}, + {"denied with wget timeout", "wget: download timed out\nDENIED\n", "DENIED"}, + {"denied with carriage return", "wget: connection refused\r\nDENIED\r\n", "DENIED"}, + {"diagnostic only", "wget: download timed out\n", ""}, + {"marker substring", "unexpected ALLOWED response\n", ""}, + {"contradictory markers", "ALLOWED\nDENIED\n", ""}, + {"duplicate markers", "DENIED\nDENIED\n", ""}, + } { + t.Run(tc.name, func(t *testing.T) { + if got := isolationProbeVerdict(tc.output); got != tc.want { + t.Fatalf("probe verdict = %q, want %q for %q", got, tc.want, tc.output) + } + }) + } +} From 6923113719d1b13db3046f53414aeed104420661 Mon Sep 17 00:00:00 2001 From: Baylem Date: Thu, 22 Oct 2026 20:20:43 -0400 Subject: [PATCH 06/18] test(e2e): drive event streams and retry monitoring downloads Signed-off-by: Baylem --- deploy/kind/monitoring.sh | 2 + test/e2e/api_events_roles_scope_e2e_test.go | 57 +++++++++++++++++++-- 2 files changed, 56 insertions(+), 3 deletions(-) diff --git a/deploy/kind/monitoring.sh b/deploy/kind/monitoring.sh index 1bbc20063..f5c2e4a4e 100644 --- a/deploy/kind/monitoring.sh +++ b/deploy/kind/monitoring.sh @@ -6,7 +6,9 @@ context="${CTX:-kind-gameplane-e2e}" kubectl() { command kubectl --context "$context" "$@"; } scratch="$(mktemp -d)" trap 'rm -rf "$scratch"' EXIT +# Retry transient upstream failures without relaxing the pinned checksum below. curl --fail --silent --show-error --location \ + --connect-timeout 10 --max-time 30 --retry 5 --retry-max-time 120 \ https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.94.1/bundle.yaml \ -o "$scratch/bundle.yaml" echo "7ab177610b271c70c57c7d6fc550a1e934eff4f707a5d8b61dfaa3845d52deea $scratch/bundle.yaml" | sha256sum --check diff --git a/test/e2e/api_events_roles_scope_e2e_test.go b/test/e2e/api_events_roles_scope_e2e_test.go index 2ab19b2b4..4c3b388c0 100644 --- a/test/e2e/api_events_roles_scope_e2e_test.go +++ b/test/e2e/api_events_roles_scope_e2e_test.go @@ -12,6 +12,9 @@ import ( "strings" "testing" "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" ) // TestAPI_EventStreamAndRoleEdits_FollowCallerPermissions checks two @@ -64,19 +67,37 @@ func TestAPI_EventStreamAndRoleEdits_FollowCallerPermissions(t *testing.T) { applyBusyboxTemplate(t, tmpl) createGameServerViaAPI(t, admin, ns, gsName, tmpl) + // /events starts at each collection's current resourceVersion. Drive real + // updates throughout both streams instead of depending on initial replay + // or other parallel tests to happen to mutate a readable resource kind. + changes := func(ctx context.Context) error { + patch, err := json.Marshal(map[string]any{"metadata": map[string]any{"annotations": map[string]string{ + "gameplane.local/e2e-event-sequence": time.Now().UTC().Format(time.RFC3339Nano), + }}}) + if err != nil { + return fmt.Errorf("encode event trigger: %w", err) + } + if _, err := envInstance.Dyn.Resource(gameTemplateGVR).Patch(ctx, tmpl, types.MergePatchType, patch, metav1.PatchOptions{}); err != nil { + return fmt.Errorf("trigger template event: %w", err) + } + if _, err := envInstance.Dyn.Resource(gameServerGVR).Namespace(ns).Patch(ctx, gsName, types.MergePatchType, patch, metav1.PatchOptions{}); err != nil { + return fmt.Errorf("trigger server event: %w", err) + } + return nil + } user := envInstance.APIClient(t, userName, userPW) defer user.Close() t.Run("admin event stream includes templates", func(t *testing.T) { - kinds, found := streamEventKinds(t, admin, ns, func(kind, _ string) bool { return kind == "templates" }, 0) + kinds, found := streamEventKinds(t, admin, ns, func(kind, name string) bool { return kind == "templates" && name == tmpl }, 0, changes) if !found { t.Fatalf("admin stream sent no templates event; kinds=%v", kinds) } }) t.Run("event stream sends only kinds the caller may read", func(t *testing.T) { - kinds, found := streamEventKinds(t, user, ns, func(kind, name string) bool { return kind == "servers" && name == gsName }, 3*time.Second) + kinds, found := streamEventKinds(t, user, ns, func(kind, name string) bool { return kind == "servers" && name == gsName }, 3*time.Second, changes) if !found { t.Fatalf("user stream never sent server %s; kinds=%v", gsName, kinds) } @@ -126,11 +147,41 @@ func TestAPI_EventStreamAndRoleEdits_FollowCallerPermissions(t *testing.T) { // streamEventKinds reads GET /events as cli and returns the set of event // kinds it received. When match reports true for a frame, it keeps // reading for grace (0 = stop at once); otherwise it stops after 30s. +// changes keeps both permitted and forbidden kinds active while connecting and +// reading; its worker is cancelled and joined before the helper returns. // found reports whether match was ever true. -func streamEventKinds(t *testing.T, cli *APIClient, ns string, match func(kind, name string) bool, grace time.Duration) (map[string]bool, bool) { +func streamEventKinds(t *testing.T, cli *APIClient, ns string, match func(kind, name string) bool, grace time.Duration, changes func(context.Context) error) (map[string]bool, bool) { t.Helper() ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) defer cancel() + changesDone := make(chan struct{}) + var changesErr error + go func() { + defer close(changesDone) + ticker := time.NewTicker(500 * time.Millisecond) + defer ticker.Stop() + for { + if err := changes(ctx); err != nil { + if ctx.Err() == nil { + changesErr = err + } + cancel() + return + } + select { + case <-ctx.Done(): + return + case <-ticker.C: + } + } + }() + defer func() { + cancel() + <-changesDone + if changesErr != nil { + t.Errorf("event trigger failed: %v", changesErr) + } + }() req, err := http.NewRequestWithContext(ctx, http.MethodGet, cli.BaseURL+"/events?namespace="+ns, nil) if err != nil { t.Fatalf("new events request: %v", err) From d31f279f47b37f4f1961c2de41c4886e2bb59c91 Mon Sep 17 00:00:00 2001 From: Baylem Date: Thu, 22 Oct 2026 20:46:40 -0400 Subject: [PATCH 07/18] chore: synchronize CRDs with workspace generator Signed-off-by: Baylem --- .../crds/gameplane.local_backups.yaml | 4 ++-- .../crds/gameplane.local_backupschedules.yaml | 4 ++-- .../crds/gameplane.local_clusters.yaml | 4 ++-- .../crds/gameplane.local_gameservers.yaml | 4 ++-- .../crds/gameplane.local_gametemplates.yaml | 4 ++-- .../crds/gameplane.local_modules.yaml | 4 ++-- .../crds/gameplane.local_modulesources.yaml | 4 ++-- .../crds/gameplane.local_networkcaptures.yaml | 23 +++++++------------ .../crds/gameplane.local_restores.yaml | 4 ++-- .../gameplane.local_backups.yaml | 4 ++-- .../gameplane.local_backupschedules.yaml | 4 ++-- .../gameplane.local_clusters.yaml | 4 ++-- .../gameplane.local_gameservers.yaml | 4 ++-- .../gameplane.local_gametemplates.yaml | 4 ++-- .../gameplane.local_modules.yaml | 4 ++-- .../gameplane.local_modulesources.yaml | 4 ++-- .../gameplane.local_networkcaptures.yaml | 23 +++++++------------ .../gameplane.local_restores.yaml | 4 ++-- go.work.sum | 1 + .../config/crd/gameplane.local_backups.yaml | 2 +- .../crd/gameplane.local_backupschedules.yaml | 2 +- .../config/crd/gameplane.local_clusters.yaml | 2 +- .../crd/gameplane.local_gameservers.yaml | 2 +- .../crd/gameplane.local_gametemplates.yaml | 2 +- .../config/crd/gameplane.local_modules.yaml | 2 +- .../crd/gameplane.local_modulesources.yaml | 2 +- .../crd/gameplane.local_networkcaptures.yaml | 21 ++++++----------- .../config/crd/gameplane.local_restores.yaml | 2 +- 28 files changed, 64 insertions(+), 84 deletions(-) diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backups.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backups.yaml index 448222049..2c964a9c7 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backups.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backups.yaml @@ -3,8 +3,8 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 1dc8f563ef15434a03e2f9c2f61c92576d3a3483dd19c787434ed8b4b743cacd - controller-gen.kubebuilder.io/version: v0.20.1 + gameplane.local/crd-bundle-sha256: 519f827ad83dfea7450e7d4eb30f2433a7b3c04f111795fcef0496d81ee3a770 + controller-gen.kubebuilder.io/version: v0.22.0 name: backups.gameplane.local spec: group: gameplane.local diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backupschedules.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backupschedules.yaml index cae7c3a68..f35faf587 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backupschedules.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backupschedules.yaml @@ -3,8 +3,8 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 1dc8f563ef15434a03e2f9c2f61c92576d3a3483dd19c787434ed8b4b743cacd - controller-gen.kubebuilder.io/version: v0.20.1 + gameplane.local/crd-bundle-sha256: 519f827ad83dfea7450e7d4eb30f2433a7b3c04f111795fcef0496d81ee3a770 + controller-gen.kubebuilder.io/version: v0.22.0 name: backupschedules.gameplane.local spec: group: gameplane.local diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_clusters.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_clusters.yaml index 0dff5ac61..a00a67448 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_clusters.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_clusters.yaml @@ -3,8 +3,8 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 1dc8f563ef15434a03e2f9c2f61c92576d3a3483dd19c787434ed8b4b743cacd - controller-gen.kubebuilder.io/version: v0.20.1 + gameplane.local/crd-bundle-sha256: 519f827ad83dfea7450e7d4eb30f2433a7b3c04f111795fcef0496d81ee3a770 + controller-gen.kubebuilder.io/version: v0.22.0 name: clusters.gameplane.local spec: group: gameplane.local diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gameservers.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gameservers.yaml index bf338daa1..dcbc60df0 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gameservers.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gameservers.yaml @@ -3,8 +3,8 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 1dc8f563ef15434a03e2f9c2f61c92576d3a3483dd19c787434ed8b4b743cacd - controller-gen.kubebuilder.io/version: v0.20.1 + gameplane.local/crd-bundle-sha256: 519f827ad83dfea7450e7d4eb30f2433a7b3c04f111795fcef0496d81ee3a770 + controller-gen.kubebuilder.io/version: v0.22.0 name: gameservers.gameplane.local spec: group: gameplane.local diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gametemplates.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gametemplates.yaml index a9b33a3f8..97d49d8bd 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gametemplates.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gametemplates.yaml @@ -3,8 +3,8 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 1dc8f563ef15434a03e2f9c2f61c92576d3a3483dd19c787434ed8b4b743cacd - controller-gen.kubebuilder.io/version: v0.20.1 + gameplane.local/crd-bundle-sha256: 519f827ad83dfea7450e7d4eb30f2433a7b3c04f111795fcef0496d81ee3a770 + controller-gen.kubebuilder.io/version: v0.22.0 name: gametemplates.gameplane.local spec: group: gameplane.local diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modules.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modules.yaml index e5d1f6c77..099d0e3b0 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modules.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modules.yaml @@ -3,8 +3,8 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 1dc8f563ef15434a03e2f9c2f61c92576d3a3483dd19c787434ed8b4b743cacd - controller-gen.kubebuilder.io/version: v0.20.1 + gameplane.local/crd-bundle-sha256: 519f827ad83dfea7450e7d4eb30f2433a7b3c04f111795fcef0496d81ee3a770 + controller-gen.kubebuilder.io/version: v0.22.0 name: modules.gameplane.local spec: group: gameplane.local diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modulesources.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modulesources.yaml index d618481a4..e31ebb242 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modulesources.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modulesources.yaml @@ -3,8 +3,8 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 1dc8f563ef15434a03e2f9c2f61c92576d3a3483dd19c787434ed8b4b743cacd - controller-gen.kubebuilder.io/version: v0.20.1 + gameplane.local/crd-bundle-sha256: 519f827ad83dfea7450e7d4eb30f2433a7b3c04f111795fcef0496d81ee3a770 + controller-gen.kubebuilder.io/version: v0.22.0 name: modulesources.gameplane.local spec: group: gameplane.local diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_networkcaptures.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_networkcaptures.yaml index 4d7d7c25f..565e4fb47 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_networkcaptures.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_networkcaptures.yaml @@ -3,8 +3,8 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 1dc8f563ef15434a03e2f9c2f61c92576d3a3483dd19c787434ed8b4b743cacd - controller-gen.kubebuilder.io/version: v0.20.1 + gameplane.local/crd-bundle-sha256: 519f827ad83dfea7450e7d4eb30f2433a7b3c04f111795fcef0496d81ee3a770 + controller-gen.kubebuilder.io/version: v0.22.0 name: networkcaptures.gameplane.local spec: group: gameplane.local @@ -209,22 +209,15 @@ spec: format: int64 type: integer phase: - allOf: - - enum: - - Pending - - Running - - Completed - - Failed - - Expired - - enum: - - Pending - - Running - - Completed - - Failed - - Expired description: |- Phase is the current lifecycle phase of the capture. Valid values: Pending, Running, Completed, Failed, Expired. + enum: + - Pending + - Running + - Completed + - Failed + - Expired type: string startTime: description: |- diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_restores.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_restores.yaml index f5a47f635..ab12e096d 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_restores.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_restores.yaml @@ -3,8 +3,8 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 1dc8f563ef15434a03e2f9c2f61c92576d3a3483dd19c787434ed8b4b743cacd - controller-gen.kubebuilder.io/version: v0.20.1 + gameplane.local/crd-bundle-sha256: 519f827ad83dfea7450e7d4eb30f2433a7b3c04f111795fcef0496d81ee3a770 + controller-gen.kubebuilder.io/version: v0.22.0 name: restores.gameplane.local spec: group: gameplane.local diff --git a/charts/gameplane/crd-manifests/gameplane.local_backups.yaml b/charts/gameplane/crd-manifests/gameplane.local_backups.yaml index 448222049..2c964a9c7 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_backups.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_backups.yaml @@ -3,8 +3,8 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 1dc8f563ef15434a03e2f9c2f61c92576d3a3483dd19c787434ed8b4b743cacd - controller-gen.kubebuilder.io/version: v0.20.1 + gameplane.local/crd-bundle-sha256: 519f827ad83dfea7450e7d4eb30f2433a7b3c04f111795fcef0496d81ee3a770 + controller-gen.kubebuilder.io/version: v0.22.0 name: backups.gameplane.local spec: group: gameplane.local diff --git a/charts/gameplane/crd-manifests/gameplane.local_backupschedules.yaml b/charts/gameplane/crd-manifests/gameplane.local_backupschedules.yaml index cae7c3a68..f35faf587 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_backupschedules.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_backupschedules.yaml @@ -3,8 +3,8 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 1dc8f563ef15434a03e2f9c2f61c92576d3a3483dd19c787434ed8b4b743cacd - controller-gen.kubebuilder.io/version: v0.20.1 + gameplane.local/crd-bundle-sha256: 519f827ad83dfea7450e7d4eb30f2433a7b3c04f111795fcef0496d81ee3a770 + controller-gen.kubebuilder.io/version: v0.22.0 name: backupschedules.gameplane.local spec: group: gameplane.local diff --git a/charts/gameplane/crd-manifests/gameplane.local_clusters.yaml b/charts/gameplane/crd-manifests/gameplane.local_clusters.yaml index 0dff5ac61..a00a67448 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_clusters.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_clusters.yaml @@ -3,8 +3,8 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 1dc8f563ef15434a03e2f9c2f61c92576d3a3483dd19c787434ed8b4b743cacd - controller-gen.kubebuilder.io/version: v0.20.1 + gameplane.local/crd-bundle-sha256: 519f827ad83dfea7450e7d4eb30f2433a7b3c04f111795fcef0496d81ee3a770 + controller-gen.kubebuilder.io/version: v0.22.0 name: clusters.gameplane.local spec: group: gameplane.local diff --git a/charts/gameplane/crd-manifests/gameplane.local_gameservers.yaml b/charts/gameplane/crd-manifests/gameplane.local_gameservers.yaml index bf338daa1..dcbc60df0 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_gameservers.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_gameservers.yaml @@ -3,8 +3,8 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 1dc8f563ef15434a03e2f9c2f61c92576d3a3483dd19c787434ed8b4b743cacd - controller-gen.kubebuilder.io/version: v0.20.1 + gameplane.local/crd-bundle-sha256: 519f827ad83dfea7450e7d4eb30f2433a7b3c04f111795fcef0496d81ee3a770 + controller-gen.kubebuilder.io/version: v0.22.0 name: gameservers.gameplane.local spec: group: gameplane.local diff --git a/charts/gameplane/crd-manifests/gameplane.local_gametemplates.yaml b/charts/gameplane/crd-manifests/gameplane.local_gametemplates.yaml index a9b33a3f8..97d49d8bd 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_gametemplates.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_gametemplates.yaml @@ -3,8 +3,8 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 1dc8f563ef15434a03e2f9c2f61c92576d3a3483dd19c787434ed8b4b743cacd - controller-gen.kubebuilder.io/version: v0.20.1 + gameplane.local/crd-bundle-sha256: 519f827ad83dfea7450e7d4eb30f2433a7b3c04f111795fcef0496d81ee3a770 + controller-gen.kubebuilder.io/version: v0.22.0 name: gametemplates.gameplane.local spec: group: gameplane.local diff --git a/charts/gameplane/crd-manifests/gameplane.local_modules.yaml b/charts/gameplane/crd-manifests/gameplane.local_modules.yaml index e5d1f6c77..099d0e3b0 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_modules.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_modules.yaml @@ -3,8 +3,8 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 1dc8f563ef15434a03e2f9c2f61c92576d3a3483dd19c787434ed8b4b743cacd - controller-gen.kubebuilder.io/version: v0.20.1 + gameplane.local/crd-bundle-sha256: 519f827ad83dfea7450e7d4eb30f2433a7b3c04f111795fcef0496d81ee3a770 + controller-gen.kubebuilder.io/version: v0.22.0 name: modules.gameplane.local spec: group: gameplane.local diff --git a/charts/gameplane/crd-manifests/gameplane.local_modulesources.yaml b/charts/gameplane/crd-manifests/gameplane.local_modulesources.yaml index d618481a4..e31ebb242 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_modulesources.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_modulesources.yaml @@ -3,8 +3,8 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 1dc8f563ef15434a03e2f9c2f61c92576d3a3483dd19c787434ed8b4b743cacd - controller-gen.kubebuilder.io/version: v0.20.1 + gameplane.local/crd-bundle-sha256: 519f827ad83dfea7450e7d4eb30f2433a7b3c04f111795fcef0496d81ee3a770 + controller-gen.kubebuilder.io/version: v0.22.0 name: modulesources.gameplane.local spec: group: gameplane.local diff --git a/charts/gameplane/crd-manifests/gameplane.local_networkcaptures.yaml b/charts/gameplane/crd-manifests/gameplane.local_networkcaptures.yaml index 4d7d7c25f..565e4fb47 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_networkcaptures.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_networkcaptures.yaml @@ -3,8 +3,8 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 1dc8f563ef15434a03e2f9c2f61c92576d3a3483dd19c787434ed8b4b743cacd - controller-gen.kubebuilder.io/version: v0.20.1 + gameplane.local/crd-bundle-sha256: 519f827ad83dfea7450e7d4eb30f2433a7b3c04f111795fcef0496d81ee3a770 + controller-gen.kubebuilder.io/version: v0.22.0 name: networkcaptures.gameplane.local spec: group: gameplane.local @@ -209,22 +209,15 @@ spec: format: int64 type: integer phase: - allOf: - - enum: - - Pending - - Running - - Completed - - Failed - - Expired - - enum: - - Pending - - Running - - Completed - - Failed - - Expired description: |- Phase is the current lifecycle phase of the capture. Valid values: Pending, Running, Completed, Failed, Expired. + enum: + - Pending + - Running + - Completed + - Failed + - Expired type: string startTime: description: |- diff --git a/charts/gameplane/crd-manifests/gameplane.local_restores.yaml b/charts/gameplane/crd-manifests/gameplane.local_restores.yaml index f5a47f635..ab12e096d 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_restores.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_restores.yaml @@ -3,8 +3,8 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 1dc8f563ef15434a03e2f9c2f61c92576d3a3483dd19c787434ed8b4b743cacd - controller-gen.kubebuilder.io/version: v0.20.1 + gameplane.local/crd-bundle-sha256: 519f827ad83dfea7450e7d4eb30f2433a7b3c04f111795fcef0496d81ee3a770 + controller-gen.kubebuilder.io/version: v0.22.0 name: restores.gameplane.local spec: group: gameplane.local diff --git a/go.work.sum b/go.work.sum index dbe6af969..da1f24437 100644 --- a/go.work.sum +++ b/go.work.sum @@ -1073,6 +1073,7 @@ k8s.io/utils v0.0.0-20260626114624-be93311217bd/go.mod h1:xDxuJ0whA3d0I4mf/C4ppK mvdan.cc/gofumpt v0.9.2/go.mod h1:iB7Hn+ai8lPvofHd9ZFGVg2GOr8sBUw1QUWjNbmIL/s= mvdan.cc/unparam v0.0.0-20251027182757-5beb8c8f8f15/go.mod h1:4M5MMXl2kW6fivUT6yRGpLLPNfuGtU2Z0cPvFquGDYU= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= +sigs.k8s.io/controller-tools v0.22.0 h1:eG3FAVja/KnlXKIWg95udIFz1cMyAtMjP11cqBh3t+k= sigs.k8s.io/controller-tools v0.22.0/go.mod h1:VizwUStoZK7rReCj704czGGrB7mLxXTiJSJt7wN5ilI= sigs.k8s.io/randfill v0.0.0-20250304075658-069ef1bbf016/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= sigs.k8s.io/release-utils v0.12.2/go.mod h1:Ab9Lb/FpGUw4lUXj1QYbUcF2TRzll+GS7Md54W1G7sA= diff --git a/operator/config/crd/gameplane.local_backups.yaml b/operator/config/crd/gameplane.local_backups.yaml index f96731731..9b3af89a3 100644 --- a/operator/config/crd/gameplane.local_backups.yaml +++ b/operator/config/crd/gameplane.local_backups.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.20.1 + controller-gen.kubebuilder.io/version: v0.22.0 name: backups.gameplane.local spec: group: gameplane.local diff --git a/operator/config/crd/gameplane.local_backupschedules.yaml b/operator/config/crd/gameplane.local_backupschedules.yaml index ac32183ae..2011ca4cd 100644 --- a/operator/config/crd/gameplane.local_backupschedules.yaml +++ b/operator/config/crd/gameplane.local_backupschedules.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.20.1 + controller-gen.kubebuilder.io/version: v0.22.0 name: backupschedules.gameplane.local spec: group: gameplane.local diff --git a/operator/config/crd/gameplane.local_clusters.yaml b/operator/config/crd/gameplane.local_clusters.yaml index fffd585ac..285dcc73d 100644 --- a/operator/config/crd/gameplane.local_clusters.yaml +++ b/operator/config/crd/gameplane.local_clusters.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.20.1 + controller-gen.kubebuilder.io/version: v0.22.0 name: clusters.gameplane.local spec: group: gameplane.local diff --git a/operator/config/crd/gameplane.local_gameservers.yaml b/operator/config/crd/gameplane.local_gameservers.yaml index b323bcd1d..4616a25b0 100644 --- a/operator/config/crd/gameplane.local_gameservers.yaml +++ b/operator/config/crd/gameplane.local_gameservers.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.20.1 + controller-gen.kubebuilder.io/version: v0.22.0 name: gameservers.gameplane.local spec: group: gameplane.local diff --git a/operator/config/crd/gameplane.local_gametemplates.yaml b/operator/config/crd/gameplane.local_gametemplates.yaml index f749bccaa..1bab7b727 100644 --- a/operator/config/crd/gameplane.local_gametemplates.yaml +++ b/operator/config/crd/gameplane.local_gametemplates.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.20.1 + controller-gen.kubebuilder.io/version: v0.22.0 name: gametemplates.gameplane.local spec: group: gameplane.local diff --git a/operator/config/crd/gameplane.local_modules.yaml b/operator/config/crd/gameplane.local_modules.yaml index 3f35114f9..9c06cec82 100644 --- a/operator/config/crd/gameplane.local_modules.yaml +++ b/operator/config/crd/gameplane.local_modules.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.20.1 + controller-gen.kubebuilder.io/version: v0.22.0 name: modules.gameplane.local spec: group: gameplane.local diff --git a/operator/config/crd/gameplane.local_modulesources.yaml b/operator/config/crd/gameplane.local_modulesources.yaml index 879583947..a65b30cdd 100644 --- a/operator/config/crd/gameplane.local_modulesources.yaml +++ b/operator/config/crd/gameplane.local_modulesources.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.20.1 + controller-gen.kubebuilder.io/version: v0.22.0 name: modulesources.gameplane.local spec: group: gameplane.local diff --git a/operator/config/crd/gameplane.local_networkcaptures.yaml b/operator/config/crd/gameplane.local_networkcaptures.yaml index 76bc066a7..40bd55ecf 100644 --- a/operator/config/crd/gameplane.local_networkcaptures.yaml +++ b/operator/config/crd/gameplane.local_networkcaptures.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.20.1 + controller-gen.kubebuilder.io/version: v0.22.0 name: networkcaptures.gameplane.local spec: group: gameplane.local @@ -208,22 +208,15 @@ spec: format: int64 type: integer phase: - allOf: - - enum: - - Pending - - Running - - Completed - - Failed - - Expired - - enum: - - Pending - - Running - - Completed - - Failed - - Expired description: |- Phase is the current lifecycle phase of the capture. Valid values: Pending, Running, Completed, Failed, Expired. + enum: + - Pending + - Running + - Completed + - Failed + - Expired type: string startTime: description: |- diff --git a/operator/config/crd/gameplane.local_restores.yaml b/operator/config/crd/gameplane.local_restores.yaml index 7c8a9e530..4984f6817 100644 --- a/operator/config/crd/gameplane.local_restores.yaml +++ b/operator/config/crd/gameplane.local_restores.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - controller-gen.kubebuilder.io/version: v0.20.1 + controller-gen.kubebuilder.io/version: v0.22.0 name: restores.gameplane.local spec: group: gameplane.local From 3248d531814585b803cccd8a96ed9c27a8068224 Mon Sep 17 00:00:00 2001 From: Baylem Date: Fri, 23 Oct 2026 00:19:18 -0400 Subject: [PATCH 08/18] ci: use Go 1.27 compatible golangci-lint Signed-off-by: Baylem --- .devcontainer/post-create.sh | 9 ++++----- .github/workflows/ci.yaml | 6 +++--- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/.devcontainer/post-create.sh b/.devcontainer/post-create.sh index fa606c5b5..10d150b19 100755 --- a/.devcontainer/post-create.sh +++ b/.devcontainer/post-create.sh @@ -52,12 +52,11 @@ if ! command -v oras >/dev/null 2>&1; then fi # ---------- golangci-lint ---------- -# v2 release binaries work with the in-container Go version. The -# .golangci.yml has been migrated from v1 to v2 schema (v2 introduces new -# linters and removes deprecated ones). -GOLANGCI_VERSION="v2.12.2" +# v2.13 is the first release line supporting the workspace's Go 1.27. +# Keep this exact patch version aligned with the CI workflow. +GOLANGCI_VERSION="v2.13.2" GOLANGCI_BIN="/go/bin/golangci-lint" -if ! "$GOLANGCI_BIN" --version 2>/dev/null | grep -qE 'version v?2\.'; then +if ! "$GOLANGCI_BIN" --version 2>/dev/null | grep -qF "version ${GOLANGCI_VERSION#v} "; then log "installing golangci-lint ${GOLANGCI_VERSION}" GOBIN=/go/bin go install \ "github.com/golangci/golangci-lint/v2/cmd/golangci-lint@${GOLANGCI_VERSION}" diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index e62537c9e..e7eb72229 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -802,21 +802,21 @@ jobs: if: matrix.module == 'operator' || matrix.module == 'api' uses: golangci/golangci-lint-action@ba0d7d2ec06a0ea1cb5fa41b2e4a3ab91d21278a # v9.3.0 with: - version: v2.12.2 + version: v2.13.2 working-directory: ${{ matrix.module }} args: --build-tags=envtest - name: lint (test/e2e - e2e build tags) if: matrix.module == 'test/e2e' uses: golangci/golangci-lint-action@ba0d7d2ec06a0ea1cb5fa41b2e4a3ab91d21278a # v9.3.0 with: - version: v2.12.2 + version: v2.13.2 working-directory: ${{ matrix.module }} args: --build-tags=e2e - name: lint (other modules - no build tags) if: matrix.module != 'operator' && matrix.module != 'api' && matrix.module != 'test/e2e' uses: golangci/golangci-lint-action@ba0d7d2ec06a0ea1cb5fa41b2e4a3ab91d21278a # v9.3.0 with: - version: v2.12.2 + version: v2.13.2 working-directory: ${{ matrix.module }} # Merge gate for spec 022: the project default telemetry destination From dcc7161d477c878cf4fcff55f803618782291bd3 Mon Sep 17 00:00:00 2001 From: Baylem Date: Fri, 23 Oct 2026 00:19:17 -0400 Subject: [PATCH 09/18] test: synchronize closed RCON response fixtures Signed-off-by: Baylem --- agent/internal/rcon/nuclearoption_test.go | 29 +++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/agent/internal/rcon/nuclearoption_test.go b/agent/internal/rcon/nuclearoption_test.go index cb46a4e21..ccde83c53 100644 --- a/agent/internal/rcon/nuclearoption_test.go +++ b/agent/internal/rcon/nuclearoption_test.go @@ -1275,18 +1275,47 @@ type pipeFrameCase struct { forbidErr string // substring the error must NOT contain (e.g. a detail suffix) } +type responseDeadlineConn struct { + net.Conn + deadlineSet chan<- struct{} +} + +func (c *responseDeadlineConn) SetDeadline(deadline time.Time) error { + if err := c.Conn.SetDeadline(deadline); err != nil { + return err + } + select { + case c.deadlineSet <- struct{}{}: + default: + } + return nil +} + // runPipeFrameCases sends each case's raw frame back to the client over net.Pipe // and checks the body or the error. func runPipeFrameCases(t *testing.T, tests []pipeFrameCase) { t.Helper() for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { + deadlineSet := make(chan struct{}, 1) client := newPipeClient(t, func(remote net.Conn) { if _, _, err := readNuclearOptionRequest(remote); err != nil { return } + if len(tt.frame) == 0 { + // Exec sets its response deadline after sending the request. + // Closing first can fail SetDeadline instead of the status read. + select { + case <-deadlineSet: + case <-t.Context().Done(): + return + } + } writeFrameSegments(remote, tt.frame, tt.oneByte) }) + if len(tt.frame) == 0 { + client.conn = &responseDeadlineConn{Conn: client.conn, deadlineSet: deadlineSet} + } body, err := client.Exec("test-command") if tt.wantErr != "" { From 59a0f447aa54c64a3e2628c526b9884d8e19c7c6 Mon Sep 17 00:00:00 2001 From: Baylem Date: Fri, 23 Oct 2026 00:26:24 -0400 Subject: [PATCH 10/18] test: clarify closed response deadline synchronization Signed-off-by: Baylem --- agent/internal/rcon/nuclearoption_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/agent/internal/rcon/nuclearoption_test.go b/agent/internal/rcon/nuclearoption_test.go index ccde83c53..d03e76514 100644 --- a/agent/internal/rcon/nuclearoption_test.go +++ b/agent/internal/rcon/nuclearoption_test.go @@ -1303,8 +1303,8 @@ func runPipeFrameCases(t *testing.T, tests []pipeFrameCase) { return } if len(tt.frame) == 0 { - // Exec sets its response deadline after sending the request. - // Closing first can fail SetDeadline instead of the status read. + // Wait until the deadline is installed before closing so the + // intended error comes from the status read, not SetDeadline. select { case <-deadlineSet: case <-t.Context().Done(): From 9522fffacfa507835333138d268490711bca4e91 Mon Sep 17 00:00:00 2001 From: Baylem Date: Fri, 23 Oct 2026 09:45:48 -0400 Subject: [PATCH 11/18] ci: align lint tooling with Go 1.27 export data Signed-off-by: Baylem --- .devcontainer/post-create.sh | 6 ++-- .github/workflows/ci.yaml | 6 ++-- docs/dependencies.md | 2 +- test/e2e/lint-gate-verify.sh | 10 +++--- test/e2e/lint-gate-verify_test.sh | 5 +++ test/e2e/specs.md | 1 + .../.github/workflows/ci.yaml | 6 ++-- .../.github/workflows/ci.yaml | 2 +- .../.github/workflows/ci.yaml | 2 +- .../.github/workflows/ci.yaml | 2 +- .../.github/workflows/ci.yaml | 2 +- .../.github/workflows/ci.yaml | 2 +- .../.github/workflows/ci.yaml | 32 +++++++++++++++++++ .../case-mismatched-version/.golangci.yml | 2 ++ .../case-mismatched-version/go.work.fixture | 17 ++++++++++ .../.github/workflows/ci.yaml | 6 ++-- .../.github/workflows/ci.yaml | 6 ++-- .../.github/workflows/ci.yaml | 8 ++--- .../.github/workflows/ci.yaml | 2 +- .../.github/workflows/ci.yaml | 6 ++-- .../.github/workflows/ci.yaml | 8 ++--- .../.github/workflows/ci.yaml | 8 ++--- 22 files changed, 100 insertions(+), 41 deletions(-) create mode 100644 test/e2e/testdata/lint-gate/case-mismatched-version/.github/workflows/ci.yaml create mode 100644 test/e2e/testdata/lint-gate/case-mismatched-version/.golangci.yml create mode 100644 test/e2e/testdata/lint-gate/case-mismatched-version/go.work.fixture diff --git a/.devcontainer/post-create.sh b/.devcontainer/post-create.sh index 10d150b19..c8b911f55 100755 --- a/.devcontainer/post-create.sh +++ b/.devcontainer/post-create.sh @@ -52,9 +52,9 @@ if ! command -v oras >/dev/null 2>&1; then fi # ---------- golangci-lint ---------- -# v2.13 is the first release line supporting the workspace's Go 1.27. -# Keep this exact patch version aligned with the CI workflow. -GOLANGCI_VERSION="v2.13.2" +# v2.14 includes the importer for Go 1.27.3's export-data version 5. +# Keep this exact version aligned with CI and lint-gate-verify.sh. +GOLANGCI_VERSION="v2.14.0" GOLANGCI_BIN="/go/bin/golangci-lint" if ! "$GOLANGCI_BIN" --version 2>/dev/null | grep -qF "version ${GOLANGCI_VERSION#v} "; then log "installing golangci-lint ${GOLANGCI_VERSION}" diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index e7eb72229..4e51ff11d 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -802,21 +802,21 @@ jobs: if: matrix.module == 'operator' || matrix.module == 'api' uses: golangci/golangci-lint-action@ba0d7d2ec06a0ea1cb5fa41b2e4a3ab91d21278a # v9.3.0 with: - version: v2.13.2 + version: v2.14.0 working-directory: ${{ matrix.module }} args: --build-tags=envtest - name: lint (test/e2e - e2e build tags) if: matrix.module == 'test/e2e' uses: golangci/golangci-lint-action@ba0d7d2ec06a0ea1cb5fa41b2e4a3ab91d21278a # v9.3.0 with: - version: v2.13.2 + version: v2.14.0 working-directory: ${{ matrix.module }} args: --build-tags=e2e - name: lint (other modules - no build tags) if: matrix.module != 'operator' && matrix.module != 'api' && matrix.module != 'test/e2e' uses: golangci/golangci-lint-action@ba0d7d2ec06a0ea1cb5fa41b2e4a3ab91d21278a # v9.3.0 with: - version: v2.13.2 + version: v2.14.0 working-directory: ${{ matrix.module }} # Merge gate for spec 022: the project default telemetry destination diff --git a/docs/dependencies.md b/docs/dependencies.md index 1017fa980..4bfad498f 100644 --- a/docs/dependencies.md +++ b/docs/dependencies.md @@ -407,7 +407,7 @@ These aren't code dependencies of any component; they're invoked by - **`github.com/vladopajic/go-test-coverage/v2@v2.18.9`** — pinned in `Makefile`'s `GO_TEST_COVERAGE_PKG`, run via `go run` (no install step) by `make cover-go-check` and CI's `go` job to enforce each module's `.testcoverage.yml` threshold. - **`github.com/wadey/gocovmerge@v0.0.0-20160331181800-b5bfa59ec0ad`** — pinned in `Makefile`'s `GOCOVMERGE_PKG`, merges each module's `unit.out` + `envtest.out` coverage profiles before the threshold gate. - **`sigs.k8s.io/controller-runtime/tools/setup-envtest@release-0.19`** — installed by `make envtest-bin`; downloads the K8s 1.36.2 envtest binaries (`kube-apiserver`, `etcd`) that back `operator`'s and `api`'s `-tags=envtest` integration tests. -- **`golangci-lint` v2.12.2** — pinned in `.devcontainer/post-create.sh` (built from source, not the release binary, because the release binary is built with an older Go than the repo's `go 1.26.0` toolchain wants). Configured by `.golangci.yml` (bodyclose, errcheck, gosec, govet, ineffassign, staticcheck, unused, misspell, gofmt, goimports, revive, unparam, nilerr, noctx, errorlint, contextcheck). **Not currently invoked by CI** — `ci.yaml`'s `go` job runs only `go vet`, not `golangci-lint run`; `make lint-go` exists for local/devcontainer use and is called out in `docs/contributing.md` as a pre-PR step, but nothing in `.github/workflows/` gates on it. Same for the web job: it runs `npm run build` and `npm run test:cover`, not `npm run lint` — ESLint isn't CI-gated either. +- **`golangci-lint` v2.14.0** — pinned in `.github/workflows/ci.yaml`, `.devcontainer/post-create.sh`, and the exact-version guard in `test/e2e/lint-gate-verify.sh`. Its `golang.org/x/tools v0.50.0` importer supports the export-data version 5 emitted by Go 1.27.3; v2.13.2's importer does not. CI uses the published binary and the devcontainer builds the same release from source. The CI `lint` matrix gates every selected workspace module with `.golangci.yml`, using `envtest` tags for operator/API and `e2e` tags for test/e2e. Update the verifier's valid fixtures with the pin; unsupported or unpinned versions must still fail. Tests and general lint run only in CI under repository policy. - **`oras` CLI ≥ 1.2.0** (pinned `1.2.0` in the devcontainer; `oras-project/setup-oras@v2.0.1` action in `release.yaml`) — `modules/build.sh` pushes each `modules//` directory as an OCI module bundle (`make modules-push`); this is a *different* thing from `operator`'s `oras.land/oras-go/v2` **library** dependency, which is the operator's own pull-side client, not the CLI. - **`cosign` CLI v3.1.2** (`sigstore/cosign-installer@v4.1.2`) — used in `publish-edge.yaml`/`release.yaml` to sign published container images, the Helm chart, and module bundles against `COSIGN_PRIVATE_KEY`, recording each signature in the public Rekor transparency log, and to round-trip-verify against the derived public key before publishing. Signatures use the classic format (`--new-bundle-format=false --use-signing-config=false`), which is what keeps them readable by the operator's `sigstore/cosign/v2` library; cosign's newer Rekor v2 flow additionally requires an RFC3161 timestamp authority that keyed signing cannot satisfy. `modules/build.sh` gates the same behaviour behind its own `--tlog-upload` flag so local and air-gapped authoring stays offline. Distinct from `operator`'s `sigstore/cosign/v2` **library** dependency, which only ever *verifies* (never signs) at reconcile time. - **`kind`** v0.32.0 (devcontainer pin) / `helm/kind-action@v1.14.0` (`install_only: true` in CI) — spins up the ephemeral clusters for `make dev-up`, `make test-e2e`, and every `e2e-*` CI job. diff --git a/test/e2e/lint-gate-verify.sh b/test/e2e/lint-gate-verify.sh index 242e8b9c1..ac5445bec 100755 --- a/test/e2e/lint-gate-verify.sh +++ b/test/e2e/lint-gate-verify.sh @@ -312,7 +312,7 @@ check_step_build_tag() { } # Check if the golangci-lint version is pinned to the expected value. -# Args: expected_version (e.g., "v2.12.2") +# Args: expected_version (e.g., "v2.14.0") # Returns: 0 if version is found and matches, 1 otherwise. On failure, # prints one line to stdout naming what was actually found so the caller # can report the offending value, not just "it didn't match". @@ -325,8 +325,8 @@ check_golangci_version() { # expected_version is interpolated directly into a =~ regex below, so # any regex metacharacter in it (notably "." in a version like - # "v2.12.2") must be escaped first — otherwise "." matches ANY - # character, and a typo like "v2X12Y2" would count as a match. + # "v2.14.0") must be escaped first — otherwise "." matches ANY + # character, and a typo like "v2X14Y0" would count as a match. local expected_version_re="${expected_version//./\\.}" while IFS= read -r line; do @@ -465,7 +465,9 @@ check_r006_no_deferral_comments() { # R-002/R-008/R-009: Operator and API have envtest build tags, test/e2e has e2e # build tags, working-directory matches matrix entry, version is pinned. check_r002_r008_r009_build_tags_and_version() { - local version_required="v2.12.2" + # x/tools v0.50.0 in this release reads Go 1.27.3 export-data version 5. + # Keep the workflow, devcontainer and valid fixtures on this exact pin. + local version_required="v2.14.0" # Check operator has envtest build tag. if ! check_step_build_tag "operator" "envtest"; then diff --git a/test/e2e/lint-gate-verify_test.sh b/test/e2e/lint-gate-verify_test.sh index 0119b061e..bc62380cf 100755 --- a/test/e2e/lint-gate-verify_test.sh +++ b/test/e2e/lint-gate-verify_test.sh @@ -198,6 +198,11 @@ run_test "case-missing-working-directory" "R-008" \ run_test "case-unpinned-version" "R-009" \ "not pinned" +# R-009: One fixed but incompatible version must fail even when other +# steps use the required version. +run_test "case-mismatched-version" "R-009" \ + "R-009:.*found: v2[.]13[.]2" + # R-003: Missing fail-fast: false (strategy omits it entirely) run_test "case-missing-fail-fast" "R-003" \ "missing 'fail-fast: false'" diff --git a/test/e2e/specs.md b/test/e2e/specs.md index 5da77d74d..59f2bcf4b 100644 --- a/test/e2e/specs.md +++ b/test/e2e/specs.md @@ -19,6 +19,7 @@ It was one of three `go.work` modules without CI lint coverage until this featur ## Responsibilities +- Keep the lint workflow, devcontainer and `lint-gate-verify.sh` on the exact `v2.14.0` linter pin, whose importer supports Go 1.27.3 export data. The verifier's valid fixtures use the same pin; regression fixtures reject `latest` and a stale fixed version in any lint step. - The serial `monitoring` bucket installs a checksum-pinned Prometheus Operator and Prometheus in a distinct namespace on the Cilium profile. It discovers real agent/tunnel targets through the documented chart values, verifies cached numeric samples, valid zero, invalid-value omission, no scrape-driven commands, policy revocation/recovery and refusal from an unapproved namespace. It performs no API login and does not opt in external modules. - The optional-components lifecycle's `frp-identity-health` subtest runs two same-template TCP/UDP echo GameServers through one real frps. It asserts backend identity, registration rejection, replacement isolation, invalid credentials, relay outage/recovery and secret-free logs. It shares the lifecycle's controlled fixture and serial bucket; no extra top-level test or login budget is introduced. diff --git a/test/e2e/testdata/lint-gate/case-combined-condition/.github/workflows/ci.yaml b/test/e2e/testdata/lint-gate/case-combined-condition/.github/workflows/ci.yaml index ba9fdbbcc..8d432775f 100644 --- a/test/e2e/testdata/lint-gate/case-combined-condition/.github/workflows/ci.yaml +++ b/test/e2e/testdata/lint-gate/case-combined-condition/.github/workflows/ci.yaml @@ -14,19 +14,19 @@ jobs: if: matrix.module == 'operator' || matrix.module == 'api' uses: golangci/golangci-lint-action@v9 with: - version: v2.12.2 + version: v2.14.0 working-directory: ${{ matrix.module }} args: --build-tags=envtest - name: lint (test/e2e - e2e build tags) if: matrix.module == 'test/e2e' uses: golangci/golangci-lint-action@v9 with: - version: v2.12.2 + version: v2.14.0 working-directory: ${{ matrix.module }} args: --build-tags=e2e - name: lint (other modules - no build tags) if: matrix.module != 'operator' && matrix.module != 'api' && matrix.module != 'test/e2e' uses: golangci/golangci-lint-action@v9 with: - version: v2.12.2 + version: v2.14.0 working-directory: ${{ matrix.module }} diff --git a/test/e2e/testdata/lint-gate/case-continue-on-error/.github/workflows/ci.yaml b/test/e2e/testdata/lint-gate/case-continue-on-error/.github/workflows/ci.yaml index ec40a8ad4..d94ac26d2 100644 --- a/test/e2e/testdata/lint-gate/case-continue-on-error/.github/workflows/ci.yaml +++ b/test/e2e/testdata/lint-gate/case-continue-on-error/.github/workflows/ci.yaml @@ -14,4 +14,4 @@ jobs: continue-on-error: true uses: golangci/golangci-lint-action@v9 with: - version: v2.12.2 + version: v2.14.0 diff --git a/test/e2e/testdata/lint-gate/case-deferral-comment-only/.github/workflows/ci.yaml b/test/e2e/testdata/lint-gate/case-deferral-comment-only/.github/workflows/ci.yaml index 4a170ff0d..a8de8bde0 100644 --- a/test/e2e/testdata/lint-gate/case-deferral-comment-only/.github/workflows/ci.yaml +++ b/test/e2e/testdata/lint-gate/case-deferral-comment-only/.github/workflows/ci.yaml @@ -14,5 +14,5 @@ jobs: - name: lint uses: golangci/golangci-lint-action@v9 with: - version: v2.12.2 + version: v2.14.0 working-directory: ${{ matrix.module }} diff --git a/test/e2e/testdata/lint-gate/case-deferral-comment/.github/workflows/ci.yaml b/test/e2e/testdata/lint-gate/case-deferral-comment/.github/workflows/ci.yaml index 76f43cd62..f0c6939cc 100644 --- a/test/e2e/testdata/lint-gate/case-deferral-comment/.github/workflows/ci.yaml +++ b/test/e2e/testdata/lint-gate/case-deferral-comment/.github/workflows/ci.yaml @@ -13,4 +13,4 @@ jobs: - name: lint (pending cleanup) uses: golangci/golangci-lint-action@v9 with: - version: v2.12.2 + version: v2.14.0 diff --git a/test/e2e/testdata/lint-gate/case-duplicate-module/.github/workflows/ci.yaml b/test/e2e/testdata/lint-gate/case-duplicate-module/.github/workflows/ci.yaml index 3e300f2eb..44cd8ecbc 100644 --- a/test/e2e/testdata/lint-gate/case-duplicate-module/.github/workflows/ci.yaml +++ b/test/e2e/testdata/lint-gate/case-duplicate-module/.github/workflows/ci.yaml @@ -13,4 +13,4 @@ jobs: - name: lint uses: golangci/golangci-lint-action@v9 with: - version: v2.12.2 + version: v2.14.0 diff --git a/test/e2e/testdata/lint-gate/case-extra-module/.github/workflows/ci.yaml b/test/e2e/testdata/lint-gate/case-extra-module/.github/workflows/ci.yaml index eead80902..c5320feae 100644 --- a/test/e2e/testdata/lint-gate/case-extra-module/.github/workflows/ci.yaml +++ b/test/e2e/testdata/lint-gate/case-extra-module/.github/workflows/ci.yaml @@ -13,4 +13,4 @@ jobs: - name: lint uses: golangci/golangci-lint-action@v9 with: - version: v2.12.2 + version: v2.14.0 diff --git a/test/e2e/testdata/lint-gate/case-mismatched-version/.github/workflows/ci.yaml b/test/e2e/testdata/lint-gate/case-mismatched-version/.github/workflows/ci.yaml new file mode 100644 index 000000000..58ca405c9 --- /dev/null +++ b/test/e2e/testdata/lint-gate/case-mismatched-version/.github/workflows/ci.yaml @@ -0,0 +1,32 @@ +name: ci +on: push +jobs: + lint: + name: lint (${{ matrix.module }}) + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + module: [netguard, gameaction, gameproto, operator, agent, api, sentinel, audit-syslog-bridge, telemetry-receiver, mcp-server, svcutil, tunnel, test/e2e] + steps: + - uses: actions/checkout@v7 + - name: lint (operator/api - envtest build tags) + if: matrix.module == 'operator' || matrix.module == 'api' + uses: golangci/golangci-lint-action@v9 + with: + version: v2.13.2 + working-directory: ${{ matrix.module }} + args: --build-tags=envtest + - name: lint (test/e2e - e2e build tags) + if: matrix.module == 'test/e2e' + uses: golangci/golangci-lint-action@v9 + with: + version: v2.14.0 + working-directory: ${{ matrix.module }} + args: --build-tags=e2e + - name: lint (other modules - no build tags) + if: matrix.module != 'operator' && matrix.module != 'api' && matrix.module != 'test/e2e' + uses: golangci/golangci-lint-action@v9 + with: + version: v2.14.0 + working-directory: ${{ matrix.module }} diff --git a/test/e2e/testdata/lint-gate/case-mismatched-version/.golangci.yml b/test/e2e/testdata/lint-gate/case-mismatched-version/.golangci.yml new file mode 100644 index 000000000..600bef78e --- /dev/null +++ b/test/e2e/testdata/lint-gate/case-mismatched-version/.golangci.yml @@ -0,0 +1,2 @@ +run: + timeout: 5m diff --git a/test/e2e/testdata/lint-gate/case-mismatched-version/go.work.fixture b/test/e2e/testdata/lint-gate/case-mismatched-version/go.work.fixture new file mode 100644 index 000000000..8f757dec4 --- /dev/null +++ b/test/e2e/testdata/lint-gate/case-mismatched-version/go.work.fixture @@ -0,0 +1,17 @@ +go 1.26.0 + +use ( + ./agent + ./api + ./audit-syslog-bridge + ./gameaction + ./gameproto + ./mcp-server + ./netguard + ./operator + ./sentinel + ./svcutil + ./telemetry-receiver + ./test/e2e + ./tunnel +) diff --git a/test/e2e/testdata/lint-gate/case-missing-api-tag/.github/workflows/ci.yaml b/test/e2e/testdata/lint-gate/case-missing-api-tag/.github/workflows/ci.yaml index e6394b285..abb1755f6 100644 --- a/test/e2e/testdata/lint-gate/case-missing-api-tag/.github/workflows/ci.yaml +++ b/test/e2e/testdata/lint-gate/case-missing-api-tag/.github/workflows/ci.yaml @@ -14,19 +14,19 @@ jobs: if: matrix.module == "operator" uses: golangci/golangci-lint-action@v9 with: - version: v2.12.2 + version: v2.14.0 working-directory: ${{ matrix.module }} args: --build-tags=envtest - name: lint (test/e2e - e2e build tags) if: matrix.module == "test/e2e" uses: golangci/golangci-lint-action@v9 with: - version: v2.12.2 + version: v2.14.0 working-directory: ${{ matrix.module }} args: --build-tags=e2e - name: lint (other modules - no build tags) if: matrix.module != "operator" && matrix.module != "api" && matrix.module != "test/e2e" uses: golangci/golangci-lint-action@v9 with: - version: v2.12.2 + version: v2.14.0 working-directory: ${{ matrix.module }} diff --git a/test/e2e/testdata/lint-gate/case-missing-e2e-tag/.github/workflows/ci.yaml b/test/e2e/testdata/lint-gate/case-missing-e2e-tag/.github/workflows/ci.yaml index b7cbc7a7e..c11e684af 100644 --- a/test/e2e/testdata/lint-gate/case-missing-e2e-tag/.github/workflows/ci.yaml +++ b/test/e2e/testdata/lint-gate/case-missing-e2e-tag/.github/workflows/ci.yaml @@ -14,19 +14,19 @@ jobs: if: matrix.module == "operator" uses: golangci/golangci-lint-action@v9 with: - version: v2.12.2 + version: v2.14.0 working-directory: ${{ matrix.module }} args: --build-tags=envtest - name: lint (api - envtest build tags) if: matrix.module == "api" uses: golangci/golangci-lint-action@v9 with: - version: v2.12.2 + version: v2.14.0 working-directory: ${{ matrix.module }} args: --build-tags=envtest - name: lint (other modules - no build tags) if: matrix.module != "operator" && matrix.module != "api" && matrix.module != "test/e2e" uses: golangci/golangci-lint-action@v9 with: - version: v2.12.2 + version: v2.14.0 working-directory: ${{ matrix.module }} diff --git a/test/e2e/testdata/lint-gate/case-missing-fail-fast/.github/workflows/ci.yaml b/test/e2e/testdata/lint-gate/case-missing-fail-fast/.github/workflows/ci.yaml index e06330716..9f1f321f2 100644 --- a/test/e2e/testdata/lint-gate/case-missing-fail-fast/.github/workflows/ci.yaml +++ b/test/e2e/testdata/lint-gate/case-missing-fail-fast/.github/workflows/ci.yaml @@ -13,26 +13,26 @@ jobs: if: matrix.module == "operator" uses: golangci/golangci-lint-action@v9 with: - version: v2.12.2 + version: v2.14.0 working-directory: ${{ matrix.module }} args: --build-tags=envtest - name: lint (api - envtest build tags) if: matrix.module == "api" uses: golangci/golangci-lint-action@v9 with: - version: v2.12.2 + version: v2.14.0 working-directory: ${{ matrix.module }} args: --build-tags=envtest - name: lint (test/e2e - e2e build tags) if: matrix.module == "test/e2e" uses: golangci/golangci-lint-action@v9 with: - version: v2.12.2 + version: v2.14.0 working-directory: ${{ matrix.module }} args: --build-tags=e2e - name: lint (other modules - no build tags) if: matrix.module != "operator" && matrix.module != "api" && matrix.module != "test/e2e" uses: golangci/golangci-lint-action@v9 with: - version: v2.12.2 + version: v2.14.0 working-directory: ${{ matrix.module }} diff --git a/test/e2e/testdata/lint-gate/case-missing-module/.github/workflows/ci.yaml b/test/e2e/testdata/lint-gate/case-missing-module/.github/workflows/ci.yaml index 9b31ad9f0..cd6dd92c4 100644 --- a/test/e2e/testdata/lint-gate/case-missing-module/.github/workflows/ci.yaml +++ b/test/e2e/testdata/lint-gate/case-missing-module/.github/workflows/ci.yaml @@ -13,4 +13,4 @@ jobs: - name: lint uses: golangci/golangci-lint-action@v9 with: - version: v2.12.2 + version: v2.14.0 diff --git a/test/e2e/testdata/lint-gate/case-missing-operator-tag/.github/workflows/ci.yaml b/test/e2e/testdata/lint-gate/case-missing-operator-tag/.github/workflows/ci.yaml index 40aaf3f77..a689993d9 100644 --- a/test/e2e/testdata/lint-gate/case-missing-operator-tag/.github/workflows/ci.yaml +++ b/test/e2e/testdata/lint-gate/case-missing-operator-tag/.github/workflows/ci.yaml @@ -14,19 +14,19 @@ jobs: if: matrix.module == "api" uses: golangci/golangci-lint-action@v9 with: - version: v2.12.2 + version: v2.14.0 working-directory: ${{ matrix.module }} args: --build-tags=envtest - name: lint (test/e2e - e2e build tags) if: matrix.module == "test/e2e" uses: golangci/golangci-lint-action@v9 with: - version: v2.12.2 + version: v2.14.0 working-directory: ${{ matrix.module }} args: --build-tags=e2e - name: lint (other modules - no build tags) if: matrix.module != "operator" && matrix.module != "api" && matrix.module != "test/e2e" uses: golangci/golangci-lint-action@v9 with: - version: v2.12.2 + version: v2.14.0 working-directory: ${{ matrix.module }} diff --git a/test/e2e/testdata/lint-gate/case-missing-working-directory/.github/workflows/ci.yaml b/test/e2e/testdata/lint-gate/case-missing-working-directory/.github/workflows/ci.yaml index 82c7cc599..46960b4a2 100644 --- a/test/e2e/testdata/lint-gate/case-missing-working-directory/.github/workflows/ci.yaml +++ b/test/e2e/testdata/lint-gate/case-missing-working-directory/.github/workflows/ci.yaml @@ -14,22 +14,22 @@ jobs: if: matrix.module == "operator" uses: golangci/golangci-lint-action@v9 with: - version: v2.12.2 + version: v2.14.0 args: --build-tags=envtest - name: lint (api - envtest build tags) if: matrix.module == "api" uses: golangci/golangci-lint-action@v9 with: - version: v2.12.2 + version: v2.14.0 args: --build-tags=envtest - name: lint (test/e2e - e2e build tags) if: matrix.module == "test/e2e" uses: golangci/golangci-lint-action@v9 with: - version: v2.12.2 + version: v2.14.0 args: --build-tags=e2e - name: lint (other modules - no build tags) if: matrix.module != "operator" && matrix.module != "api" && matrix.module != "test/e2e" uses: golangci/golangci-lint-action@v9 with: - version: v2.12.2 + version: v2.14.0 diff --git a/test/e2e/testdata/lint-gate/case-no-golangci-yml/.github/workflows/ci.yaml b/test/e2e/testdata/lint-gate/case-no-golangci-yml/.github/workflows/ci.yaml index 94a6f2695..615214ea6 100644 --- a/test/e2e/testdata/lint-gate/case-no-golangci-yml/.github/workflows/ci.yaml +++ b/test/e2e/testdata/lint-gate/case-no-golangci-yml/.github/workflows/ci.yaml @@ -14,26 +14,26 @@ jobs: if: matrix.module == "operator" uses: golangci/golangci-lint-action@v9 with: - version: v2.12.2 + version: v2.14.0 working-directory: ${{ matrix.module }} args: --build-tags=envtest - name: lint (api - envtest build tags) if: matrix.module == "api" uses: golangci/golangci-lint-action@v9 with: - version: v2.12.2 + version: v2.14.0 working-directory: ${{ matrix.module }} args: --build-tags=envtest - name: lint (test/e2e - e2e build tags) if: matrix.module == "test/e2e" uses: golangci/golangci-lint-action@v9 with: - version: v2.12.2 + version: v2.14.0 working-directory: ${{ matrix.module }} args: --build-tags=e2e - name: lint (other modules - no build tags) if: matrix.module != "operator" && matrix.module != "api" && matrix.module != "test/e2e" uses: golangci/golangci-lint-action@v9 with: - version: v2.12.2 + version: v2.14.0 working-directory: ${{ matrix.module }} From 9197f6c7bb75b2267705a4396dbc92a9c807782c Mon Sep 17 00:00:00 2001 From: Baylem Date: Fri, 23 Oct 2026 13:36:28 -0400 Subject: [PATCH 12/18] docs: classify lint tool dependency versions Signed-off-by: Baylem --- docs/dependencies.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/dependencies.md b/docs/dependencies.md index 4bfad498f..ac72ac668 100644 --- a/docs/dependencies.md +++ b/docs/dependencies.md @@ -407,7 +407,7 @@ These aren't code dependencies of any component; they're invoked by - **`github.com/vladopajic/go-test-coverage/v2@v2.18.9`** — pinned in `Makefile`'s `GO_TEST_COVERAGE_PKG`, run via `go run` (no install step) by `make cover-go-check` and CI's `go` job to enforce each module's `.testcoverage.yml` threshold. - **`github.com/wadey/gocovmerge@v0.0.0-20160331181800-b5bfa59ec0ad`** — pinned in `Makefile`'s `GOCOVMERGE_PKG`, merges each module's `unit.out` + `envtest.out` coverage profiles before the threshold gate. - **`sigs.k8s.io/controller-runtime/tools/setup-envtest@release-0.19`** — installed by `make envtest-bin`; downloads the K8s 1.36.2 envtest binaries (`kube-apiserver`, `etcd`) that back `operator`'s and `api`'s `-tags=envtest` integration tests. -- **`golangci-lint` v2.14.0** — pinned in `.github/workflows/ci.yaml`, `.devcontainer/post-create.sh`, and the exact-version guard in `test/e2e/lint-gate-verify.sh`. Its `golang.org/x/tools v0.50.0` importer supports the export-data version 5 emitted by Go 1.27.3; v2.13.2's importer does not. CI uses the published binary and the devcontainer builds the same release from source. The CI `lint` matrix gates every selected workspace module with `.golangci.yml`, using `envtest` tags for operator/API and `e2e` tags for test/e2e. Update the verifier's valid fixtures with the pin; unsupported or unpinned versions must still fail. Tests and general lint run only in CI under repository policy. +- **`golangci-lint` v2.14.0** — pinned in `.github/workflows/ci.yaml`, `.devcontainer/post-create.sh`, and the exact-version guard in `test/e2e/lint-gate-verify.sh`. Its `golang.org/x/tools v0.50.0` importer supports the export-data version 5 emitted by Go 1.27.3; v2.13.2's importer does not. CI uses the published binary and the devcontainer builds the same release from source. The CI `lint` matrix gates every selected workspace module with `.golangci.yml`, using `envtest` tags for operator/API and `e2e` tags for test/e2e. Update the verifier's valid fixtures with the pin; unsupported or unpinned versions must still fail. Tests and general lint run only in CI under repository policy. - **`oras` CLI ≥ 1.2.0** (pinned `1.2.0` in the devcontainer; `oras-project/setup-oras@v2.0.1` action in `release.yaml`) — `modules/build.sh` pushes each `modules//` directory as an OCI module bundle (`make modules-push`); this is a *different* thing from `operator`'s `oras.land/oras-go/v2` **library** dependency, which is the operator's own pull-side client, not the CLI. - **`cosign` CLI v3.1.2** (`sigstore/cosign-installer@v4.1.2`) — used in `publish-edge.yaml`/`release.yaml` to sign published container images, the Helm chart, and module bundles against `COSIGN_PRIVATE_KEY`, recording each signature in the public Rekor transparency log, and to round-trip-verify against the derived public key before publishing. Signatures use the classic format (`--new-bundle-format=false --use-signing-config=false`), which is what keeps them readable by the operator's `sigstore/cosign/v2` library; cosign's newer Rekor v2 flow additionally requires an RFC3161 timestamp authority that keyed signing cannot satisfy. `modules/build.sh` gates the same behaviour behind its own `--tlog-upload` flag so local and air-gapped authoring stays offline. Distinct from `operator`'s `sigstore/cosign/v2` **library** dependency, which only ever *verifies* (never signs) at reconcile time. - **`kind`** v0.32.0 (devcontainer pin) / `helm/kind-action@v1.14.0` (`install_only: true` in CI) — spins up the ephemeral clusters for `make dev-up`, `make test-e2e`, and every `e2e-*` CI job. From 71cd384b5e4cd5a628ae2038c48478871faf3be9 Mon Sep 17 00:00:00 2001 From: Baylem Date: Fri, 23 Oct 2026 13:40:04 -0400 Subject: [PATCH 13/18] fix(operator): use explicit waits for expected reconcile retries Signed-off-by: Baylem --- operator/internal/controller/conflict.go | 10 ++- operator/internal/controller/conflict_test.go | 8 ++- .../controller/gameserver_controller.go | 6 +- .../controller/gameserver_wipe_guard_test.go | 7 ++- .../internal/controller/module_controller.go | 12 +++- .../controller/module_finalizer_poll_test.go | 61 +++++++++++++++++++ .../controller/module_markpending_test.go | 9 ++- .../internal/controller/restore_controller.go | 8 +-- .../controller/restore_fence_requeue_test.go | 18 +++++- .../controller/restore_volumesnapshot.go | 2 +- operator/specs.md | 10 +++ 11 files changed, 132 insertions(+), 19 deletions(-) create mode 100644 operator/internal/controller/module_finalizer_poll_test.go diff --git a/operator/internal/controller/conflict.go b/operator/internal/controller/conflict.go index 988c7d2fa..0559053f1 100644 --- a/operator/internal/controller/conflict.go +++ b/operator/internal/controller/conflict.go @@ -1,17 +1,25 @@ package controller import ( + "time" + apierrors "k8s.io/apimachinery/pkg/api/errors" ctrl "sigs.k8s.io/controller-runtime" ) +// reconcileRetryInterval yields after expected write races or persisted +// lifecycle transitions so the next pass can read fresh state. These are not +// failures: real errors retain controller-runtime's exponential backoff. +// A positive delay avoids both dropping retries and spinning on a stale cache. +const reconcileRetryInterval = time.Second + // requeueOnConflict turns an optimistic-concurrency conflict ("the object has // been modified") into a quiet requeue. Conflicts are an expected, transient // race between reconcilers and the API server cache; returning them as errors // logs a spurious ERROR on every race. Any other result passes through. func requeueOnConflict(res ctrl.Result, err error) (ctrl.Result, error) { if apierrors.IsConflict(err) { - return ctrl.Result{Requeue: true}, nil + return ctrl.Result{RequeueAfter: reconcileRetryInterval}, nil } return res, err } diff --git a/operator/internal/controller/conflict_test.go b/operator/internal/controller/conflict_test.go index 73adf02e3..5b8d30e42 100644 --- a/operator/internal/controller/conflict_test.go +++ b/operator/internal/controller/conflict_test.go @@ -24,10 +24,11 @@ func TestRequeueOnConflict(t *testing.T) { wantRes ctrl.Result wantErr bool }{ - {name: "conflict becomes quiet requeue", res: keep, err: conflict, wantRes: ctrl.Result{Requeue: true}}, - {name: "wrapped conflict becomes quiet requeue", res: keep, err: fmt.Errorf("update status: %w", conflict), wantRes: ctrl.Result{Requeue: true}}, + {name: "conflict becomes quiet paced retry", res: keep, err: conflict, wantRes: ctrl.Result{RequeueAfter: time.Second}}, + {name: "wrapped conflict becomes quiet paced retry", res: keep, err: fmt.Errorf("update status: %w", conflict), wantRes: ctrl.Result{RequeueAfter: time.Second}}, {name: "other error passes through", res: keep, err: other, wantRes: keep, wantErr: true}, {name: "nil error passes result through", res: keep, err: nil, wantRes: keep}, + {name: "settled reconcile stays settled", res: ctrl.Result{}}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { @@ -35,6 +36,9 @@ func TestRequeueOnConflict(t *testing.T) { if (err != nil) != tc.wantErr { t.Fatalf("err = %v, wantErr %v", err, tc.wantErr) } + if tc.wantErr && !errors.Is(err, tc.err) { + t.Fatalf("non-conflict error changed: %v, want %v", err, tc.err) + } if got != tc.wantRes { t.Errorf("result = %+v, want %+v", got, tc.wantRes) } diff --git a/operator/internal/controller/gameserver_controller.go b/operator/internal/controller/gameserver_controller.go index 3389a4c2b..9338ac693 100644 --- a/operator/internal/controller/gameserver_controller.go +++ b/operator/internal/controller/gameserver_controller.go @@ -252,7 +252,7 @@ func (r *GameServerReconciler) Reconcile(ctx context.Context, req ctrl.Request) if changed, err := r.ensureWipeGuard(ctx, &gs, &wipeState); err != nil { return requeueOnConflict(ctrl.Result{}, err) } else if changed { - return ctrl.Result{Requeue: true}, nil + return ctrl.Result{RequeueAfter: reconcileRetryInterval}, nil } // Resolve the template this GameServer points at. Templates are @@ -397,7 +397,7 @@ func (r *GameServerReconciler) Reconcile(ctx context.Context, req ctrl.Request) } if err := r.reconcileStatefulSet(ctx, &gs, &tmpl, ver, mc, replicas, &wipeState); err != nil { if apierrors.IsConflict(err) { - return ctrl.Result{Requeue: true}, nil + return requeueOnConflict(ctrl.Result{}, err) } logger.Error(err, "reconcile StatefulSet") return ctrl.Result{}, err @@ -425,7 +425,7 @@ func (r *GameServerReconciler) Reconcile(ctx context.Context, req ctrl.Request) } if err := wipeErr; err != nil { if apierrors.IsConflict(err) { - return ctrl.Result{Requeue: true}, nil + return requeueOnConflict(ctrl.Result{}, err) } logger.Error(err, "reconcile data wipe") return ctrl.Result{}, err diff --git a/operator/internal/controller/gameserver_wipe_guard_test.go b/operator/internal/controller/gameserver_wipe_guard_test.go index 1e1fca1ba..330786b28 100644 --- a/operator/internal/controller/gameserver_wipe_guard_test.go +++ b/operator/internal/controller/gameserver_wipe_guard_test.go @@ -4,12 +4,14 @@ import ( "context" "fmt" "testing" + "time" appsv1 "k8s.io/api/apps/v1" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" "sigs.k8s.io/controller-runtime/pkg/client/interceptor" @@ -52,7 +54,10 @@ func TestWipePendingStartAcquiresGuardBeforeLaunching(t *testing.T) { gs := wipeGameServer(false, "current") c := fake.NewClientBuilder().WithScheme(s).WithObjects(gs, wipeStoppedSet(gs)).Build() r := &GameServerReconciler{Client: c, APIReader: c, Scheme: s} - wipePasses(t, r, gs, 1) + result, err := r.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(gs)}) + if err != nil || result != (ctrl.Result{RequeueAfter: time.Second}) { + t.Fatalf("guard acquisition must yield for a fresh reconcile: result=%+v err=%v", result, err) + } current := wipeCurrent(t, c, gs) if current.Annotations[wipeGuardAnnotation] != "current" || !current.Spec.Suspend { t.Fatal("direct start bypassed durable wipe acquisition") diff --git a/operator/internal/controller/module_controller.go b/operator/internal/controller/module_controller.go index c3c54b48a..975cfe5d9 100644 --- a/operator/internal/controller/module_controller.go +++ b/operator/internal/controller/module_controller.go @@ -36,6 +36,14 @@ import ( // refresh floor instead of hot-looping (F-258). const moduleFailedRetryInterval = minRefreshInterval +// Pending catalog/version state is refreshed by the ModuleSource watch; its +// refresh floor is only a polling backstop. Finalization has no GameServer +// watch, so check references every five seconds until removal is safe. +const ( + modulePendingRetryInterval = minRefreshInterval + moduleFinalizerPollInterval = 5 * time.Second +) + // ModuleReconciler materializes Module CRs into GameTemplate CRs. The // produced GameTemplate carries an OwnerReference back to the Module so // uninstall = `kubectl delete module ` and the K8s GC reaps the @@ -368,7 +376,7 @@ func (r *ModuleReconciler) finalize(ctx context.Context, mod *gameplanev1alpha1. } // Don't release the finalizer; requeue so we re-check after the // user removes the GameServers. - return ctrl.Result{Requeue: true}, nil + return ctrl.Result{RequeueAfter: moduleFinalizerPollInterval}, nil } // Delete the materialized GameTemplate. SetControllerReference would @@ -423,7 +431,7 @@ func (r *ModuleReconciler) markPending(ctx context.Context, mod *gameplanev1alph if uerr := r.updateStatusIfChanged(ctx, mod, before); uerr != nil { return ctrl.Result{}, uerr } - return ctrl.Result{Requeue: true}, nil + return ctrl.Result{RequeueAfter: modulePendingRetryInterval}, nil } func (r *ModuleReconciler) markFailed(ctx context.Context, mod *gameplanev1alpha1.Module, reason string, err error) (ctrl.Result, error) { diff --git a/operator/internal/controller/module_finalizer_poll_test.go b/operator/internal/controller/module_finalizer_poll_test.go new file mode 100644 index 000000000..f9fe9f23f --- /dev/null +++ b/operator/internal/controller/module_finalizer_poll_test.go @@ -0,0 +1,61 @@ +package controller + +import ( + "testing" + "time" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" +) + +func TestModuleFinalizerPollRetainsTemplateUntilUnreferenced(t *testing.T) { + t.Parallel() + scheme := scrapeScheme(t) + mod := &gameplanev1alpha1.Module{ObjectMeta: metav1.ObjectMeta{Name: "module", UID: "module-uid", Finalizers: []string{gameplanev1alpha1.ModuleFinalizer}}} + tmpl := &gameplanev1alpha1.GameTemplate{ObjectMeta: metav1.ObjectMeta{Name: mod.Name}} + if err := controllerutil.SetControllerReference(mod, tmpl, scheme); err != nil { + t.Fatal(err) + } + gs := &gameplanev1alpha1.GameServer{ + ObjectMeta: metav1.ObjectMeta{Name: "server", Namespace: "ns"}, + Spec: gameplanev1alpha1.GameServerSpec{TemplateRef: gameplanev1alpha1.GameTemplateRef{Name: tmpl.Name}}, + } + c := fake.NewClientBuilder().WithScheme(scheme).WithObjects(mod, tmpl, gs). + WithStatusSubresource(&gameplanev1alpha1.Module{}).Build() + r := &ModuleReconciler{Client: c, Scheme: scheme} + if err := c.Delete(t.Context(), mod); err != nil { + t.Fatal(err) + } + if err := c.Get(t.Context(), client.ObjectKeyFromObject(mod), mod); err != nil { + t.Fatal(err) + } + result, err := r.finalize(t.Context(), mod) + if err != nil || result != (ctrl.Result{RequeueAfter: 5 * time.Second}) { + t.Fatalf("in-use finalizer must poll: result=%+v err=%v", result, err) + } + if !controllerutil.ContainsFinalizer(mod, gameplanev1alpha1.ModuleFinalizer) { + t.Fatal("finalizer released while template is referenced") + } + if err := c.Get(t.Context(), client.ObjectKeyFromObject(tmpl), tmpl); err != nil { + t.Fatalf("referenced template removed: %v", err) + } + if err := c.Delete(t.Context(), gs); err != nil { + t.Fatal(err) + } + result, err = r.finalize(t.Context(), mod) + if err != nil || !result.IsZero() { + t.Fatalf("unreferenced module must finish: result=%+v err=%v", result, err) + } + if err := c.Get(t.Context(), client.ObjectKeyFromObject(tmpl), tmpl); !apierrors.IsNotFound(err) { + t.Fatalf("owned template survived finalization: %v", err) + } + if err := c.Get(t.Context(), client.ObjectKeyFromObject(mod), mod); !apierrors.IsNotFound(err) { + t.Fatalf("Module finalizer was not released: %v", err) + } +} diff --git a/operator/internal/controller/module_markpending_test.go b/operator/internal/controller/module_markpending_test.go index 15f6da4fc..6f408dcb9 100644 --- a/operator/internal/controller/module_markpending_test.go +++ b/operator/internal/controller/module_markpending_test.go @@ -4,6 +4,7 @@ import ( "context" "errors" "testing" + "time" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -38,10 +39,14 @@ func TestMarkPending_ClearsStalePullingCondition(t *testing.T) { Build() r := &ModuleReconciler{Client: cl} - if _, err := r.markPending(context.Background(), mod, "WaitingForCatalog", - errors.New("source has not yet indexed module")); err != nil { + result, err := r.markPending(context.Background(), mod, "WaitingForCatalog", + errors.New("source has not yet indexed module")) + if err != nil { t.Fatalf("markPending: %v", err) } + if result.RequeueAfter != time.Minute { + t.Fatalf("catalog retry = %s, want the one-minute refresh floor", result.RequeueAfter) + } got := &gameplanev1alpha1.Module{} if err := r.Get(context.Background(), types.NamespacedName{Name: "mc"}, got); err != nil { diff --git a/operator/internal/controller/restore_controller.go b/operator/internal/controller/restore_controller.go index fe46011ef..fe2692cd8 100644 --- a/operator/internal/controller/restore_controller.go +++ b/operator/internal/controller/restore_controller.go @@ -113,7 +113,7 @@ func (r *RestoreReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct if err := r.Status().Update(ctx, &rs); err != nil { return ctrl.Result{}, err } - return ctrl.Result{Requeue: true}, nil + return ctrl.Result{RequeueAfter: reconcileRetryInterval}, nil } // Re-resolve the source Backup so we know its strategy on every pass @@ -147,7 +147,7 @@ func (r *RestoreReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct return ctrl.Result{}, err } if changed { - return ctrl.Result{Requeue: true}, nil + return ctrl.Result{RequeueAfter: reconcileRetryInterval}, nil } target, err := parseRestoreIdentity(rs.Annotations[restoreTargetAnnotation]) if err != nil { @@ -196,7 +196,7 @@ func (r *RestoreReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct if err := r.Status().Update(ctx, &rs); err != nil { return ctrl.Result{}, err } - return ctrl.Result{Requeue: true}, nil + return ctrl.Result{RequeueAfter: reconcileRetryInterval}, nil } // src (the source Backup, with RepoRef for the Job env) was resolved @@ -231,7 +231,7 @@ func (r *RestoreReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct } if err := r.Create(ctx, job); err != nil { if apierrors.IsAlreadyExists(err) { - return ctrl.Result{Requeue: true}, nil + return ctrl.Result{RequeueAfter: reconcileRetryInterval}, nil } return ctrl.Result{}, err } diff --git a/operator/internal/controller/restore_fence_requeue_test.go b/operator/internal/controller/restore_fence_requeue_test.go index 9371c7b4a..acf997a0a 100644 --- a/operator/internal/controller/restore_fence_requeue_test.go +++ b/operator/internal/controller/restore_fence_requeue_test.go @@ -18,7 +18,11 @@ func TestRestoreFenceWriteRequeuesImmediatelyAndRechecksWriters(t *testing.T) { for _, phase := range []gameplanev1alpha1.RestorePhase{gameplanev1alpha1.RestorePhaseSuspending, gameplanev1alpha1.RestorePhaseRunning} { t.Run(string(phase), func(t *testing.T) { r, _, rs := restoreLifecycleFixture(t) - lifecyclePasses(t, r, "first", 1) // Persist target identity first. + request := ctrl.Request{NamespacedName: client.ObjectKeyFromObject(rs)} + result, err := r.Reconcile(t.Context(), request) // Persist target identity first. + if err != nil || result != (ctrl.Result{RequeueAfter: time.Second}) { + t.Fatalf("identity binding must yield for a fresh reconcile: result=%+v err=%v", result, err) + } if err := r.Get(t.Context(), client.ObjectKeyFromObject(rs), rs); err != nil { t.Fatal(err) } @@ -26,8 +30,7 @@ func TestRestoreFenceWriteRequeuesImmediatelyAndRechecksWriters(t *testing.T) { if err := r.Status().Update(t.Context(), rs); err != nil { t.Fatal(err) } - request := ctrl.Request{NamespacedName: client.ObjectKeyFromObject(rs)} - result, err := r.Reconcile(t.Context(), request) + result, err = r.Reconcile(t.Context(), request) if err != nil { t.Fatal(err) } @@ -69,6 +72,15 @@ func TestRestoreFenceWriteRequeuesImmediatelyAndRechecksWriters(t *testing.T) { if err := r.Delete(t.Context(), writer); err != nil { t.Fatal(err) } + if phase == gameplanev1alpha1.RestorePhaseSuspending { + result, err = r.Reconcile(t.Context(), request) + if err != nil || result != (ctrl.Result{RequeueAfter: time.Second}) { + t.Fatalf("Running transition must yield before Job creation: result=%+v err=%v", result, err) + } + if len(lifecycleJobs(t, r)) != 0 { + t.Fatal("restore Job created in the Running transition pass") + } + } lifecyclePasses(t, r, "first", 4) jobs = lifecycleJobs(t, r) if len(jobs) != 1 || jobs[0].Name != "restore-first" { diff --git a/operator/internal/controller/restore_volumesnapshot.go b/operator/internal/controller/restore_volumesnapshot.go index c2d1819f5..440356c0b 100644 --- a/operator/internal/controller/restore_volumesnapshot.go +++ b/operator/internal/controller/restore_volumesnapshot.go @@ -157,7 +157,7 @@ func (r *RestoreReconciler) reconcileVolumeSnapshotRestore( if err := r.Create(ctx, created); err != nil { if apierrors.IsAlreadyExists(err) { // Lost a create race; the next pass finds it via the annotation. - return ctrl.Result{Requeue: true}, nil + return ctrl.Result{RequeueAfter: reconcileRetryInterval}, nil } // Route non-AlreadyExists errors through the same deadline-aware path // used for reference-copy failures. This ensures transient errors do not diff --git a/operator/specs.md b/operator/specs.md index dede0e559..9113c443c 100644 --- a/operator/specs.md +++ b/operator/specs.md @@ -10,6 +10,16 @@ The operator is a Kubernetes controller-runtime-based process that reconciles Ga ## Responsibilities +Expected write conflicts, create races and persisted lifecycle transitions +request a fresh reconcile after one second without reporting an error. Module +catalog/version waits use the existing one-minute source refresh floor as a +backstop; ModuleSource events can reconcile them sooner. Module deletion polls +every five seconds while a GameServer references its template and retains the +finalizer until those references are gone. These explicit waits use +`RequeueAfter`; genuine errors retain controller-runtime's exponential backoff. +Existing restore fence-write requeues and writer-drain polling remain unchanged, +and every resumed restore pass still verifies the workload fence and live writers. + - Preserve explicit optional numeric performance declarations in template capabilities passed to the agent; CRD validation restricts signal/unit pairs and finite decimal thresholds. Tunnel containers name their existing 8091 listener `relay-metrics` for PodMonitor discovery; health, resources and provider identity policies remain unchanged. - **Relay health:** frp identities use GameServer UID without changing advertised port names. frp/Playit Deployments use one replica, Recreate, a wrapper startup probe and a provider readiness probe on TCP 8091. `TunnelReady` requires the current observed generation and updated/ready replicas with the new probe; legacy unprobed Pods cannot report success. Playit v1.0.10's cached IPC assignment cannot confirm a relay connection, so it reports unknown/not ready rather than optimistic success. Preserve `TunnelResources` defaults and override removal. See `docs/relay-health.md` for health bounds, migration and the provider limitation. From 5042ab115f9b1fe1bcd7e433064345f869257ab4 Mon Sep 17 00:00:00 2001 From: Baylem Date: Fri, 23 Oct 2026 13:42:13 -0400 Subject: [PATCH 14/18] fix(agent): confine mod deletion to an open directory root Signed-off-by: Baylem --- agent/internal/mods/manifest.go | 50 +++++- agent/internal/mods/mods.go | 43 ++--- agent/internal/mods/mods_test.go | 10 +- agent/internal/mods/removal.go | 75 +++++++++ agent/internal/mods/removal_test.go | 237 ++++++++++++++++++++++++++++ agent/specs.md | 2 +- 6 files changed, 380 insertions(+), 37 deletions(-) create mode 100644 agent/internal/mods/removal.go create mode 100644 agent/internal/mods/removal_test.go diff --git a/agent/internal/mods/manifest.go b/agent/internal/mods/manifest.go index dea1d1cd6..cef5cbda7 100644 --- a/agent/internal/mods/manifest.go +++ b/agent/internal/mods/manifest.go @@ -1,6 +1,7 @@ package mods import ( + "crypto/rand" "encoding/json" "errors" "io/fs" @@ -11,7 +12,8 @@ import ( // manifestName is the per-volume install ledger. The leading dot keeps it // out of mod listings and out of reach of client-supplied names (safeName -// rejects dotfiles), so it needs no extra guarding. Each per-(version+loader) +// rejects dotfiles). Filesystem confinement still applies to ledger reads and +// replacements. Each per-(version+loader) // mod volume carries its own manifest, so metadata follows the files across // game-version switches. const manifestName = ".gameplane-mods.json" @@ -44,11 +46,15 @@ type manifest struct { // degrades to an empty manifest — mods then list as unmanaged rather than // breaking the endpoint; the next mutation rewrites a clean file. func loadManifest(dir string) *manifest { - empty := &manifest{Version: 1, Mods: map[string]*ModMeta{}} // Validate that dir doesn't escape expected bounds. dirClean := filepath.Clean(dir) manifestPath := filepath.Join(dirClean, manifestName) data, err := os.ReadFile(manifestPath) + return decodeManifest(data, err) +} + +func decodeManifest(data []byte, err error) *manifest { + empty := &manifest{Version: 1, Mods: map[string]*ModMeta{}} if err != nil { if !errors.Is(err, fs.ErrNotExist) { slog.Warn("mod manifest read", "err", err) @@ -67,6 +73,46 @@ func loadManifest(dir string) *manifest { return &parsed } +// updateManifestInRoot keeps DELETE's metadata on the same directory handle as +// the removal. Reopening h.dir here would let a concurrent directory replacement +// redirect this otherwise confined operation. Metadata remains best-effort. +func (h *handler) updateManifestInRoot(root *os.Root, fn func(map[string]*ModMeta)) { + h.mu.Lock() + defer h.mu.Unlock() + m := decodeManifest(root.ReadFile(manifestName)) + fn(m.Mods) + for name := range m.Mods { + if _, err := root.Lstat(name); errors.Is(err, fs.ErrNotExist) { + delete(m.Mods, name) + } + } + if err := m.saveInRoot(root); err != nil { + slog.Warn("mod manifest write", "err", err) + } +} + +func (m *manifest) saveInRoot(root *os.Root) error { + data, err := json.Marshal(m) + if err != nil { + return err + } + tmpName := ".manifest-" + rand.Text() + tmp, err := root.OpenFile(tmpName, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600) + if err != nil { + return err + } + defer func() { _ = root.Remove(tmpName) }() + _, writeErr := tmp.Write(data) + closeErr := tmp.Close() + if writeErr != nil { + return writeErr + } + if closeErr != nil { + return closeErr + } + return root.Rename(tmpName, manifestName) +} + // save writes the manifest atomically (dot-prefixed temp + rename, the same // pattern as downloads) so a crash never leaves a torn file behind. func (m *manifest) save(dir string) error { diff --git a/agent/internal/mods/mods.go b/agent/internal/mods/mods.go index 52f2f705b..a34b12650 100644 --- a/agent/internal/mods/mods.go +++ b/agent/internal/mods/mods.go @@ -52,6 +52,7 @@ const ( ) type handler struct { + dataRoot string // trusted data-volume root; removal opens descendants relative to it dir string // absolute mods directory, "" when unconfigured exts []string // lowercased extension allowlist ("" = any) allowed []string // download host allowlist; nil disables installs @@ -96,6 +97,7 @@ func newHandler(dataRoot string, spec *caps.Mods) *handler { return h } h.dir = dir + h.dataRoot = root h.extract = spec.Extract for _, e := range spec.Extensions { h.exts = append(h.exts, strings.ToLower(e)) @@ -455,23 +457,12 @@ func (h *handler) upload(w http.ResponseWriter, req *http.Request) { // removeEntry deletes an installed mod by name — the whole folder for // extract loaders, a single file otherwise. func (h *handler) removeEntry(name string) error { - // Use ConfinePath to validate and resolve the target within h.dir. - // ConfinePath consolidates validation and symlink resolution in one function, - // making the guard legible to CodeQL taint analysis. - target, err := ConfinePath(h.dir, name) + root, err := h.openModsRoot() if err != nil { - return fmt.Errorf("path escape attempt: %w", err) - } - // Re-check inline so the guard is visible at the point of use. - rootClean := filepath.Clean(h.dir) - target = filepath.Clean(target) - if target != rootClean && !strings.HasPrefix(target, rootClean+string(os.PathSeparator)) { - return errors.New("path escape attempt") - } - if h.extract { - return os.RemoveAll(target) + return err } - return os.Remove(target) + defer func() { _ = root.Close() }() + return removeModEntry(root, name, h.extract) } // downloadTemp streams rawURL into a temp file under the mods dir, enforcing @@ -738,31 +729,25 @@ func (h *handler) remove(w http.ResponseWriter, req *http.Request) { httpjson.Error(w, http.StatusBadRequest, err.Error()) return } - // Use ConfinePath to validate and resolve the target within h.dir. - // ConfinePath consolidates validation and symlink resolution in one function, - // making the guard legible to CodeQL taint analysis. - target, err := ConfinePath(h.dir, name) - if err != nil { - httpjson.Error(w, http.StatusBadRequest, err.Error()) - return + root, err := h.openModsRoot() + if err == nil { + defer func() { _ = root.Close() }() + err = removeModEntry(root, name, h.extract) } - // Re-check inline so the guard is visible at the point of use. - rootClean := filepath.Clean(h.dir) - target = filepath.Clean(target) - if target != rootClean && !strings.HasPrefix(target, rootClean+string(os.PathSeparator)) { + if errors.Is(err, ErrEscapesRoot) { httpjson.Error(w, http.StatusBadRequest, "path escape attempt") return } - if _, statErr := os.Stat(target); errors.Is(statErr, os.ErrNotExist) { + if errors.Is(err, fs.ErrNotExist) { httpjson.Error(w, http.StatusNotFound, "no such mod") return } - if err := h.removeEntry(name); err != nil { + if err != nil { slog.Warn("mod remove", "err", err) httpjson.Error(w, http.StatusInternalServerError, "could not remove mod") return } - h.updateManifest(func(mods map[string]*ModMeta) { + h.updateManifestInRoot(root, func(mods map[string]*ModMeta) { delete(mods, name) }) w.WriteHeader(http.StatusNoContent) diff --git a/agent/internal/mods/mods_test.go b/agent/internal/mods/mods_test.go index 5026b1463..fb65a80cf 100644 --- a/agent/internal/mods/mods_test.go +++ b/agent/internal/mods/mods_test.go @@ -689,8 +689,8 @@ func TestInstall_ExtractBadArchive(t *testing.T) { // --- Tests for ConfinePath-based refactors (T006) --- -// TestRemoveEntry_ConfinePath verifies that removeEntry uses ConfinePath -// to validate the target and rejects path traversal attempts. +// TestRemoveEntry_ConfinePath preserves the original valid-name and traversal +// contract now enforced by descriptor-relative removal. func TestRemoveEntry_ConfinePath(t *testing.T) { root := t.TempDir() modsDir := filepath.Join(root, "mods") @@ -786,8 +786,8 @@ func TestSwapInArchive_ConfinePath(t *testing.T) { } } -// TestRemoveWithConfinePath verifies that the remove handler uses ConfinePath -// to validate the name parameter and rejects traversal attempts. +// TestRemoveWithConfinePath preserves the HTTP name-validation contract when +// deletion is performed through the opened mods root. func TestRemoveWithConfinePath(t *testing.T) { root := t.TempDir() if err := os.MkdirAll(filepath.Join(root, "mods"), 0o755); err != nil { @@ -808,7 +808,7 @@ func TestRemoveWithConfinePath(t *testing.T) { t.Fatal("file should be gone") } - // remove should reject traversal attempts via ConfinePath validation. + // Rooted removal must still reject the original traversal cases. for _, bad := range []string{"../escape", "a/b", ".."} { status, _ := do(t, srv, http.MethodDelete, "/mods?name="+bad, nil) if status != http.StatusBadRequest { diff --git a/agent/internal/mods/removal.go b/agent/internal/mods/removal.go new file mode 100644 index 000000000..b1f4a00c8 --- /dev/null +++ b/agent/internal/mods/removal.go @@ -0,0 +1,75 @@ +package mods + +import ( + "fmt" + "os" + "path/filepath" + "strings" +) + +// openModsRoot anchors removal to the configured data volume. Open each mods +// directory component without accepting symlink aliases, then retain its handle +// so a later rename cannot redirect removal or its manifest update. +func (h *handler) openModsRoot() (*os.Root, error) { + rel, err := filepath.Rel(h.dataRoot, h.dir) + if err != nil || !filepath.IsLocal(rel) { + return nil, ErrEscapesRoot + } + root, err := os.OpenRoot(h.dataRoot) + if err != nil { + return nil, fmt.Errorf("open mod data root: %w", err) + } + if rel == "." { + return root, nil + } + for _, component := range strings.Split(rel, string(os.PathSeparator)) { + next, err := openModDirectory(root, component) + _ = root.Close() + if err != nil { + return nil, err + } + root = next + } + return root, nil +} + +func openModDirectory(parent *os.Root, name string) (*os.Root, error) { + info, err := parent.Lstat(name) + if err != nil { + return nil, fmt.Errorf("inspect mod directory: %w", err) + } + if info.Mode()&os.ModeSymlink != 0 || !info.IsDir() { + return nil, ErrEscapesRoot + } + root, err := parent.OpenRoot(name) + if err != nil { + return nil, fmt.Errorf("open mod directory: %w", err) + } + opened, err := root.Stat(".") + if err != nil || !os.SameFile(info, opened) { + _ = root.Close() + return nil, ErrEscapesRoot + } + return root, nil +} + +// removeModEntry never resolves the final name to a symlink's referent. Existing +// links are rejected; a link swapped in after Lstat is only unlinked by Remove +// or RemoveAll. Recursive removal likewise never follows child symlinks. +func removeModEntry(root *os.Root, name string, recursive bool) error { + name, err := safeName(name) + if err != nil { + return fmt.Errorf("invalid mod name: %w", err) + } + info, err := root.Lstat(name) + if err != nil { + return fmt.Errorf("inspect mod entry: %w", err) + } + if info.Mode()&os.ModeSymlink != 0 { + return ErrEscapesRoot + } + if recursive { + return root.RemoveAll(name) + } + return root.Remove(name) +} diff --git a/agent/internal/mods/removal_test.go b/agent/internal/mods/removal_test.go new file mode 100644 index 000000000..3da0f9444 --- /dev/null +++ b/agent/internal/mods/removal_test.go @@ -0,0 +1,237 @@ +package mods + +import ( + "errors" + "net/http" + "os" + "path/filepath" + "testing" + + "github.com/GameplanePanel/gameplane/agent/internal/caps" +) + +func TestRemove_RejectsSymlinkEntries(t *testing.T) { + for _, extract := range []bool{false, true} { + for _, target := range []string{"inside", "outside", "root", "dangling", "loop"} { + t.Run(target+map[bool]string{false: "/file", true: "/extracted"}[extract], func(t *testing.T) { + data := t.TempDir() + dir := filepath.Join(data, "mods") + outside := t.TempDir() + if err := os.Mkdir(dir, 0o755); err != nil { + t.Fatal(err) + } + for _, parent := range []string{dir, outside} { + if err := os.WriteFile(filepath.Join(parent, "kept.jar"), []byte("keep"), 0o600); err != nil { + t.Fatal(err) + } + } + ledger := []byte(`{"version":1,"mods":{"alias":{"provider":"upload"}}}`) + if err := os.WriteFile(filepath.Join(dir, manifestName), ledger, 0o600); err != nil { + t.Fatal(err) + } + destination := map[string]string{"inside": "kept.jar", "outside": filepath.Join(outside, "kept.jar"), "root": ".", "dangling": "missing.jar", "loop": "alias"}[target] + if err := os.Symlink(destination, filepath.Join(dir, "alias")); err != nil { + t.Fatal(err) + } + srv := newSrv(t, data, &caps.Mods{Path: "mods", Extract: extract}) + if status, _ := do(t, srv, http.MethodDelete, "/mods?name=alias", nil); status != http.StatusBadRequest { + t.Fatalf("delete symlink: status=%d, want 400", status) + } + for _, parent := range []string{dir, outside} { + assertRemovalFile(t, filepath.Join(parent, "kept.jar"), "keep") + } + assertRemovalFile(t, filepath.Join(dir, manifestName), string(ledger)) + if info, err := os.Lstat(filepath.Join(dir, "alias")); err != nil || info.Mode()&os.ModeSymlink == 0 { + t.Fatalf("rejected link changed: info=%v err=%v", info, err) + } + }) + } + } +} + +func TestRemove_RejectsAliasedModsDirectory(t *testing.T) { + for _, target := range []string{"outside", "root", "inside"} { + t.Run(target, func(t *testing.T) { + data, outside := t.TempDir(), t.TempDir() + inside := filepath.Join(data, "other") + if err := os.Mkdir(inside, 0o755); err != nil { + t.Fatal(err) + } + for _, parent := range []string{data, outside, inside} { + if err := os.WriteFile(filepath.Join(parent, "kept.jar"), []byte("keep"), 0o600); err != nil { + t.Fatal(err) + } + } + destination := map[string]string{"outside": outside, "root": ".", "inside": "other"}[target] + if err := os.Symlink(destination, filepath.Join(data, "mods")); err != nil { + t.Fatal(err) + } + srv := newSrv(t, data, &caps.Mods{Path: "mods"}) + if status, _ := do(t, srv, http.MethodDelete, "/mods?name=kept.jar", nil); status != http.StatusBadRequest { + t.Fatalf("delete through aliased root: status=%d, want 400", status) + } + for _, parent := range []string{data, outside, inside} { + assertRemovalFile(t, filepath.Join(parent, "kept.jar"), "keep") + } + }) + } +} + +func TestRemove_RecursivePreservesSymlinkTargets(t *testing.T) { + data, outside := t.TempDir(), t.TempDir() + dir := filepath.Join(data, "mods", "pack") + if err := os.MkdirAll(dir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(outside, "kept.jar"), []byte("keep"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.Symlink(outside, filepath.Join(dir, "child")); err != nil { + t.Fatal(err) + } + srv := newSrv(t, data, &caps.Mods{Path: "mods", Extract: true}) + if status, _ := do(t, srv, http.MethodDelete, "/mods?name=pack", nil); status != http.StatusNoContent { + t.Fatalf("delete pack: status=%d, want 204", status) + } + if _, err := os.Lstat(dir); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("pack remains: %v", err) + } + assertRemovalFile(t, filepath.Join(outside, "kept.jar"), "keep") +} + +func TestRemove_RootAndManifestSurviveDirectoryReplacement(t *testing.T) { + data, outside := t.TempDir(), t.TempDir() + dir := filepath.Join(data, "mods") + if err := os.Mkdir(dir, 0o755); err != nil { + t.Fatal(err) + } + ledger := `{"version":1,"mods":{"gone.jar":{"provider":"upload"}}}` + for _, parent := range []string{dir, outside} { + if err := os.WriteFile(filepath.Join(parent, "gone.jar"), []byte("keep"), 0o600); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(parent, manifestName), []byte(ledger), 0o600); err != nil { + t.Fatal(err) + } + } + h := newHandler(data, &caps.Mods{Path: "mods"}) + root, err := h.openModsRoot() + if err != nil { + t.Fatal(err) + } + defer root.Close() + // Deterministically replace the pathname after the operation acquires its + // root handle. Neither deletion nor metadata may re-open that pathname. + moved := filepath.Join(data, "moved") + if err := os.Rename(dir, moved); err != nil { + t.Fatal(err) + } + if err := os.Symlink(outside, dir); err != nil { + t.Fatal(err) + } + if err := removeModEntry(root, "gone.jar", false); err != nil { + t.Fatal(err) + } + h.updateManifestInRoot(root, func(mods map[string]*ModMeta) { delete(mods, "gone.jar") }) + if _, err := os.Stat(filepath.Join(moved, "gone.jar")); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("original entry remains: %v", err) + } + if entries := readManifest(t, moved); len(entries) != 0 { + t.Fatalf("original manifest did not update: %+v", entries) + } + assertRemovalFile(t, filepath.Join(outside, "gone.jar"), "keep") + assertRemovalFile(t, filepath.Join(outside, manifestName), ledger) +} + +func TestRemove_ConfiguredDirectories(t *testing.T) { + for _, path := range []string{"mods", "nested/mods", "/mods", "."} { + t.Run(path, func(t *testing.T) { + data := t.TempDir() + h := newHandler(data, &caps.Mods{Path: path}) + if err := os.MkdirAll(h.dir, 0o755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(h.dir, "gone.jar"), []byte("remove"), 0o600); err != nil { + t.Fatal(err) + } + if err := h.removeEntry("gone.jar"); err != nil { + t.Fatal(err) + } + if _, err := os.Stat(filepath.Join(h.dir, "gone.jar")); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("entry remains: %v", err) + } + }) + } +} + +func TestRemove_ManifestOutsideSymlinkPreservesTarget(t *testing.T) { + data, outside := t.TempDir(), t.TempDir() + h := newHandler(data, &caps.Mods{Path: "mods"}) + if err := os.Mkdir(h.dir, 0o755); err != nil { + t.Fatal(err) + } + ledger := `{"version":1,"mods":{"foreign.jar":{"provider":"upload"}}}` + target := filepath.Join(outside, manifestName) + if err := os.WriteFile(target, []byte(ledger), 0o600); err != nil { + t.Fatal(err) + } + if err := os.Symlink(target, filepath.Join(h.dir, manifestName)); err != nil { + t.Fatal(err) + } + root, err := h.openModsRoot() + if err != nil { + t.Fatal(err) + } + defer root.Close() + h.updateManifestInRoot(root, func(mods map[string]*ModMeta) { + if len(mods) != 0 { + t.Fatalf("loaded outside manifest: %+v", mods) + } + }) + assertRemovalFile(t, target, ledger) + if info, err := os.Lstat(filepath.Join(h.dir, manifestName)); err != nil || !info.Mode().IsRegular() { + t.Fatalf("confined manifest was not published: info=%v err=%v", info, err) + } +} + +func TestManifestSaveInRoot_PublishFailureCleansTemporaryFile(t *testing.T) { + data := t.TempDir() + // A directory at the ledger path deterministically prevents atomic rename, + // even when CI runs as root. Its existing contents must remain intact. + if err := os.Mkdir(filepath.Join(data, manifestName), 0o755); err != nil { + t.Fatal(err) + } + previous := filepath.Join(data, manifestName, "previous") + if err := os.WriteFile(previous, []byte("keep"), 0o600); err != nil { + t.Fatal(err) + } + root, err := os.OpenRoot(data) + if err != nil { + t.Fatal(err) + } + defer root.Close() + m := &manifest{Version: 1, Mods: map[string]*ModMeta{}} + if err := m.saveInRoot(root); err == nil { + t.Fatal("publishing over a directory succeeded") + } + assertRemovalFile(t, previous, "keep") + entries, err := os.ReadDir(data) + if err != nil || len(entries) != 1 || entries[0].Name() != manifestName { + t.Fatalf("temporary manifest leaked: entries=%v err=%v", entries, err) + } +} + +func TestRemove_MissingDirectory(t *testing.T) { + srv := newSrv(t, t.TempDir(), &caps.Mods{Path: "mods"}) + if status, _ := do(t, srv, http.MethodDelete, "/mods?name=missing.jar", nil); status != http.StatusNotFound { + t.Fatalf("missing mods directory: status=%d, want 404", status) + } +} + +func assertRemovalFile(t *testing.T, name, want string) { + t.Helper() + data, err := os.ReadFile(name) + if err != nil || string(data) != want { + t.Fatalf("file changed: %s: data=%q err=%v", name, data, err) + } +} diff --git a/agent/specs.md b/agent/specs.md index 6806637ae..059790c31 100644 --- a/agent/specs.md +++ b/agent/specs.md @@ -200,7 +200,7 @@ All endpoints on the `--addr` control mux, except `/healthz`, return `401 Unauth - **RCON is a lower-trust boundary**: The agent uses `netguard.IsAllowed()` for WebRcon dial operations (permissive, allows loopback and private addresses on the assumption game servers run inside the cluster), and `netguard.IsPublic()` (strict, permissive only for well-known registries) for mod-install downloads, assuming modules are less trusted than the operator. - **Gameaction validation is independent**: Both the API (stdin pod-attach) and the agent (RCON) call `gameaction.Resolve()` independently to validate action inputs (no control characters, 512-char cap, required-ness checks, etc.). Neither trusts the other. The players moderation endpoints (kick/ban reason) also call gameaction.CheckText, so moderation text follows the same character policy as actions. - **No persistent storage**: The agent has no database. GameServer status patches flow through the operator; all transient state (WebSocket streams, RCON sessions) is in-memory. -- **Path confinement via per-component validation**: The `files` package validates the requested path lexically in `resolve()` (direct traversal: `..`, `/`, absolute paths; any dot-prefixed component is rejected as a dotfile) and then performs every operation relative to a directory descriptor opened on the data root (`rooted_linux.go`). Each component is opened with `openat(O_NOFOLLOW)` (directories also `O_DIRECTORY`), so a symlink at any component of the path, ancestor or final, is refused with HTTP 400 (`dotfile access denied` when the link's target lies below a dot-prefixed component, otherwise `path escapes root (symbolic link in path)`). Reads, writes, creates, deletes and listings use the descriptors they obtained (`fstatat`, `mkdirat`, `unlinkat`, `renameat`, `ServeContent` on the opened fd) and never re-open a path string, so replacing a checked directory or file with a symlink between validation and use cannot move the operation outside the root; a delete removes a symlink as the link it is and a recursive delete never follows links. The data-root path itself is the one trusted, followed component (it may be a symlink). `savePart` likewise rejects dot-prefixed upload filenames; `/files/list` excludes dot-prefixed entries from its output and a delete refuses trees containing them. The `mods` package uses `ConfinePath(rootDir, untrustedName)` as the authoritative guard for single-component mod paths (upload, download, removal, archive swap), and `ConfineRelPath(root, relPath)` for multi-component archive entry paths during extraction. ConfinePath returns a cleaned, absolute path guaranteed to be confined within rootDir (or raises an error if escape is attempted), validating against both direct traversal and symlink escape. The prior `safeName()` function remains in use as caller-side defense-in-depth (pre-filtering before ConfinePath), but ConfinePath is the authoritative guard within the function boundary where the archive operation occurs. +- **Path confinement via per-component validation**: The `files` package validates the requested path lexically in `resolve()` (direct traversal: `..`, `/`, absolute paths; any dot-prefixed component is rejected as a dotfile) and then performs every operation relative to a directory descriptor opened on the data root (`rooted_linux.go`). Each component is opened with `openat(O_NOFOLLOW)` (directories also `O_DIRECTORY`), so a symlink at any component of the path, ancestor or final, is refused with HTTP 400 (`dotfile access denied` when the link's target lies below a dot-prefixed component, otherwise `path escapes root (symbolic link in path)`). Reads, writes, creates, deletes and listings use the descriptors they obtained (`fstatat`, `mkdirat`, `unlinkat`, `renameat`, `ServeContent` on the opened fd) and never re-open a path string, so replacing a checked directory or file with a symlink between validation and use cannot move the operation outside the root; a delete removes a symlink as the link it is and a recursive delete never follows links. The data-root path itself is the one trusted, followed component (it may be a symlink). `savePart` likewise rejects dot-prefixed upload filenames; `/files/list` excludes dot-prefixed entries from its output and a delete refuses trees containing them. The `mods` package retains `ConfinePath(rootDir, untrustedName)` for upload, download and archive swap, and `ConfineRelPath(root, relPath)` for archive entry paths during extraction; these validate paths before those operations. Mod deletion instead validates a bare name with `safeName()` and opens an `os.Root` at the trusted data-volume root. It refuses symlinks in each configured mods-directory component, checks the opened directory identity, and retains the resulting directory handle for removal and the HTTP DELETE manifest update. Existing mod-entry symlinks are rejected, including aliases to the mods root; recursive deletion never follows child symlinks. Replacing a checked path cannot redirect the operation to a symlink target. Missing mods directories still return HTTP 404. Manifest writes remain best-effort and publish through a temporary file and atomic rename on that same handle, cleaning the temporary file on failure. Separate upload, download and archive-publication paths are unchanged by this deletion boundary. - **WebRcon dials through netguard**: `rcon.websocket.go`'s `ensureLocked()` method dials the Rust WebSocket using `netguard.IsAllowed()` dial policy for defense-in-depth, allowing loopback and private addresses (since game servers legitimately run inside the cluster) while blocking the ranges `netguard.IsAllowed` rejects (link-local, multicast, NAT64/6to4). - **WebRcon dial errors are redacted**: On dial failure, `ensureLocked()` strips the URL path (which carries the escaped password) from any `*url.Error` via `redactURLErr`. If the redacted message still contains the password in raw, path-escaped, or query-escaped form (`errorLeaksSecret`; passwords shorter than three characters are not checked), it returns a generic `connection failed` error without the original cause; otherwise it wraps the redacted error with `%w` so `errors.Is`/`errors.As` still work. - **Agent-created directories are 0755**: `files` (mkdir, upload and write ancestors) and `mods` (mods dir, archive extraction dirs) create directories 0o755, matching the 0o644 files, because the game container runs as a different uid and on templates without an fsGroup reaches the shared volume only through the "other" bits. The gosec G301 finding for these two files is scoped in `.golangci.yml`. From d1ded7f1992037430e025713869b4a76551cdb7f Mon Sep 17 00:00:00 2001 From: Baylem Date: Sat, 24 Oct 2026 10:53:14 -0400 Subject: [PATCH 15/18] fix: align tailscale condition documentation and coverage Signed-off-by: Baylem --- operator/internal/controller/gameserver_tailscale_test.go | 4 ++++ operator/specs.md | 4 ++-- specs/done_018-v0-3-release-readiness/OPEN-DECISIONS.md | 2 +- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/operator/internal/controller/gameserver_tailscale_test.go b/operator/internal/controller/gameserver_tailscale_test.go index 1653fccf2..597c60b9a 100644 --- a/operator/internal/controller/gameserver_tailscale_test.go +++ b/operator/internal/controller/gameserver_tailscale_test.go @@ -45,4 +45,8 @@ func TestTailscaleAdvertisedProtocolsAndReadiness(t *testing.T) { tmpl.Spec.Ports[0].Protocol = corev1.ProtocolSCTP plan = r.planTunnel(context.Background(), gs, tmpl) assert(metav1.ConditionFalse) + condition := meta.FindStatusCondition(computeTunnelConditions(gs, plan, dep), "TunnelReady") + if condition == nil || condition.Reason != "UnsupportedProtocol" { + t.Fatalf("unsupported protocol condition=%+v, want reason UnsupportedProtocol", condition) + } } diff --git a/operator/specs.md b/operator/specs.md index 9113c443c..155b5eaff 100644 --- a/operator/specs.md +++ b/operator/specs.md @@ -162,7 +162,7 @@ Chart placement and optional PDBs are documented in Primary reconcilers register with the manager in `cmd/main.go` and handle CRD lifecycle: ### GameServerReconciler -- **Tailscale forwarding (#722):** the tunnel receives advertised `name:port:protocol` mappings and forwards only to `..svc` through embedded tsnet. Its Deployment uses Recreate and a per-Pod EmptyDir for identity across container restarts. `/readyz` on 8091 requires an online tailnet identity and every TCP/UDP listener. TunnelReady additionally requires the forwarding probe, current Deployment observedGeneration and an updated ready replica; unsupported protocols report InvalidConfig. Tailscale endpoints are withheld while not ready. Pod replacement requires a reusable auth key and an available MagicDNS hostname. No extra privileges or Kubernetes RBAC are added. +- **Tailscale forwarding (#722):** the tunnel receives advertised `name:port:protocol` mappings and forwards only to `..svc` through embedded tsnet. Its Deployment uses Recreate and a per-Pod EmptyDir for identity across container restarts. `/readyz` on 8091 requires an online tailnet identity and every TCP/UDP listener. TunnelReady additionally requires the forwarding probe, current Deployment observedGeneration and an updated ready replica; unsupported protocols report UnsupportedProtocol. Tailscale endpoints are withheld while not ready. Pod replacement requires a reusable auth key and an available MagicDNS hostname. No extra privileges or Kubernetes RBAC are added. - **Responsibility:** Reconcile GameServer → StatefulSet, Service, Config ConfigMap, PVC, NetworkPolicy. - **Template recovery:** A missing template reports Failed and retries every 15 seconds. Template creation or spec changes enqueue referencing servers immediately, so a GameServer applied before its Module/template recovers without a manual edit; template status updates do not enqueue servers. - **Key functions:** @@ -238,7 +238,7 @@ Primary reconcilers register with the manager in `cmd/main.go` and handle CRD li - **Endpoint population:** `endpointsFromService()` populates `GameServerEndpoint.pool` only when a pool request was actually translated (outcome == Assigned and Manager supports it); never claims a pool for the ClusterIP address shown while assignment is pending, for an ignored request, or for a cluster without an address manager. - **status.startedAt:** while the server is Running, `status.startedAt` follows the start time of the running game container, matched by container name (`gameContainerStartedAt` in `gameserver_status.go`). Uptime therefore resets when the game container restarts inside the same pod. When that time is unavailable, `startedAt` is set once to the current time and kept. Test: `TestGameContainerStartedAt` (`gameserver_status_test.go`). - **conditionArgs:** `status.conditionArgs[]` carries the values interpolated into the English `message` of the condition's current reason, recomputed on every `reconcileStatus` pass by `deriveConditionArgs` (`gameserver_condition_args.go`); `setPhase`, `setTunnelCredentialRefused` and `clearTunnelCredentialRefused` maintain their own entry. The `message` is unchanged. Translation ids are `condition..` (lower-cased) and are listed in `i18n/locales/en/app.json`. - - **TunnelReady condition:** Reports tunnel pod readiness when `spec.networking.tunnel.enabled=true` (absent when disabled). Computed in `computeTunnelConditions` (`gameserver_status.go`) each pass reconcileStatus runs, with reasons `InvalidConfig`, `DeploymentNotReady`, `PortNotMapped` (frp, unmapped advertised ports), `NoCredentials` (tailscale/playit missing `credentialsSecretRef`), `UnknownProvider`, and `Ready`. **Credential ownership:** `reconcileTunnel` refuses to mount a `credentialsSecretRef` Secret that has no `ownerReference` matching the GameServer's name and UID (including a stale reference from a deleted same-name GameServer) (`isServerOwnedSecret`) — supported credential paths are the dashboard, the `PUT /servers/{name}:tunnel-credentials` API (which sets the ownerReference), or a manually-created Secret with an explicit ownerReference. A refused Secret is never mounted: `reconcileTunnel` still writes the tunnel Deployment, with no credential volume and zero replicas, so a tunnel already running with that Secret stops. It also writes `TunnelReady=False/TunnelCredentialRefused` (naming the Secret, never its data) directly from `reconcileTunnel` before `Reconcile` returns its error — this bypasses the normal `reconcileStatus` pass since Reconcile returns early on that error — The condition carries its own current `observedGeneration`, but `status.observedGeneration` is not advanced on a refused pass (the rest of status is not recomputed). Once the credential becomes acceptable (or the tunnel is no longer wanted), `reconcileTunnel` removes the refusal condition itself (`clearTunnelCredentialRefused`) before returning, so it does not linger if a later step (NetworkPolicy, Service, StatefulSet) fails; reconcileStatus then writes the real TunnelReady. The refusal message points at the dashboard/API, except when the refused Secret is the canonical `-tunnel-auth` name, where the API answers 409 and the message says to fix the ownerReference or delete the Secret first. + - **TunnelReady condition:** Reports tunnel pod readiness when `spec.networking.tunnel.enabled=true` (absent when disabled). Computed in `computeTunnelConditions` (`gameserver_status.go`) each pass reconcileStatus runs, with reasons `InvalidConfig`, `UnsupportedProtocol` (Tailscale, advertised protocols other than TCP/UDP), `DeploymentNotReady`, `PortNotMapped` (frp, unmapped advertised ports), `NoCredentials` (tailscale/playit missing `credentialsSecretRef`), `UnknownProvider`, and `Ready`. **Credential ownership:** `reconcileTunnel` refuses to mount a `credentialsSecretRef` Secret that has no `ownerReference` matching the GameServer's name and UID (including a stale reference from a deleted same-name GameServer) (`isServerOwnedSecret`) — supported credential paths are the dashboard, the `PUT /servers/{name}:tunnel-credentials` API (which sets the ownerReference), or a manually-created Secret with an explicit ownerReference. A refused Secret is never mounted: `reconcileTunnel` still writes the tunnel Deployment, with no credential volume and zero replicas, so a tunnel already running with that Secret stops. It also writes `TunnelReady=False/TunnelCredentialRefused` (naming the Secret, never its data) directly from `reconcileTunnel` before `Reconcile` returns its error — this bypasses the normal `reconcileStatus` pass since Reconcile returns early on that error — The condition carries its own current `observedGeneration`, but `status.observedGeneration` is not advanced on a refused pass (the rest of status is not recomputed). Once the credential becomes acceptable (or the tunnel is no longer wanted), `reconcileTunnel` removes the refusal condition itself (`clearTunnelCredentialRefused`) before returning, so it does not linger if a later step (NetworkPolicy, Service, StatefulSet) fails; reconcileStatus then writes the real TunnelReady. The refusal message points at the dashboard/API, except when the refused Secret is the canonical `-tunnel-auth` name, where the API answers 409 and the message says to fix the ownerReference or delete the Secret first. ### GameTemplateReconciler - **Responsibility:** Lightweight; only maintains status.inUseCount (how many GameServers ref this template). diff --git a/specs/done_018-v0-3-release-readiness/OPEN-DECISIONS.md b/specs/done_018-v0-3-release-readiness/OPEN-DECISIONS.md index effd4cb5b..933823283 100644 --- a/specs/done_018-v0-3-release-readiness/OPEN-DECISIONS.md +++ b/specs/done_018-v0-3-release-readiness/OPEN-DECISIONS.md @@ -1,6 +1,6 @@ # Open Decisions: v0.3 Release Readiness Audit -### OD-031: Tailscale game forwarding (F-173) — RESOLVED 2026-10-21 +## OD-031: Tailscale game forwarding (F-173) — RESOLVED 2026-10-21 Decision (maintainer reply on #722): embed `tailscale.com/tsnet` and forward both TCP and UDP from advertised tailnet listeners to the GameServer's backing Service. This explicitly ends the tunnel module's stdlib-only dependency policy. Device registration alone cannot establish readiness; the same change must provide actual tailnet/listener health. The previous Known Gaps entry referred to a decision that had never been recorded. From cfe8c19a3de667659a94b135b71269a76ec92c42 Mon Sep 17 00:00:00 2001 From: Baylem Date: Sun, 25 Oct 2026 19:14:18 -0400 Subject: [PATCH 16/18] fix: preserve known relay outages and reviewed prerequisites Signed-off-by: Baylem --- charts/gameplane/templates/NOTES.txt | 6 + .../gameplane/templates/monitoring-rules.yaml | 2 +- .../templates/operator-strategy-migrate.yaml | 2 +- charts/gameplane/templates/operator.yaml | 4 +- charts/gameplane/templates/validate.yaml | 3 - .../tests/monitoring-rules.test.yaml | 8 +- charts/tests/check_workloads.py | 9 +- docs/monitoring.md | 14 +- docs/production-placement.md | 7 +- docs/relay-health.md | 17 ++- docs/release-notes.md | 5 +- i18n/locales/en/app.json | 4 + operator/hack/condition-ids/main.go | 1 + .../controller/gameserver_controller.go | 5 + .../gameserver_relay_health_test.go | 74 +++++++-- .../internal/controller/gameserver_status.go | 18 ++- .../controller/gameserver_tailscale_test.go | 49 ++++++ .../internal/controller/gameserver_tunnel.go | 28 ++++ .../internal/controller/sentinel_isolation.go | 42 ++++++ .../controller/sentinel_isolation_test.go | 128 ++++++++++++++++ operator/specs.md | 14 +- tunnel/main.go | 2 +- tunnel/relay_health.go | 18 +++ tunnel/relay_metrics.go | 3 +- tunnel/relay_metrics_test.go | 16 ++ tunnel/specs.md | 19 ++- tunnel/tailscale.go | 60 ++++---- tunnel/tailscale_forward.go | 3 + tunnel/tailscale_forward_test.go | 54 +++++++ tunnel/tailscale_retry_test.go | 111 ++++++++++++++ tunnel/tailscale_test.go | 141 +++++++++++++++++- 31 files changed, 800 insertions(+), 67 deletions(-) create mode 100644 tunnel/tailscale_retry_test.go diff --git a/charts/gameplane/templates/NOTES.txt b/charts/gameplane/templates/NOTES.txt index e33acd801..63b9f80d1 100644 --- a/charts/gameplane/templates/NOTES.txt +++ b/charts/gameplane/templates/NOTES.txt @@ -4,6 +4,12 @@ persistence and dashboard blocks are absent. */ -}} {{- $r := .Values.api.telemetry.receiver }} +{{- if and (eq (include "gameplane.operatorEnabled" .) "true") (gt (int .Values.operator.replicas) 1) (not .Values.operator.leaderElect) }} + +Operator leader election is enabled automatically because operator.replicas is +greater than one. The saved leaderElect=false value remains compatible with +upgrades; all operator replicas coordinate through the leader-election Lease. +{{- end }} {{- if .Values.serviceMonitors.enabled }} Monitoring: configure Prometheus monitor label selectors AND discovery of the diff --git a/charts/gameplane/templates/monitoring-rules.yaml b/charts/gameplane/templates/monitoring-rules.yaml index 120c5c9a6..98b383683 100644 --- a/charts/gameplane/templates/monitoring-rules.yaml +++ b/charts/gameplane/templates/monitoring-rules.yaml @@ -24,7 +24,7 @@ spec: labels: { severity: warning } annotations: { summary: "Gameplane relay connection or forwards are unavailable" } - alert: GameplaneRelayHealthUnknown - expr: max by (namespace, pod) (gameplane_tunnel_connection_known{namespace="{{ .Values.gamesNamespace }}"}) == 0 + expr: (max by (namespace, pod) (gameplane_tunnel_connection_known{namespace="{{ .Values.gamesNamespace }}"}) == 0) and on (namespace, pod) (max by (namespace, pod) (gameplane_tunnel_connection_supported{namespace="{{ .Values.gamesNamespace }}"}) == 1) for: 5m labels: { severity: info } annotations: { summary: "Gameplane provider cannot confirm live relay health" } diff --git a/charts/gameplane/templates/operator-strategy-migrate.yaml b/charts/gameplane/templates/operator-strategy-migrate.yaml index e376fca4b..c6b51ef85 100644 --- a/charts/gameplane/templates/operator-strategy-migrate.yaml +++ b/charts/gameplane/templates/operator-strategy-migrate.yaml @@ -1,6 +1,6 @@ {{- /* SSA cannot remove a server-defaulted rollingUpdate field. Patch it before Helm applies Recreate, including installations that manage CRDs externally. */ -}} -{{- if and .Release.IsUpgrade (eq (include "gameplane.operatorEnabled" .) "true") (not .Values.operator.leaderElect) }} +{{- if and .Release.IsUpgrade (eq (include "gameplane.operatorEnabled" .) "true") (not (or .Values.operator.leaderElect (gt (int .Values.operator.replicas) 1))) }} {{- $existing := lookup "apps/v1" "Deployment" .Release.Namespace "gameplane-operator" }} {{- if $existing }} {{- $name := printf "%s-op-roll" .Release.Name }} diff --git a/charts/gameplane/templates/operator.yaml b/charts/gameplane/templates/operator.yaml index 0111536df..d14eb7c4c 100644 --- a/charts/gameplane/templates/operator.yaml +++ b/charts/gameplane/templates/operator.yaml @@ -239,7 +239,7 @@ metadata: labels: {{- include "gameplane.labels" . | nindent 4 }} spec: replicas: {{ .Values.operator.replicas }} - {{- if not .Values.operator.leaderElect }} + {{- if not (or .Values.operator.leaderElect (gt (int .Values.operator.replicas) 1)) }} # Without leader election, prevent a rolling update from overlapping writers. strategy: type: Recreate @@ -274,7 +274,7 @@ spec: imagePullPolicy: {{ .Values.image.pullPolicy }} args: {{- include "gameplane.auxiliaryResourceArgs" .Values.operator | nindent 12 }} - {{- if .Values.operator.leaderElect }} + {{- if or .Values.operator.leaderElect (gt (int .Values.operator.replicas) 1) }} - --leader-elect {{- end }} - --agent-image={{ include "gameplane.agentImage" . }} diff --git a/charts/gameplane/templates/validate.yaml b/charts/gameplane/templates/validate.yaml index ace7eba13..bf4a4580f 100644 --- a/charts/gameplane/templates/validate.yaml +++ b/charts/gameplane/templates/validate.yaml @@ -1,6 +1,3 @@ -{{- if and (eq (include "gameplane.operatorEnabled" .) "true") (gt (int .Values.operator.replicas) 1) (not .Values.operator.leaderElect) }} -{{- fail "operator.leaderElect must be enabled with more than one replica" }} -{{- end }} {{- $network := .Values.networkPolicies | default dict }} {{- if $network.strict }} {{- if not $network.enabled }} diff --git a/charts/gameplane/tests/monitoring-rules.test.yaml b/charts/gameplane/tests/monitoring-rules.test.yaml index 3bd63c84f..2d285dedf 100644 --- a/charts/gameplane/tests/monitoring-rules.test.yaml +++ b/charts/gameplane/tests/monitoring-rules.test.yaml @@ -60,7 +60,13 @@ tests: - series: 'gameplane_agent_performance_available{namespace="gameplane-games",pod="game-0",metric="tps",signal="tick_rate",unit="hertz"}' values: '1x10 0x10' - series: 'gameplane_tunnel_connection_known{namespace="gameplane-games",pod="playit"}' + values: '0x20' + - series: 'gameplane_tunnel_connection_supported{namespace="gameplane-games",pod="playit"}' + values: '0x20' + - series: 'gameplane_tunnel_connection_known{namespace="gameplane-games",pod="frp"}' values: '0x10 1x10' + - series: 'gameplane_tunnel_connection_supported{namespace="gameplane-games",pod="frp"}' + values: '1x20' - series: 'gameplane_tunnel_ready{namespace="gameplane-games",pod="playit"}' values: '0x10 1x10' alert_rule_test: @@ -80,7 +86,7 @@ tests: - eval_time: 6m alertname: GameplaneRelayHealthUnknown exp_alerts: - - exp_labels: {namespace: gameplane-games, pod: playit, severity: info} + - exp_labels: {namespace: gameplane-games, pod: frp, severity: info} exp_annotations: {summary: "Gameplane provider cannot confirm live relay health"} - eval_time: 6m alertname: GameplaneRelayUnavailable diff --git a/charts/tests/check_workloads.py b/charts/tests/check_workloads.py index a4c433528..9d8e4f860 100644 --- a/charts/tests/check_workloads.py +++ b/charts/tests/check_workloads.py @@ -39,6 +39,8 @@ def check_operator_migration(): assert not any(obj["kind"] == "Job" for obj in render(values, release=release, chart=chart)) values["operator"]["leaderElect"] = True assert not any(obj["kind"] == "Job" for obj in render(values, upgrade=True, release=release, chart=chart)) + values["operator"].update(replicas=2, leaderElect=False) + assert not any(obj["kind"] == "Job" for obj in render(values, upgrade=True, release=release, chart=chart)) def main(): @@ -81,7 +83,10 @@ def main(): assert overrides["--agent-resources"] == {"requests": {"cpu": "80m"}} render({"api": {"pdb": {"enabled": True, "minAvailable": 1}}}, failure="api.pdb must allow the singleton API to be evicted") render({"api": {"pdb": {"enabled": True, "maxUnavailable": 0}}}, failure="api.pdb must allow the singleton API to be evicted") - render({"operator": {"replicas": 2, "leaderElect": False}}, failure="operator.leaderElect must be enabled") + upgraded = render({"operator": {"replicas": 2, "leaderElect": False}}, upgrade=True) + operator = one(upgraded, "Deployment", "gameplane-operator") + assert operator["spec"]["strategy"]["type"] == "RollingUpdate" + assert "--leader-elect" in operator["spec"]["template"]["spec"]["containers"][0]["args"] single = render({"operator": {"leaderElect": False}}) assert one(single, "Deployment", "gameplane-operator")["spec"]["strategy"] == {"type": "Recreate", "rollingUpdate": None} removed = render({"operator": {"agentResources": None, "tunnelResources": {}}}) @@ -100,12 +105,14 @@ def main(): old_chart = Path(directory) / "chart" shutil.copytree(CHART, old_chart) old_values = yaml.safe_load((old_chart / "values.yaml").read_text()) + old_values["operator"].update(replicas=2, leaderElect=False) for key in ["agentResources", "sentinelResources", "tunnelResources", "backupResources", "pdb", *placement]: old_values["operator"].pop(key, None) (old_chart / "values.yaml").write_text(yaml.safe_dump(old_values)) old = render(chart=old_chart) args = one(old, "Deployment", "gameplane-operator")["spec"]["template"]["spec"]["containers"][0]["args"] assert not any(arg.startswith("--tunnel-resources=") for arg in args) + assert "--leader-elect" in args if __name__ == "__main__": diff --git a/docs/monitoring.md b/docs/monitoring.md index 34c6346ec..83a22b300 100644 --- a/docs/monitoring.md +++ b/docs/monitoring.md @@ -114,14 +114,22 @@ zero; value and timestamp remain absent. ## Relays and infrastructure Tunnel gauges read the same snapshot as readiness: `gameplane_tunnel_ready`, -`gameplane_tunnel_connection_known`, `gameplane_tunnel_connected`, +`gameplane_tunnel_connection_supported`, `gameplane_tunnel_connection_known`, `gameplane_tunnel_connected`, `gameplane_tunnel_forwards_registered`, `gameplane_tunnel_endpoint_assigned`, and `gameplane_tunnel_observation_timestamp_seconds`. `connected` and observation timestamp are absent when their evidence is unknown or stale. Readiness and registration flags mean *confirmed*, so zero does not invent a disconnected provider. The rules distinguish an unavailable relay from unknown health. -Pinned Playit cannot confirm live connectivity and therefore raises the -informational unknown-health alert; see [relay-health.md](relay-health.md). +Pinned Playit cannot confirm live connectivity: its support gauge is zero and +the informational unknown-health alert excludes it. Its known/connected metrics +still represent unknown evidence, without claiming a healthy relay. Supported +providers raise the info alert when their observations become unknown or stale; +an observed tsnet failure is unavailable and raises the warning-level outage +alert. Between Tailscale attempts and while a replacement joins, the supervisor +knows that no forwarding attempt is serving, so that unavailable state remains +known throughout backoff. It carries no provider observation timestamp. Fresh +observations resume when forwarding is established and retain their normal +staleness limit. See [relay-health.md](relay-health.md). Tailscale, frp and Playit all use the same metrics path. Metrics contain no credentials, login URLs, proxy names or provider error messages. diff --git a/docs/production-placement.md b/docs/production-placement.md index 175d23f3a..9bdd26dab 100644 --- a/docs/production-placement.md +++ b/docs/production-placement.md @@ -82,9 +82,10 @@ cause API downtime. A PDB is not an availability guarantee and does not prevent node failures. The persistent telemetry receiver is also a singleton; avoid a zero-unavailable budget unless intentionally accepting a blocked drain. -Keep operator leader election enabled with redundant replicas and rolling -updates. The chart rejects multiple replicas with leader election disabled; -with one replica and election disabled it uses Recreate to avoid overlapping +Redundant operator replicas use leader election and rolling updates. The chart +automatically enables election when replicas exceed one, even when an older +release retained `leaderElect: false`; upgrades continue rendering and install +notes explain the effective setting. With one replica and election disabled it uses Recreate to avoid overlapping reconcilers during upgrades. API replica/process restrictions remain unchanged. When switching an existing operator to Recreate, a pre-upgrade hook clears the diff --git a/docs/relay-health.md b/docs/relay-health.md index 6be7a1658..f44c4267f 100644 --- a/docs/relay-health.md +++ b/docs/relay-health.md @@ -5,7 +5,8 @@ On Pod TCP port 8091, `/healthz` reports that the wrapper's HTTP server is alive `/readyz` returns 200 only when the provider connection and configured forwards are confirmed, and `/status` returns a small JSON state without credentials or provider response text. The operator installs a startup probe on `/healthz` and -a readiness probe on `/readyz`. An external relay outage does not trigger a +a readiness probe on `/readyz` for frp. Playit uses `/healthz` for Pod readiness +because its pinned provider cannot supply live connection evidence. An external relay outage does not trigger a liveness restart of the wrapper. Provider checks run every two seconds with a two-second query deadline. Local @@ -42,6 +43,11 @@ Kubernetes probe/reconcile intervals for an outage to reach the dashboard. Upgrade the operator and tunnel image together. The new image requires `GAMESERVER_UID`, so a mixed old-operator/new-image installation fails closed. +The upgraded operator reconciles existing tunnel Deployments in place, replaces +their Pod templates with the UID environment variable and current probes, and +uses Recreate to retire the old client before starting its replacement. No +manual Deployment environment edit is required; upgrade the operator first or +together with its configured tunnel image. Existing unprefixed proxy names change to UID-prefixed names. Stop the old client, wait for frps to unregister its old proxies and release their public ports, or remove those registrations using the frps administrator's normal @@ -65,11 +71,12 @@ response therefore cannot prove that a relay is connected or refresh a health lease. Process age and the presence of an assigned address do not prove it either. -Gameplane keeps reporting assigned addresses, but Playit's `/readyz` returns +Gameplane keeps reporting assigned addresses, but Playit's relay `/readyz` returns 503 and `TunnelReady` reports `RelayConnectionUnknown`; before assignment it -reports `AwaitingAddress`. `/healthz` remains successful, the child keeps running -and forwarding when it can, and Kubernetes does not restart it solely for this -unknown connection state. The JSON status distinguishes known assignment from +reports `AwaitingAddress`. Its Kubernetes readiness probe uses `/healthz`, so a +running wrapper can become Pod Ready while its connection evidence remains +unknown. The child keeps running and forwarding when it can, and Kubernetes does +not restart it solely for this unknown connection state. The JSON status distinguishes known assignment from unknown connection. This is a deliberate limitation of the pinned provider, not a claim that every Playit tunnel is disconnected. diff --git a/docs/release-notes.md b/docs/release-notes.md index f3da01f11..ff07383fa 100644 --- a/docs/release-notes.md +++ b/docs/release-notes.md @@ -16,6 +16,9 @@ keys use the defaults even with `--reuse-values`. Chart-managed pods now accept placement, topology-spread and priority settings, with optional PDBs for Deployments. API remains a singleton and its PDB uses `maxUnavailable: 1`; node drains can cause API downtime. Operator replicas greater -than one require leader election. Disabling election on a singleton uses Recreate. +than one automatically enable leader election, including upgrades retaining +`leaderElect: false` through `--reuse-values`; that old setting no longer fails +chart rendering. Install notes explain the effective behavior. Disabling +election on a singleton uses Recreate. See [production placement](production-placement.md) for upgrade controls and a worker-pool example. diff --git a/i18n/locales/en/app.json b/i18n/locales/en/app.json index a39c85d1a..d450aa1e1 100644 --- a/i18n/locales/en/app.json +++ b/i18n/locales/en/app.json @@ -607,6 +607,10 @@ "defaultMessage": "Tailscale forwards only advertised TCP and UDP ports. Remove unsupported protocols from the advertised ports.", "description": "TunnelReady condition when a Tailscale template advertises a protocol other than TCP or UDP." }, + "condition.tunnelready.invalidtailscaletags": { + "defaultMessage": "Correct the Tailscale tags: use an optional tag: prefix, a letter first, then only letters, digits or hyphens.", + "description": "TunnelReady condition when Tailscale access-control tags have invalid syntax." + }, "condition.unquiesced.failed": { "defaultMessage": "The server could not be resumed after the backup. Its world may still have auto-save turned off.", "description": "Unquiesced condition when the unfreeze failed." diff --git a/operator/hack/condition-ids/main.go b/operator/hack/condition-ids/main.go index cf7502c17..364afe085 100644 --- a/operator/hack/condition-ids/main.go +++ b/operator/hack/condition-ids/main.go @@ -122,6 +122,7 @@ var rows = []row{ // GameServer: tunnel and other {types: []string{"TunnelReady"}, reason: "UnsupportedProtocol", english: "Tailscale forwards only advertised TCP and UDP ports. Remove unsupported protocols from the advertised ports.", description: "TunnelReady condition when a Tailscale template advertises a protocol other than TCP or UDP."}, + {types: []string{"TunnelReady"}, reason: "InvalidTailscaleTags", english: "Correct the Tailscale tags: use an optional tag: prefix, a letter first, then only letters, digits or hyphens.", description: "TunnelReady condition when Tailscale access-control tags have invalid syntax."}, {types: []string{"TunnelReady"}, reason: "InvalidConfig", english: "{provider, select, none {The tunnel is enabled but the server has no tunnel configuration.} other {The ''{provider}'' tunnel provider is selected but its configuration is missing.}}", description: "TunnelReady condition for an incomplete tunnel setup. provider is the selected provider, or none."}, {types: []string{"TunnelReady"}, reason: "DeploymentNotReady", english: "Waiting for the tunnel deployment to become ready.", description: "TunnelReady condition while the tunnel pod is not ready."}, {types: []string{"TunnelReady"}, reason: "RelayConnectionUnknown", english: "The relay connection cannot be confirmed. Check the provider's connection status and the tunnel deployment's readiness probe.", description: "TunnelReady condition when provider connection evidence is missing or unsupported."}, diff --git a/operator/internal/controller/gameserver_controller.go b/operator/internal/controller/gameserver_controller.go index 9338ac693..be4d63d21 100644 --- a/operator/internal/controller/gameserver_controller.go +++ b/operator/internal/controller/gameserver_controller.go @@ -266,6 +266,11 @@ func (r *GameServerReconciler) Reconcile(ctx context.Context, req ctrl.Request) return ctrl.Result{}, err } + if tmpl.Spec.SentinelIsolation && len(networkPolicyPortsFromTemplate(&tmpl)) == 0 { + return ctrl.Result{}, r.setPhase(ctx, &gs, + "sentinelIsolation requires at least one advertised port; advertise a game port or disable sentinelIsolation in the GameTemplate") + } + // Resolve spec.config against the template's configSchema before // touching any children: invalid config must fail loudly (mirroring // the missing-template path) instead of materializing a pod that diff --git a/operator/internal/controller/gameserver_relay_health_test.go b/operator/internal/controller/gameserver_relay_health_test.go index edbce76a0..9e9628789 100644 --- a/operator/internal/controller/gameserver_relay_health_test.go +++ b/operator/internal/controller/gameserver_relay_health_test.go @@ -98,6 +98,59 @@ func TestRelayConditionsRequireCurrentProviderProbe(t *testing.T) { assert(metav1.ConditionUnknown) } +func TestExistingFrpDeploymentGainsUIDDuringReconciliation(t *testing.T) { + t.Parallel() + s := wipeScheme(t) + gs := &gameplanev1alpha1.GameServer{ObjectMeta: metav1.ObjectMeta{Name: "legacy", Namespace: "games", UID: "legacy-uid"}, Spec: gameplanev1alpha1.GameServerSpec{Networking: gameplanev1alpha1.GameServerNetworking{Tunnel: &gameplanev1alpha1.GameServerTunnel{Enabled: true, Provider: "frp", Frp: &gameplanev1alpha1.FrpTunnelSpec{ServerAddr: "relay"}}}}} + old := &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "legacy-tunnel", Namespace: "games", UID: "deployment-uid"}, Spec: appsv1.DeploymentSpec{Strategy: appsv1.DeploymentStrategy{Type: appsv1.RollingUpdateDeploymentStrategyType, RollingUpdate: &appsv1.RollingUpdateDeployment{}}, Template: corev1.PodTemplateSpec{Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "tunnel", Image: "old", Env: []corev1.EnvVar{{Name: "TUNNEL_TYPE", Value: "frp"}}}}}}}} + c := fake.NewClientBuilder().WithScheme(s).WithObjects(gs, old).Build() + r := &GameServerReconciler{Client: c, Scheme: s} + for i := 0; i < 2; i++ { + if err := r.reconcileTunnel(t.Context(), gs, singlePortTemplate(true, "minecraft"), true); err != nil { + t.Fatal(err) + } + var updated appsv1.Deployment + if err := c.Get(t.Context(), client.ObjectKeyFromObject(old), &updated); err != nil { + t.Fatal(err) + } + if updated.UID != old.UID || updated.Spec.Strategy.Type != appsv1.RecreateDeploymentStrategyType || updated.Spec.Strategy.RollingUpdate != nil { + t.Fatalf("legacy Deployment was not migrated in place: %+v", updated.Spec.Strategy) + } + found := false + for _, env := range updated.Spec.Template.Spec.Containers[0].Env { + if env.Name == "GAMESERVER_UID" { + found = env.Value == string(gs.UID) + } + } + if !found || !hasRelayReadinessProbe(&updated) { + t.Fatal("existing Deployment lacks UID or forwarding probe") + } + } +} + +func TestPlayitPodReadinessUsesAvailableProcessSignal(t *testing.T) { + t.Parallel() + s := wipeScheme(t) + gs := &gameplanev1alpha1.GameServer{ObjectMeta: metav1.ObjectMeta{Name: "playit", Namespace: "games", UID: "playit-uid"}, Spec: gameplanev1alpha1.GameServerSpec{Networking: gameplanev1alpha1.GameServerNetworking{Tunnel: &gameplanev1alpha1.GameServerTunnel{Enabled: true, Provider: "playit", Playit: &gameplanev1alpha1.PlayitTunnelSpec{}}}}} + c := fake.NewClientBuilder().WithScheme(s).WithObjects(gs).Build() + r := &GameServerReconciler{Client: c, Scheme: s} + if err := r.reconcileTunnel(t.Context(), gs, singlePortTemplate(true, "minecraft"), true); err != nil { + t.Fatal(err) + } + var dep appsv1.Deployment + if err := c.Get(t.Context(), client.ObjectKey{Namespace: gs.Namespace, Name: gs.Name + "-tunnel"}, &dep); err != nil { + t.Fatal(err) + } + probe := dep.Spec.Template.Spec.Containers[0].ReadinessProbe + if probe == nil || probe.HTTPGet == nil || probe.HTTPGet.Path != "/healthz" || probe.HTTPGet.Port.IntValue() != 8091 { + t.Fatalf("Playit requires unsupported connection evidence: %+v", probe) + } + condition := meta.FindStatusCondition(computeTunnelConditions(gs, tunnelPlan{wantTunnel: true}, &dep), "TunnelReady") + if condition == nil || condition.Status == metav1.ConditionTrue { + t.Fatalf("process readiness implied relay readiness: %+v", condition) + } +} + func TestPlayitAssignmentDoesNotImplyRelayReady(t *testing.T) { t.Parallel() gs := &gameplanev1alpha1.GameServer{Spec: gameplanev1alpha1.GameServerSpec{Networking: gameplanev1alpha1.GameServerNetworking{Tunnel: &gameplanev1alpha1.GameServerTunnel{Provider: "playit", Playit: &gameplanev1alpha1.PlayitTunnelSpec{}, CredentialsSecretRef: &gameplanev1alpha1.SecretNameRef{Name: "relay-credentials"}}}}} @@ -112,15 +165,18 @@ func TestPlayitAssignmentDoesNotImplyRelayReady(t *testing.T) { dep.Status.ObservedGeneration = 1 dep.Status.UpdatedReplicas = 1 dep.Spec.Template.Spec.Containers = []corev1.Container{{Name: "tunnel", ReadinessProbe: &corev1.Probe{ProbeHandler: corev1.ProbeHandler{HTTPGet: &corev1.HTTPGetAction{Path: "/readyz", Port: intstr.FromInt32(8091)}}}}} - for _, endpoints := range [][]gameplanev1alpha1.GameServerEndpoint{nil, {{Name: "game", Host: "example.playit.gg", Port: 30000}}} { - gs.Status.TunnelEndpoints = endpoints - cond := meta.FindStatusCondition(computeTunnelConditions(gs, tunnelPlan{wantTunnel: true}, dep), "TunnelReady") - wantStatus, wantReason := metav1.ConditionUnknown, "RelayConnectionUnknown" - if len(endpoints) == 0 { - wantStatus, wantReason = metav1.ConditionFalse, "AwaitingAddress" - } - if cond == nil || cond.Status != wantStatus || cond.Reason != wantReason { - t.Fatalf("playit readiness=%+v, want %s/%s", cond, wantStatus, wantReason) + for _, probePath := range []string{"/readyz", "/healthz"} { + dep.Spec.Template.Spec.Containers[0].ReadinessProbe.HTTPGet.Path = probePath + for _, endpoints := range [][]gameplanev1alpha1.GameServerEndpoint{nil, {{Name: "game", Host: "example.playit.gg", Port: 30000}}} { + gs.Status.TunnelEndpoints = endpoints + cond := meta.FindStatusCondition(computeTunnelConditions(gs, tunnelPlan{wantTunnel: true}, dep), "TunnelReady") + wantStatus, wantReason := metav1.ConditionUnknown, "RelayConnectionUnknown" + if len(endpoints) == 0 { + wantStatus, wantReason = metav1.ConditionFalse, "AwaitingAddress" + } + if cond == nil || cond.Status != wantStatus || cond.Reason != wantReason { + t.Fatalf("playit readiness=%+v, want %s/%s", cond, wantStatus, wantReason) + } } } } diff --git a/operator/internal/controller/gameserver_status.go b/operator/internal/controller/gameserver_status.go index 19ecbe8d4..dcc1454ec 100644 --- a/operator/internal/controller/gameserver_status.go +++ b/operator/internal/controller/gameserver_status.go @@ -1116,6 +1116,12 @@ func computeTunnelConditions( } // Tunnel Deployment not yet created or not ready. + if tunnel.Provider == "tailscale" && tunnel.Tailscale != nil && !validTailscaleTags(tunnel.Tailscale.Tags) { + tunnelReady.Status = metav1.ConditionFalse + tunnelReady.Reason = "InvalidTailscaleTags" + tunnelReady.Message = "Tailscale tags must be optional tag: prefixes followed by a letter and only letters, digits or hyphens; correct spec.networking.tunnel.tailscale.tags" + return upsertCondition(conds, tunnelReady) + } if tunnel.Provider == "tailscale" && len(plan.unsupportedPorts) > 0 { tunnelReady.Status = metav1.ConditionFalse tunnelReady.Reason = "UnsupportedProtocol" @@ -1139,7 +1145,11 @@ func computeTunnelConditions( return conds } if tunnel.Provider == "frp" || tunnel.Provider == "playit" { - if !hasRelayReadinessProbe(tunnelDep) || tunnelDep.Status.ObservedGeneration < tunnelDep.Generation || tunnelDep.Status.UpdatedReplicas == 0 { + probePresent := hasRelayReadinessProbe(tunnelDep) + if tunnel.Provider == "playit" { + probePresent = probePresent || hasTunnelReadinessProbe(tunnelDep, "/healthz") + } + if !probePresent || tunnelDep.Status.ObservedGeneration < tunnelDep.Generation || tunnelDep.Status.UpdatedReplicas == 0 { tunnelReady.Status = metav1.ConditionUnknown tunnelReady.Reason = "RelayConnectionUnknown" tunnelReady.Message = "waiting for the current relay deployment and its connection readiness probe" @@ -1241,10 +1251,14 @@ func computeTunnelConditions( } func hasRelayReadinessProbe(dep *appsv1.Deployment) bool { + return hasTunnelReadinessProbe(dep, "/readyz") +} + +func hasTunnelReadinessProbe(dep *appsv1.Deployment, path string) bool { for _, container := range dep.Spec.Template.Spec.Containers { if container.Name == "tunnel" && container.ReadinessProbe != nil && container.ReadinessProbe.HTTPGet != nil { probe := container.ReadinessProbe.HTTPGet - return probe.Path == "/readyz" && probe.Port.IntValue() == 8091 + return probe.Path == path && probe.Port.IntValue() == 8091 } } return false diff --git a/operator/internal/controller/gameserver_tailscale_test.go b/operator/internal/controller/gameserver_tailscale_test.go index 597c60b9a..a33e3eede 100644 --- a/operator/internal/controller/gameserver_tailscale_test.go +++ b/operator/internal/controller/gameserver_tailscale_test.go @@ -6,9 +6,12 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" ) @@ -50,3 +53,49 @@ func TestTailscaleAdvertisedProtocolsAndReadiness(t *testing.T) { t.Fatalf("unsupported protocol condition=%+v, want reason UnsupportedProtocol", condition) } } + +func TestInvalidTailscaleTagsExposeConditionAndRemoveOldDeployment(t *testing.T) { + t.Parallel() + s := wipeScheme(t) + gs := &gameplanev1alpha1.GameServer{ObjectMeta: metav1.ObjectMeta{Name: "game", Namespace: "games", UID: "game-uid"}, Spec: gameplanev1alpha1.GameServerSpec{Networking: gameplanev1alpha1.GameServerNetworking{Tunnel: &gameplanev1alpha1.GameServerTunnel{Enabled: true, Provider: "tailscale", Tailscale: &gameplanev1alpha1.TailscaleTunnelSpec{Tags: []string{"tag:bad tag"}}}}}} + old := &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "game-tunnel", Namespace: "games"}} + c := fake.NewClientBuilder().WithScheme(s).WithObjects(gs, old).Build() + r := &GameServerReconciler{Client: c, Scheme: s} + tmpl := singlePortTemplate(true, "minecraft") + plan := r.planTunnel(t.Context(), gs, tmpl) + if err := r.reconcileTunnel(t.Context(), gs, tmpl, plan.wantTunnel); err != nil { + t.Fatal(err) + } + if err := c.Get(t.Context(), client.ObjectKeyFromObject(old), old); !apierrors.IsNotFound(err) { + t.Fatalf("old tunnel remains: %v", err) + } + condition := meta.FindStatusCondition(computeTunnelConditions(gs, plan, nil), "TunnelReady") + if condition == nil || condition.Status != metav1.ConditionFalse || condition.Reason != "InvalidTailscaleTags" { + t.Fatalf("invalid tags hidden: %+v", condition) + } + gs.Spec.Networking.Tunnel.Tailscale.Tags = []string{" game ", "tag:friends"} + if err := r.reconcileTunnel(t.Context(), gs, tmpl, true); err != nil { + t.Fatal(err) + } + if err := c.Get(t.Context(), client.ObjectKeyFromObject(old), old); err != nil { + t.Fatalf("corrected tags did not recreate deployment: %v", err) + } + condition = meta.FindStatusCondition(computeTunnelConditions(gs, plan, old), "TunnelReady") + if condition == nil || condition.Reason == "InvalidTailscaleTags" { + t.Fatalf("invalid tag condition survived correction: %+v", condition) + } +} + +func TestTailscaleTagValidationMatchesSupervisorSyntax(t *testing.T) { + t.Parallel() + for _, tags := range [][]string{nil, {""}, {"game", " tag:Friends-2 "}, {"game,,tag:game,"}} { + if !validTailscaleTags(tags) { + t.Fatalf("valid tags refused: %v", tags) + } + } + for _, tags := range [][]string{{"tag:"}, {"user:game"}, {"tag:9game"}, {"tag:bad tag"}, {"tag:gäme"}, {"good", "bad_name"}} { + if validTailscaleTags(tags) { + t.Fatalf("invalid tags accepted: %v", tags) + } + } +} diff --git a/operator/internal/controller/gameserver_tunnel.go b/operator/internal/controller/gameserver_tunnel.go index 8a6c10aed..7764240c8 100644 --- a/operator/internal/controller/gameserver_tunnel.go +++ b/operator/internal/controller/gameserver_tunnel.go @@ -3,6 +3,7 @@ package controller import ( "context" "fmt" + "regexp" "strings" appsv1 "k8s.io/api/apps/v1" @@ -26,6 +27,20 @@ const ( tunnelAuthMountDir = "/etc/gameplane/tunnel-auth" ) +var tailscaleTagPattern = regexp.MustCompile(`^(tag:)?[A-Za-z][A-Za-z0-9-]*$`) + +// validTailscaleTags mirrors the supervisor's normalization before it can +// register a node. Invalid configuration must be visible before Pod startup. +func validTailscaleTags(tags []string) bool { + for _, tag := range strings.Split(strings.Join(tags, ","), ",") { + tag = strings.TrimSpace(tag) + if tag != "" && !tailscaleTagPattern.MatchString(tag) { + return false + } + } + return true +} + // tunnelCredentialKey returns the Secret key name for a given tunnel provider. // Must match the keys in tunnel/main.go allowedCredentialKeys and // api/internal/handlers/tunnelcreds.go credentialKeysByProvider. @@ -188,6 +203,14 @@ func (r *GameServerReconciler) reconcileTunnel( if tunnel == nil { return r.clearTunnelCredentialRefused(ctx, gs) } + if tunnel.Provider == "tailscale" && tunnel.Tailscale != nil && !validTailscaleTags(tunnel.Tailscale.Tags) { + // Stop a previous configuration rather than crash-looping a replacement + // or registering without the requested access-control tags. + if err := r.deleteTunnel(ctx, gs.Namespace, gs.Name); err != nil { + return err + } + return r.clearTunnelCredentialRefused(ctx, gs) + } // Create or update the tunnel Deployment. dep := &appsv1.Deployment{ @@ -434,6 +457,11 @@ func (r *GameServerReconciler) reconcileTunnel( container := &dep.Spec.Template.Spec.Containers[0] container.StartupProbe = &corev1.Probe{ProbeHandler: corev1.ProbeHandler{HTTPGet: &corev1.HTTPGetAction{Path: "/healthz", Port: intstr.FromInt32(8091)}}, PeriodSeconds: 2, TimeoutSeconds: 1, FailureThreshold: 30} container.ReadinessProbe = &corev1.Probe{ProbeHandler: corev1.ProbeHandler{HTTPGet: &corev1.HTTPGetAction{Path: "/readyz", Port: intstr.FromInt32(8091)}}, PeriodSeconds: 2, TimeoutSeconds: 1, FailureThreshold: 1} + if tunnel.Provider == "playit" { + // The pinned Playit IPC has no live connection signal. Pod + // readiness uses process health; TunnelReady remains Unknown. + container.ReadinessProbe.HTTPGet.Path = "/healthz" + } } return controllerutil.SetControllerReference(gs, dep, r.Scheme) diff --git a/operator/internal/controller/sentinel_isolation.go b/operator/internal/controller/sentinel_isolation.go index 6cd2969bd..c01e161f1 100644 --- a/operator/internal/controller/sentinel_isolation.go +++ b/operator/internal/controller/sentinel_isolation.go @@ -4,7 +4,10 @@ import ( "context" "fmt" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" networkingv1 "k8s.io/api/networking/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" @@ -17,6 +20,45 @@ const sentinelIsolationLabel = "gameplane.local/sentinel-isolation" // The chart's legacy policies exclude opted-in pods on both sides. Policies // are additive: these per-server grants cannot subtract a namespace-wide grant. func (r *GameServerReconciler) reconcileSentinelIsolation(ctx context.Context, gs *gameplanev1alpha1.GameServer, tmpl *gameplanev1alpha1.GameTemplate) error { + if !tmpl.Spec.SentinelIsolation { + // Workload reconciliation removes the opt-in labels by rolling both + // pod templates. Keep both grants until every live pod has switched + // back to the legacy policies, including during a partial rollout. + reader := r.APIReader + if reader == nil { + reader = r.Client + } + // An empty Pod list alone is insufficient: an old workload template + // could still create labeled replacements after that observation. + var game appsv1.StatefulSet + if err := reader.Get(ctx, client.ObjectKey{Namespace: gs.Namespace, Name: gs.Name}, &game); err != nil { + if !apierrors.IsNotFound(err) { + return err + } + } else if game.Spec.Template.Labels[sentinelIsolationLabel] == "true" || game.Status.ObservedGeneration < game.Generation || game.Status.CurrentRevision != game.Status.UpdateRevision { + return nil + } + var sentinel appsv1.Deployment + if err := reader.Get(ctx, client.ObjectKey{Namespace: gs.Namespace, Name: gs.Name + "-waker"}, &sentinel); err != nil { + if !apierrors.IsNotFound(err) { + return err + } + } else if sentinel.Spec.Template.Labels[sentinelIsolationLabel] == "true" || sentinel.Status.ObservedGeneration < sentinel.Generation || sentinel.Status.UpdatedReplicas != sentinel.Status.Replicas { + return nil + } + var pods corev1.PodList + if err := reader.List(ctx, &pods, client.InNamespace(gs.Namespace), client.MatchingLabels{ + "app.kubernetes.io/instance": gs.Name, sentinelIsolationLabel: "true", + }); err != nil { + return err + } + for i := range pods.Items { + name := pods.Items[i].Labels["app.kubernetes.io/name"] + if name == "gameplane-game" || name == "gameplane-waker" { + return nil + } + } + } ports := networkPolicyPortsFromTemplate(tmpl) for _, ingress := range []bool{true, false} { suffix, target, peer := "egress", "gameplane-waker", "gameplane-game" diff --git a/operator/internal/controller/sentinel_isolation_test.go b/operator/internal/controller/sentinel_isolation_test.go index 1db7f3587..d28f70f9e 100644 --- a/operator/internal/controller/sentinel_isolation_test.go +++ b/operator/internal/controller/sentinel_isolation_test.go @@ -1,12 +1,18 @@ package controller import ( + "context" + "strings" "testing" + appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" networkingv1 "k8s.io/api/networking/v1" + "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" @@ -15,6 +21,8 @@ import ( func TestSentinelIsolationPolicies(t *testing.T) { scheme := runtime.NewScheme() + _ = corev1.AddToScheme(scheme) + _ = appsv1.AddToScheme(scheme) if err := networkingv1.AddToScheme(scheme); err != nil { t.Fatal(err) } @@ -91,6 +99,8 @@ func TestSentinelIsolationPolicies(t *testing.T) { func TestSentinelIsolationDoesNotDeleteForeignPolicy(t *testing.T) { scheme := runtime.NewScheme() + _ = corev1.AddToScheme(scheme) + _ = appsv1.AddToScheme(scheme) _ = networkingv1.AddToScheme(scheme) _ = gameplanev1alpha1.AddToScheme(scheme) foreign := &networkingv1.NetworkPolicy{ObjectMeta: metav1.ObjectMeta{Name: "alpha-sentinel-ingress", Namespace: "games"}} @@ -104,3 +114,121 @@ func TestSentinelIsolationDoesNotDeleteForeignPolicy(t *testing.T) { t.Fatal("deleted foreign policy", err) } } + +type staleSentinelPodClient struct{ client.Client } + +func (c staleSentinelPodClient) List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error { + if pods, ok := list.(*corev1.PodList); ok { + pods.Items = nil + return nil + } + return c.Client.List(ctx, list, opts...) +} + +func TestSentinelIsolationOptOutWaitsForBothLivePodLabels(t *testing.T) { + scheme := testScheme(t) + _ = networkingv1.AddToScheme(scheme) + c := fake.NewClientBuilder().WithScheme(scheme).Build() + r := &GameServerReconciler{Client: staleSentinelPodClient{c}, APIReader: c, Scheme: scheme} + gs := &gameplanev1alpha1.GameServer{ObjectMeta: metav1.ObjectMeta{Name: "alpha", Namespace: "games", UID: "original"}} + tmpl := singlePortTemplate(true, "minecraft") + tmpl.Spec.SentinelIsolation = true + if err := r.reconcileSentinelIsolation(t.Context(), gs, tmpl); err != nil { + t.Fatal(err) + } + pods := []*corev1.Pod{ + {ObjectMeta: metav1.ObjectMeta{Name: "alpha-0", Namespace: gs.Namespace, Labels: map[string]string{"app.kubernetes.io/name": "gameplane-game", "app.kubernetes.io/instance": gs.Name, sentinelIsolationLabel: "true"}}}, + {ObjectMeta: metav1.ObjectMeta{Name: "alpha-waker", Namespace: gs.Namespace, Labels: map[string]string{"app.kubernetes.io/name": "gameplane-waker", "app.kubernetes.io/instance": gs.Name, sentinelIsolationLabel: "true"}}}, + } + for _, pod := range pods { + if err := c.Create(t.Context(), pod); err != nil { + t.Fatal(err) + } + } + tmpl.Spec.SentinelIsolation = false + for remaining := 2; remaining >= 0; remaining-- { + if err := r.reconcileSentinelIsolation(t.Context(), gs, tmpl); err != nil { + t.Fatal(err) + } + var policies networkingv1.NetworkPolicyList + if err := c.List(t.Context(), &policies); err != nil { + t.Fatal(err) + } + want := 2 + if remaining == 0 { + want = 0 + } + if len(policies.Items) != want { + t.Fatalf("%d labeled pods remain: want %d grants, got %d", remaining, want, len(policies.Items)) + } + if remaining > 0 { + pod := pods[2-remaining] + delete(pod.Labels, sentinelIsolationLabel) + if err := c.Update(t.Context(), pod); err != nil { + t.Fatal(err) + } + } + } +} + +func TestSentinelIsolationRejectsNoAdvertisedPortsBeforeCreatingWorkloads(t *testing.T) { + scheme := testScheme(t) + _ = batchv1.AddToScheme(scheme) + tmpl := singlePortTemplate(false, "none") + tmpl.Name = "isolated" + tmpl.Spec.SentinelIsolation = true + tmpl.Spec.Ports = []gameplanev1alpha1.GamePort{{Name: "management", ContainerPort: 25575, Protocol: corev1.ProtocolTCP}} + gs := &gameplanev1alpha1.GameServer{ObjectMeta: metav1.ObjectMeta{Name: "alpha", Namespace: "games", UID: "original"}} + gs.Spec.TemplateRef.Name = tmpl.Name + c := fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(gs).WithObjects(gs, tmpl).Build() + r := &GameServerReconciler{Client: c, APIReader: c, Scheme: scheme} + if _, err := r.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(gs)}); err != nil { + t.Fatal(err) + } + if err := c.Get(t.Context(), client.ObjectKeyFromObject(gs), gs); err != nil { + t.Fatal(err) + } + condition := meta.FindStatusCondition(gs.Status.Conditions, "Ready") + if gs.Status.Phase != gameplanev1alpha1.GameServerPhaseFailed || condition == nil || condition.Status != metav1.ConditionFalse || !strings.Contains(condition.Message, "sentinelIsolation requires at least one advertised port") { + t.Fatalf("missing actionable isolation failure: %+v", gs.Status) + } + var workloads appsv1.StatefulSetList + if err := c.List(t.Context(), &workloads); err != nil || len(workloads.Items) != 0 { + t.Fatalf("invalid template created workloads: %v %+v", err, workloads) + } +} + +func TestSentinelIsolationOptOutWaitsForWorkloadControllers(t *testing.T) { + scheme := testScheme(t) + _ = networkingv1.AddToScheme(scheme) + for _, test := range []struct { + name string + object client.Object + }{ + {"game-old-template", &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{Name: "alpha", Namespace: "games"}, Spec: appsv1.StatefulSetSpec{Template: corev1.PodTemplateSpec{ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{sentinelIsolationLabel: "true"}}}}}}, + {"game-unobserved", &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{Name: "alpha", Namespace: "games", Generation: 2}, Status: appsv1.StatefulSetStatus{ObservedGeneration: 1}}}, + {"game-old-revision", &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{Name: "alpha", Namespace: "games"}, Status: appsv1.StatefulSetStatus{CurrentRevision: "old", UpdateRevision: "new"}}}, + {"sentinel-old-template", &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "alpha-waker", Namespace: "games"}, Spec: appsv1.DeploymentSpec{Template: corev1.PodTemplateSpec{ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{sentinelIsolationLabel: "true"}}}}}}, + {"sentinel-unobserved", &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "alpha-waker", Namespace: "games", Generation: 2}, Status: appsv1.DeploymentStatus{ObservedGeneration: 1}}}, + {"sentinel-old-replicas", &appsv1.Deployment{ObjectMeta: metav1.ObjectMeta{Name: "alpha-waker", Namespace: "games"}, Status: appsv1.DeploymentStatus{Replicas: 2, UpdatedReplicas: 1}}}, + } { + t.Run(test.name, func(t *testing.T) { + c := fake.NewClientBuilder().WithScheme(scheme).WithObjects(test.object).Build() + r := &GameServerReconciler{Client: c, APIReader: c, Scheme: scheme} + gs := &gameplanev1alpha1.GameServer{ObjectMeta: metav1.ObjectMeta{Name: "alpha", Namespace: "games", UID: "original"}} + tmpl := singlePortTemplate(true, "minecraft") + tmpl.Spec.SentinelIsolation = true + if err := r.reconcileSentinelIsolation(t.Context(), gs, tmpl); err != nil { + t.Fatal(err) + } + tmpl.Spec.SentinelIsolation = false + if err := r.reconcileSentinelIsolation(t.Context(), gs, tmpl); err != nil { + t.Fatal(err) + } + var policies networkingv1.NetworkPolicyList + if err := c.List(t.Context(), &policies); err != nil || len(policies.Items) != 2 { + t.Fatalf("old workload could recreate labeled pods without grants: %v %+v", err, policies.Items) + } + }) + } +} diff --git a/operator/specs.md b/operator/specs.md index 155b5eaff..2a0e1628c 100644 --- a/operator/specs.md +++ b/operator/specs.md @@ -46,6 +46,18 @@ not recreated to enable remote parity. ## Non-goals / boundaries +Tailscale tags are normalized and syntax-checked before tunnel Deployment creation. +Invalid tags remove a previous Deployment and report `TunnelReady=False` with +`InvalidTailscaleTags`, naming the configuration field to correct. Correcting +the tags permits normal creation again; denied provider authorization never +falls back to an untagged identity. + +Playit Pod readiness uses the wrapper's `/healthz` endpoint because the pinned +provider cannot confirm a live relay connection. This never promotes +`TunnelReady` to True: relay evidence stays Unknown, or AwaitingAddress before +assignment. Existing frp Deployments receive `GAMESERVER_UID`, current probes +and Recreate strategy automatically on reconciliation after an operator upgrade. + The operator **does not** handle: - User authentication or authorization (RBAC is Kubernetes role-based, not application-level). - REST API, WebSocket, or dashboard concerns (the API server, not the operator, owns the UX layer). @@ -178,7 +190,7 @@ Primary reconcilers register with the manager in `cmd/main.go` and handle CRD li - Graceful stop: if template has Lifecycle.Stop sequence, orchestrate via agent before scaling down. - Node affinity: honor spec.nodeSelector and spec.affinity preferences. - Ingress NetworkPolicy: enforce per-template advertised ports, admit CIDRs from `--game-ingress-from-cidr`. - - Sentinel NetworkPolicies: `GameTemplate.spec.sentinelIsolation` defaults false. When true, reconcile owned `-sentinel-ingress` and `-sentinel-egress` policies joining only the same namespace/server instance on advertised protocol/port tuples. Zero advertised ports removes owned grants. Set `gameplane.local/sentinel-isolation=true` on both pod templates, without changing immutable workload selectors. The matching chart excludes opted-in sources and destinations from legacy grants; other policies remain additive. + - Sentinel NetworkPolicies: `GameTemplate.spec.sentinelIsolation` defaults false. When true, reconcile owned `-sentinel-ingress` and `-sentinel-egress` policies joining only the same namespace/server instance on advertised protocol/port tuples. An opted-in template with zero advertised ports fails before child reconciliation with an actionable `Ready=False` condition; no unrestricted grant is created. Set `gameplane.local/sentinel-isolation=true` on both pod templates, without changing immutable workload selectors. On opt-out, roll the labels off both workloads and retain the per-server grants until uncached reads confirm both controllers observed the unlabeled templates and completed their rollouts, and neither game nor sentinel pods still carry the label. The matching chart excludes opted-in sources and destinations from legacy grants; other policies remain additive. - Fixed-name child cleanup: Delete fixed-name objects (`-config` Secret, `-files` Secret, `-rcon` Secret, `-auto` BackupSchedule, `-wipe` Job) only when the GameServer controls them via OwnerReference; unowned objects with these names stay in place (same rule as game-ingress NetworkPolicy). Wipe cleanup carries the verified Job UID as a delete precondition; a foreign Job cannot acknowledge a pending wipe or resume its server. - **Conflict requeue:** When reconcileStatefulSet or reconcileWipe encounters an apierrors.IsConflict error (concurrent update races), requeue without error logging — conflicts are expected and transient. The Module, NetworkCapture and Backup reconcilers apply the same rule to every error they return, via `requeueOnConflict` (`conflict.go`). - Load-balancer address management: translate spec.networking.addressPool / .address preferences onto the Service based on the cluster's address-manager flavor (MetalLB, Cilium, or none), report assignment status via the AddressAssignment condition. diff --git a/tunnel/main.go b/tunnel/main.go index a9cb48140..1c8844122 100644 --- a/tunnel/main.go +++ b/tunnel/main.go @@ -231,7 +231,7 @@ func run(ctx context.Context, cfg Config) error { }() } - h := &relayHealth{} + h := &relayHealth{connectionUnsupported: cfg.TunnelType == "playit"} if ctx.Err() != nil { return nil } diff --git a/tunnel/relay_health.go b/tunnel/relay_health.go index 290efb25b..15a0b7984 100644 --- a/tunnel/relay_health.go +++ b/tunnel/relay_health.go @@ -39,7 +39,10 @@ type relaySnapshot struct { } type relayHealth struct { + // Immutable provider capability, independent of fresh observation state. + connectionUnsupported bool mu sync.Mutex + supervisorUnavailable bool observation relayObservation checked, healthySince time.Time } @@ -47,10 +50,21 @@ type relayHealth struct { func (h *relayHealth) clear() { h.mu.Lock() defer h.mu.Unlock() + h.supervisorUnavailable = false h.observation = relayObservation{Connection: "unknown"} h.checked, h.healthySince = time.Time{}, time.Time{} } +// markUnavailable records that no forwarding attempt is serving. Unlike a +// provider observation, this local fact remains true throughout retry/backoff. +func (h *relayHealth) markUnavailable() { + h.mu.Lock() + defer h.mu.Unlock() + h.supervisorUnavailable = true + h.observation = relayObservation{Connection: "unavailable"} + h.checked, h.healthySince = time.Time{}, time.Time{} +} + func (h *relayHealth) observe(now time.Time, observation relayObservation, backoff *exponentialBackoff) { h.mu.Lock() defer h.mu.Unlock() @@ -58,6 +72,7 @@ func (h *relayHealth) observe(now time.Time, observation relayObservation, backo } func (h *relayHealth) observeLocked(now time.Time, observation relayObservation, backoff *exponentialBackoff) { + h.supervisorUnavailable = false if !observation.ready() { h.healthySince = time.Time{} } else { @@ -74,6 +89,9 @@ func (h *relayHealth) observeLocked(now time.Time, observation relayObservation, func (h *relayHealth) snapshot(now time.Time) relaySnapshot { h.mu.Lock() defer h.mu.Unlock() + if h.supervisorUnavailable { + return relaySnapshot{relayObservation: relayObservation{Connection: "unavailable"}} + } if h.checked.IsZero() || now.Sub(h.checked) >= healthStaleAfter { return relaySnapshot{relayObservation: relayObservation{Connection: "unknown"}} } diff --git a/tunnel/relay_metrics.go b/tunnel/relay_metrics.go index f671a3ba2..aee7bf024 100644 --- a/tunnel/relay_metrics.go +++ b/tunnel/relay_metrics.go @@ -19,8 +19,9 @@ func relayMetrics(h *relayHealth) http.Handler { name, help string value bool }{ + {"gameplane_tunnel_connection_supported", "Whether the provider can report live connection health.", !h.connectionUnsupported}, {"gameplane_tunnel_ready", "Whether the relay and all configured forwards are confirmed ready.", snapshot.Ready}, - {"gameplane_tunnel_connection_known", "Whether the provider exposes a fresh confirmed connection state.", known}, + {"gameplane_tunnel_connection_known", "Whether current provider or supervisor evidence confirms the connection state.", known}, {"gameplane_tunnel_forwards_registered", "Whether every configured forward is confirmed registered.", snapshot.Forwards}, {"gameplane_tunnel_endpoint_assigned", "Whether the configured endpoint is assigned.", snapshot.Assigned}, } diff --git a/tunnel/relay_metrics_test.go b/tunnel/relay_metrics_test.go index 90f73ff48..1a0bcf306 100644 --- a/tunnel/relay_metrics_test.go +++ b/tunnel/relay_metrics_test.go @@ -35,3 +35,19 @@ func TestRelayMetricsUnknownStaleAndRecovery(t *testing.T) { } } } + +func TestRelayMetricsDistinguishUnsupportedHealth(t *testing.T) { + for _, unsupported := range []bool{false, true} { + h := &relayHealth{connectionUnsupported: unsupported} + h.observe(time.Now(), relayObservation{Connection: "unknown", Assigned: true}, nil) + w := httptest.NewRecorder() + relayHealthHandler(h).ServeHTTP(w, httptest.NewRequestWithContext(t.Context(), "GET", "/metrics", nil)) + want := "gameplane_tunnel_connection_supported 1\n" + if unsupported { + want = "gameplane_tunnel_connection_supported 0\n" + } + if !strings.Contains(w.Body.String(), want) || !strings.Contains(w.Body.String(), "gameplane_tunnel_connection_known 0\n") { + t.Fatal(w.Body.String()) + } + } +} diff --git a/tunnel/specs.md b/tunnel/specs.md index 0f93f568c..ab43ccc1e 100644 --- a/tunnel/specs.md +++ b/tunnel/specs.md @@ -121,9 +121,10 @@ playit.gg assigns a tunnel's public address server-side, so the supervisor learn The maintainer chose embedded `tailscale.com/tsnet` for TCP and UDP; the decision is recorded in [OPEN-DECISIONS.md](../specs/done_018-v0-3-release-readiness/OPEN-DECISIONS.md#od-031-tailscale-game-forwarding-f-173--resolved-2026-10-21). No tailscaled executable, CLI, subnet router, exit node, Serve or Funnel configuration is used. - `tailscaleForwardPlan` accepts only the exact `..svc` host and explicitly advertised protocol/port pairs. Inbound bytes never select a backend address. TCP listens on both tailnet families; UDP binds each assigned tailnet IP because tsnet requires an explicit address. -- The ordinary Go network dialer resolves the Service for each new TCP connection or UDP session. Existing streams/sessions can fail during Service/backend replacement; reconnecting clients resolve the current Service. UDP sessions expire after 30 seconds without client packets. +- The ordinary Go network dialer resolves the Service for each new TCP connection or UDP session. Existing streams/sessions can fail during Service/backend replacement; reconnecting clients resolve the current Service. UDP sessions expire after 30 seconds without traffic in either direction; server-only updates keep an established session alive. - A shared 128-session budget covers all ports/protocols/families. Each UDP session queues at most four datagrams (65,507 bytes each) and uses one response buffer. Full budgets or queues drop incoming UDP; excess TCP connections close. Dials and UDP writes time out after five seconds. TCP sockets have five-minute activity timeouts. Shutdown closes listeners and sockets, cancels pending dials, then joins workers. - `tsnet.Server` receives the existing hostname, normalized requested tags and mounted auth key. There is no untagged fallback. The fixed state directory `/var/lib/gameplane-tailscale` uses a per-Pod EmptyDir owned through fsGroup 65532. Container restarts reuse identity. Pod replacement loses state and needs a reusable valid auth key; the Deployment uses Recreate. This is not durable identity across Pod replacement. Remove stale devices in the tailnet admin console to free the hostname. A suffixed/conflicting MagicDNS name cannot report the originally advertised name as ready. +- The operator validates tag syntax before creating the Deployment. Invalid tags remove a prior Deployment and report `TunnelReady=False/InvalidTailscaleTags` with correction guidance; fixing the tags resumes reconciliation. Authentication or tag-authorization failures still fail closed inside tsnet, with no registration attempt that omits requested tags. - `Up` has a two-minute attempt deadline; failed attempts use the existing capped retry delay. After binding every listener, the binary polls local tailnet status every two seconds with a two-second deadline. Loss of online status, tags or hostname clears readiness. Address changes recreate listeners. Failed listeners close all forwarding sockets and retry. Credentials and interactive login URLs are excluded from logs. - HTTP `:8091/readyz` returns 200 only while that state is ready, otherwise 503. `/healthz` remains independent of tailnet availability. The operator probes every five seconds and requires a current observed Deployment generation with an updated ready replica. Legacy deployments without the forwarding probe cannot report `TunnelReady=True`. Tailscale endpoints are published only while ready. - Readiness establishes local forwarding capacity and tailnet authentication. It does not prove a particular player's grant or game application health, and it deliberately remains ready while the game sleeps so traffic can reach the sentinel. Tailnet grants/ACLs still determine which peers may connect. Missing tags, denied registration, device approval and hostname conflicts require administrator repair. @@ -288,14 +289,23 @@ The maintainer explicitly approved ending the stdlib-only policy for tsnet in #7 `relay_health.go` owns the frp/Playit wrapper health interface and supervised attempt lifecycle. `/healthz`, `/readyz` and `/status` listen on Pod TCP 8091. -The startup probe checks wrapper liveness; the readiness probe requires actual -provider evidence. No liveness probe restarts a healthy wrapper for a provider +The startup probe checks wrapper liveness; frp's readiness probe requires actual +provider evidence. Playit Pod readiness uses `/healthz`, independently of its +unknown relay connection. No liveness probe restarts a healthy wrapper for a provider outage. Each attempt owns its monitor, cancels and joins it on exit, then clears readiness so a late response cannot resurrect the exited child. Queries have a two-second deadline and two-second cadence. Observations expire after ten seconds. All health responses use closed-set state; raw provider errors and credentials are not exposed. +Tailscale reports observed failures as unavailable. While no forwarding attempt +is serving, including retry backoff and replacement registration, the supervisor +retains that known unavailable state without a provider-observation timestamp. +This local lifecycle fact does not expire. Once forwarding is established, fresh +provider observations replace it and retain the ten-second staleness limit. +The served health handler and metrics are checked after a real listener failure, +during a blocked replacement join, and after recovery in the regression source. + frpc's authenticated loopback API `/api/status` on 127.0.0.1:7400 must confirm every configured proxy is running without error, with the expected protocol. The generated `user` is the GameServer UID, preserving advertised proxy names @@ -305,7 +315,8 @@ Recreate and a single replica to avoid simultaneous owners of an identity. Playit v1.0.10 IPC has no relay heartbeat: `Running` and addresses are cached API data that survive refresh errors. Assignment remains reportable, but connection -readiness stays unknown and the probe fails closed. The operator reports +readiness stays unknown and `/readyz` fails closed. The Kubernetes probe instead +uses `/healthz` so the Pod can become ready. The operator reports `RelayConnectionUnknown`, or `AwaitingAddress` before assignment. Future positive readiness needs a provider interface with fresh connection evidence. Never infer it from process lifetime or repeated reads of a cached address. diff --git a/tunnel/tailscale.go b/tunnel/tailscale.go index 93a42b989..18329c847 100644 --- a/tunnel/tailscale.go +++ b/tunnel/tailscale.go @@ -95,16 +95,8 @@ func runTailscale(ctx context.Context, cfg Config, authKey string) error { if err != nil { return err } - healthHistory.clear() - err = serveTailnetObserved(ctx, embeddedTailnet{server}, tags, ports, &ready, 2*time.Second, func(healthy bool) { - observation := relayObservation{Connection: "unknown"} - if healthy { - observation = relayObservation{Connection: "connected", Forwards: true, Assigned: true} - } - healthHistory.observe(time.Now(), observation, backoff) - }) + err = serveTailnetWithHealth(ctx, embeddedTailnet{server}, tags, ports, &ready, 2*time.Second, healthHistory, backoff) ready.Store(false) - healthHistory.clear() _ = server.Close() if ctx.Err() != nil { return nil @@ -114,28 +106,46 @@ func runTailscale(ctx context.Context, cfg Config, authKey string) error { if err != nil { log.Print("Tailscale forwarding unavailable; check credentials, requested tagOwners, device approval, unique hostname, MagicDNS and tailnet connectivity") } - select { - case <-ctx.Done(): - return nil - case err := <-healthDone: - return fmt.Errorf("tailscale health server: %w", err) - case <-time.After(backoff.next()): + if err := waitTailnetRetry(ctx, healthDone, backoff.next()); err != nil { + if ctx.Err() != nil { + return nil + } + return err } } return nil } -func tailscaleHealth(ready *atomic.Bool) http.Handler { - mux := http.NewServeMux() - mux.HandleFunc("/healthz", func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) }) - mux.HandleFunc("/readyz", func(w http.ResponseWriter, _ *http.Request) { - if !ready.Load() { - http.Error(w, "tailnet or forwarding listeners unavailable", http.StatusServiceUnavailable) - return - } - w.WriteHeader(http.StatusOK) +func waitTailnetRetry(ctx context.Context, healthDone <-chan error, delay time.Duration) error { + timer := time.NewTimer(delay) + defer timer.Stop() + select { + case <-ctx.Done(): + return ctx.Err() + case err := <-healthDone: + return fmt.Errorf("tailscale health server: %w", err) + case <-timer.C: + return nil + } +} + +// serveTailnetWithHealth preserves a known outage while a replacement joins +// and after an attempt exits, until serving forwards provide fresh evidence. +func serveTailnetWithHealth(ctx context.Context, node tailnetNode, tags []string, ports []forwardPort, ready *atomic.Bool, poll time.Duration, history *relayHealth, backoff *exponentialBackoff) error { + history.markUnavailable() + defer history.markUnavailable() + return serveTailnetObserved(ctx, node, tags, ports, ready, poll, func(healthy bool) { + history.observe(time.Now(), tailnetHealthObservation(healthy), backoff) }) - return mux +} + +func tailnetHealthObservation(healthy bool) relayObservation { + if healthy { + return relayObservation{Connection: "connected", Forwards: true, Assigned: true} + } + // Unlike Playit's cached assignment, tsnet provides an observed failure. + // Keep it known so an outage raises the unavailable alert. + return relayObservation{Connection: "unavailable"} } func tailnetReady(status *ipnstate.Status, tags []string) bool { diff --git a/tunnel/tailscale_forward.go b/tunnel/tailscale_forward.go index d74eda22b..480f96476 100644 --- a/tunnel/tailscale_forward.go +++ b/tunnel/tailscale_forward.go @@ -227,6 +227,9 @@ func (f *forwarder) proxyUDP(listener net.PacketConn, peer net.Addr, packets <-c if err != nil { return } + // Server-originated traffic is activity too, even when the client + // sends nothing after joining this UDP session. + _ = upstream.SetReadDeadline(time.Now().Add(f.limits.idle)) if n <= 65507 { _ = listener.SetWriteDeadline(time.Now().Add(f.limits.dial)) if _, err := listener.WriteTo(buf[:n], peer); err != nil { diff --git a/tunnel/tailscale_forward_test.go b/tunnel/tailscale_forward_test.go index 20376bbb2..40dfbb516 100644 --- a/tunnel/tailscale_forward_test.go +++ b/tunnel/tailscale_forward_test.go @@ -138,3 +138,57 @@ func TestTailscaleUDPForwardIsolationLimitAndExpiry(t *testing.T) { t.Fatal("shutdown leaked sessions") } } + +func TestTailscaleUDPServerTrafficKeepsSessionAlive(t *testing.T) { + t.Parallel() + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + backend, err := (&net.ListenConfig{}).ListenPacket(ctx, "udp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer backend.Close() + listener, err := (&net.ListenConfig{}).ListenPacket(ctx, "udp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + f := newForwarder(ctx, forwardLimits{sessions: 1, udpQueue: 1, idle: 400 * time.Millisecond, dial: time.Second}) + f.serveUDP(listener, backend.LocalAddr().String()) + defer f.Close() + client, err := (&net.Dialer{}).DialContext(ctx, "udp", listener.LocalAddr().String()) + if err != nil { + t.Fatal(err) + } + defer client.Close() + if _, err := client.Write([]byte("join")); err != nil { + t.Fatal(err) + } + _ = backend.SetReadDeadline(time.Now().Add(3 * time.Second)) + buffer := make([]byte, 32) + _, peer, err := backend.ReadFrom(buffer) + if err != nil { + t.Fatal(err) + } + // No more client packets: server updates alone must extend the same session + // for several idle windows, then complete silence must still reclaim it. + ticker := time.NewTicker(80 * time.Millisecond) + defer ticker.Stop() + for i := 0; i < 15; i++ { + <-ticker.C + if _, err := backend.WriteTo([]byte("update"), peer); err != nil { + t.Fatal(err) + } + _ = client.SetReadDeadline(time.Now().Add(time.Second)) + n, err := client.Read(buffer) + if err != nil || string(buffer[:n]) != "update" { + t.Fatalf("server update %d lost: %q %v", i, buffer[:n], err) + } + } + deadline := time.Now().Add(3 * time.Second) + for len(f.slots) != 0 && time.Now().Before(deadline) { + time.Sleep(10 * time.Millisecond) + } + if len(f.slots) != 0 { + t.Fatal("silent session did not expire") + } +} diff --git a/tunnel/tailscale_retry_test.go b/tunnel/tailscale_retry_test.go new file mode 100644 index 000000000..3c5bc3936 --- /dev/null +++ b/tunnel/tailscale_retry_test.go @@ -0,0 +1,111 @@ +package main + +import ( + "errors" + "sync/atomic" + "testing" + "time" +) + +func TestTailnetRetryHealthRequiresContinuousConfirmation(t *testing.T) { + t.Parallel() + for _, scenario := range []string{"short session", "healthy interval", "offline interruption", "stale gap"} { + t.Run(scenario, func(t *testing.T) { + t.Parallel() + backoff := &exponentialBackoff{} + for _, want := range []time.Duration{1, 2, 4, 8, 16, 32, 64, 128, 256, 300, 300} { + if got := backoff.next(); got != want*time.Second { + t.Fatalf("failure delay=%v, want %v", got, want*time.Second) + } + } + history := &relayHealth{} + start := time.Unix(100, 0) + observe := func(seconds int, healthy bool) { + observeTailnetRetryHealth(history, start.Add(time.Duration(seconds)*time.Second), healthy, backoff) + } + for seconds := 0; seconds <= 58; seconds += 2 { + observe(seconds, true) + } + if got := backoff.next(); got != 300*time.Second { + t.Fatalf("short healthy session reset delay: %v", got) + } + switch scenario { + case "short session": + observe(59, false) + if got := backoff.next(); got != 300*time.Second { + t.Fatalf("session exit reset delay: %v", got) + } + return + case "healthy interval": + observe(60, true) + case "offline interruption": + observe(60, false) + for seconds := 62; seconds <= 120; seconds += 2 { + observe(seconds, true) + } + if got := backoff.next(); got != 300*time.Second { + t.Fatalf("offline interruption counted as healthy time: %v", got) + } + observe(122, true) + case "stale gap": + // Exactly ten seconds without an observation breaks continuity. + for seconds := 68; seconds <= 126; seconds += 2 { + observe(seconds, true) + } + if got := backoff.next(); got != 300*time.Second { + t.Fatalf("stale observation gap counted as healthy time: %v", got) + } + observe(128, true) + } + observe(130, false) + if got := backoff.next(); got != time.Second { + t.Fatalf("confirmed healthy interval did not reset: %v", got) + } + if got := backoff.next(); got != 2*time.Second { + t.Fatalf("subsequent failure did not escalate: %v", got) + } + }) + } +} + +func TestTailnetFailedAttemptsCannotResetRetryEscalation(t *testing.T) { + t.Parallel() + for _, scenario := range []string{"authentication", "denied tag", "listener"} { + t.Run(scenario, func(t *testing.T) { + t.Parallel() + node := &fakeTailnetNode{status: onlineTailnet()} + tags := []string{"tag:game"} + switch scenario { + case "authentication": + node.upErr = errors.New("unauthorized") + case "denied tag": + tags = []string{"tag:ungranted"} + case "listener": + node.listenErr = true + } + backoff := &exponentialBackoff{} + backoff.next() + history := &relayHealth{} + var ready atomic.Bool + confirmed := false + err := serveTailnetObserved(t.Context(), node, tags, []forwardPort{{network: "udp", port: "19132", target: "127.0.0.1:1"}}, &ready, time.Millisecond, func(healthy bool) { + confirmed = confirmed || healthy + observeTailnetRetryHealth(history, time.Unix(100, 0), healthy, backoff) + }) + if err == nil || ready.Load() || confirmed { + t.Fatalf("failed attempt reported health: err=%v ready=%v confirmed=%v", err, ready.Load(), confirmed) + } + if got := backoff.next(); got != 2*time.Second { + t.Fatalf("failed attempt reset delay: %v", got) + } + }) + } +} + +func observeTailnetRetryHealth(history *relayHealth, now time.Time, healthy bool, backoff *exponentialBackoff) { + observation := relayObservation{Connection: "unknown"} + if healthy { + observation = relayObservation{Connection: "connected", Forwards: true, Assigned: true} + } + history.observe(now, observation, backoff) +} diff --git a/tunnel/tailscale_test.go b/tunnel/tailscale_test.go index 196a60712..baedab707 100644 --- a/tunnel/tailscale_test.go +++ b/tunnel/tailscale_test.go @@ -7,6 +7,7 @@ import ( "net/http" "net/http/httptest" "net/netip" + "strings" "sync" "sync/atomic" "testing" @@ -30,6 +31,11 @@ func TestTailscaleServerPreservesIdentityAndTags(t *testing.T) { t.Fatal("invalid tags silently dropped") } cfg.TailscaleTags = "" + cfg.TailscaleHostname = "" + if _, _, err := tailscaleServer(cfg, "key"); err == nil { + t.Fatal("empty hostname accepted") + } + cfg.TailscaleHostname = "my-game" if _, _, err := tailscaleServer(cfg, ""); err == nil { t.Fatal("empty auth key accepted") } @@ -56,6 +62,7 @@ type fakeTailnetNode struct { mu sync.Mutex status *ipnstate.Status upErr error + upCalls int listenErr bool listened []string listeners []net.Listener @@ -63,6 +70,7 @@ type fakeTailnetNode struct { } func (f *fakeTailnetNode) Up(ctx context.Context) (*ipnstate.Status, error) { + f.upCalls++ if f.upErr != nil { return nil, f.upErr } @@ -163,6 +171,9 @@ func TestTailnetFailuresNeverBecomeReady(t *testing.T) { if err == nil || ready.Load() { t.Fatalf("failure became ready: %v", err) } + if node.upCalls != 1 { + t.Fatalf("failed registration retried without requested tags: %d attempts", node.upCalls) + } for _, listener := range node.listeners { if _, err := listener.Accept(); err == nil { t.Fatal("partially opened listener leaked") @@ -172,6 +183,23 @@ func TestTailnetFailuresNeverBecomeReady(t *testing.T) { } } +func TestTailscaleRetryBackoffCancelsImmediately(t *testing.T) { + t.Parallel() + ctx, cancel := context.WithCancel(t.Context()) + healthDone := make(chan error) + done := make(chan error, 1) + go func() { done <- waitTailnetRetry(ctx, healthDone, time.Hour) }() + cancel() + select { + case err := <-done: + if !errors.Is(err, context.Canceled) { + t.Fatalf("cancelled retry: %v", err) + } + case <-time.After(time.Second): + t.Fatal("retry backoff ignored cancellation") + } +} + func TestTailnetListenerFailureClearsReady(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -196,21 +224,128 @@ func TestTailnetListenerFailureClearsReady(t *testing.T) { } } -func TestTailnetHealthNeedsForwardingReadiness(t *testing.T) { +type gatedTailnetNode struct { + *fakeTailnetNode + started chan struct{} + release chan struct{} +} + +func (n *gatedTailnetNode) Up(ctx context.Context) (*ipnstate.Status, error) { + close(n.started) + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-n.release: + return n.fakeTailnetNode.Up(ctx) + } +} + +func TestTailnetFailedAttemptRemainsKnownDuringRetry(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + history := &relayHealth{} + handler := relayHealthHandler(history) + ports := []forwardPort{{network: "tcp", port: "25565", target: "127.0.0.1:1"}} + node := &fakeTailnetNode{status: onlineTailnet()} var ready atomic.Bool - handler := tailscaleHealth(&ready) + done := make(chan error, 1) + go func() { + done <- serveTailnetWithHealth(ctx, node, nil, ports, &ready, time.Millisecond, history, nil) + }() + awaitTailnetReady(t, &ready, true) + if err := node.listeners[0].Close(); err != nil { + t.Fatal(err) + } + select { + case err := <-done: + if err == nil { + t.Fatal("failed listener was not reported") + } + case <-time.After(time.Second): + t.Fatal("failed attempt did not finish") + } + assertUnavailable := func() { + t.Helper() + // Advance beyond the maximum retry delay without sleeping: absence of + // a serving attempt is a local fact, not a stale provider observation. + if got := history.snapshot(time.Now().Add(6 * time.Minute)); got.Connection != "unavailable" || got.Ready { + t.Fatalf("retry forgot the known outage: %+v", got) + } + rec := httptest.NewRecorder() + handler.ServeHTTP(rec, httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/readyz", nil)) + if rec.Code != http.StatusServiceUnavailable { + t.Fatalf("failed attempt readyz=%d", rec.Code) + } + metrics := httptest.NewRecorder() + handler.ServeHTTP(metrics, httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/metrics", nil)) + for _, want := range []string{"gameplane_tunnel_connection_known 1\n", "gameplane_tunnel_connected 0\n", "gameplane_tunnel_ready 0\n"} { + if !strings.Contains(metrics.Body.String(), want) { + t.Fatalf("failed attempt missing %q: %s", want, metrics.Body.String()) + } + } + if strings.Contains(metrics.Body.String(), "gameplane_tunnel_observation_timestamp_seconds") { + t.Fatalf("supervisor outage invented a provider observation timestamp: %s", metrics.Body.String()) + } + } + assertUnavailable() + + // A new registration attempt must retain that state until forwarding is + // actually established, even if its Up call takes longer than the lease. + retry := &gatedTailnetNode{fakeTailnetNode: &fakeTailnetNode{status: onlineTailnet()}, started: make(chan struct{}), release: make(chan struct{})} + go func() { + done <- serveTailnetWithHealth(ctx, retry, nil, ports, &ready, time.Millisecond, history, nil) + }() + select { + case <-retry.started: + case <-time.After(time.Second): + t.Fatal("retry did not start") + } + assertUnavailable() + close(retry.release) + awaitTailnetReady(t, &ready, true) + deadline := time.Now().Add(time.Second) + for !history.snapshot(time.Now()).Ready && time.Now().Before(deadline) { + time.Sleep(time.Millisecond) + } + if got := history.snapshot(time.Now()); got.Connection != "connected" || !got.Ready { + t.Fatalf("recovered forwarding did not restore health: %+v", got) + } + if got := history.snapshot(time.Now().Add(2 * healthStaleAfter)); got.Connection != "unknown" || got.Ready { + t.Fatalf("live provider observations no longer expire: %+v", got) + } + cancel() + select { + case err := <-done: + if err != nil { + t.Fatal(err) + } + case <-time.After(time.Second): + t.Fatal("retry shutdown stalled") + } +} + +func TestTailnetHealthNeedsForwardingReadiness(t *testing.T) { + history := &relayHealth{} + handler := relayHealthHandler(history) for _, check := range []struct { path string ready bool want int }{{"/readyz", false, 503}, {"/healthz", false, 200}, {"/readyz", true, 200}, {"/readyz", false, 503}} { - ready.Store(check.ready) + history.observe(time.Now(), tailnetHealthObservation(check.ready), nil) rec := httptest.NewRecorder() handler.ServeHTTP(rec, httptest.NewRequestWithContext(context.Background(), http.MethodGet, check.path, nil)) if rec.Code != check.want { t.Fatalf("%s=%d want%d", check.path, rec.Code, check.want) } } + metrics := httptest.NewRecorder() + handler.ServeHTTP(metrics, httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/metrics", nil)) + for _, want := range []string{"gameplane_tunnel_connection_known 1\n", "gameplane_tunnel_connected 0\n", "gameplane_tunnel_ready 0\n"} { + if !strings.Contains(metrics.Body.String(), want) { + t.Fatalf("confirmed tailnet failure missing %q: %s", want, metrics.Body.String()) + } + } } func TestTailnetConfirmedObservationsResetRetryEscalation(t *testing.T) { From cb5e73b7c21483637704e46ac483cf6e9b432a56 Mon Sep 17 00:00:00 2001 From: Baylem Date: Tue, 27 Oct 2026 11:55:44 -0400 Subject: [PATCH 17/18] test: exercise production relay observation and scoped wipe inventory Signed-off-by: Baylem --- .../gameserver_wipe_inventory_test.go | 27 ++++++++++++++++++- tunnel/tailscale.go | 10 ++++--- tunnel/tailscale_retry_test.go | 21 +++++++-------- 3 files changed, 42 insertions(+), 16 deletions(-) diff --git a/operator/internal/controller/gameserver_wipe_inventory_test.go b/operator/internal/controller/gameserver_wipe_inventory_test.go index e4075147d..4805da30a 100644 --- a/operator/internal/controller/gameserver_wipe_inventory_test.go +++ b/operator/internal/controller/gameserver_wipe_inventory_test.go @@ -106,11 +106,36 @@ func (r *wipeInventoryReader) Get(ctx context.Context, key client.ObjectKey, obj func (r *wipeInventoryReader) List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error { if _, ok := list.(*corev1.PodList); ok { - r.podLists++ + options := (&client.ListOptions{}).ApplyOptions(opts) + // Wipe inventory deliberately reads every Pod in the namespace so + // unlabeled legacy workers remain visible. Filtered safety queries, + // such as sentinel opt-out drainage, are separate inventories. + if (options.LabelSelector == nil || options.LabelSelector.Empty()) && + (options.FieldSelector == nil || options.FieldSelector.Empty()) { + r.podLists++ + } } return r.Reader.List(ctx, list, opts...) } +func TestWipeInventoryReaderSeparatesFilteredPodQueries(t *testing.T) { + reader := &wipeInventoryReader{Reader: fake.NewClientBuilder().WithScheme(wipeScheme(t)).Build()} + for range 2 { + var pods corev1.PodList + if err := reader.List(t.Context(), &pods, client.InNamespace("games")); err != nil { + t.Fatal(err) + } + if err := reader.List(t.Context(), &pods, client.InNamespace("games"), client.MatchingLabels{ + "app.kubernetes.io/instance": "server", sentinelIsolationLabel: "true", + }); err != nil { + t.Fatal(err) + } + } + if reader.podLists != 2 { + t.Fatalf("duplicate unfiltered inventories must count independently of sentinel queries: got %d", reader.podLists) + } +} + func TestWipeInventoryIsSharedWithWorkloadMutation(t *testing.T) { s := wipeScheme(t) gs := wipeGameServer(false, "") diff --git a/tunnel/tailscale.go b/tunnel/tailscale.go index 18329c847..76a9b0ad0 100644 --- a/tunnel/tailscale.go +++ b/tunnel/tailscale.go @@ -134,9 +134,13 @@ func waitTailnetRetry(ctx context.Context, healthDone <-chan error, delay time.D func serveTailnetWithHealth(ctx context.Context, node tailnetNode, tags []string, ports []forwardPort, ready *atomic.Bool, poll time.Duration, history *relayHealth, backoff *exponentialBackoff) error { history.markUnavailable() defer history.markUnavailable() - return serveTailnetObserved(ctx, node, tags, ports, ready, poll, func(healthy bool) { - history.observe(time.Now(), tailnetHealthObservation(healthy), backoff) - }) + return serveTailnetObserved(ctx, node, tags, ports, ready, poll, tailnetRetryObserver(history, backoff, time.Now)) +} + +func tailnetRetryObserver(history *relayHealth, backoff *exponentialBackoff, now func() time.Time) func(bool) { + return func(healthy bool) { + history.observe(now(), tailnetHealthObservation(healthy), backoff) + } } func tailnetHealthObservation(healthy bool) relayObservation { diff --git a/tunnel/tailscale_retry_test.go b/tunnel/tailscale_retry_test.go index 3c5bc3936..05b321f15 100644 --- a/tunnel/tailscale_retry_test.go +++ b/tunnel/tailscale_retry_test.go @@ -20,8 +20,11 @@ func TestTailnetRetryHealthRequiresContinuousConfirmation(t *testing.T) { } history := &relayHealth{} start := time.Unix(100, 0) + now := start + observer := tailnetRetryObserver(history, backoff, func() time.Time { return now }) observe := func(seconds int, healthy bool) { - observeTailnetRetryHealth(history, start.Add(time.Duration(seconds)*time.Second), healthy, backoff) + now = start.Add(time.Duration(seconds) * time.Second) + observer(healthy) } for seconds := 0; seconds <= 58; seconds += 2 { observe(seconds, true) @@ -88,12 +91,14 @@ func TestTailnetFailedAttemptsCannotResetRetryEscalation(t *testing.T) { history := &relayHealth{} var ready atomic.Bool confirmed := false + observe := tailnetRetryObserver(history, backoff, func() time.Time { return time.Unix(100, 0) }) err := serveTailnetObserved(t.Context(), node, tags, []forwardPort{{network: "udp", port: "19132", target: "127.0.0.1:1"}}, &ready, time.Millisecond, func(healthy bool) { confirmed = confirmed || healthy - observeTailnetRetryHealth(history, time.Unix(100, 0), healthy, backoff) + observe(healthy) }) - if err == nil || ready.Load() || confirmed { - t.Fatalf("failed attempt reported health: err=%v ready=%v confirmed=%v", err, ready.Load(), confirmed) + snapshot := history.snapshot(time.Unix(100, 0)) + if err == nil || ready.Load() || confirmed || snapshot.Ready { + t.Fatalf("failed attempt reported health: err=%v ready=%v confirmed=%v snapshot=%+v", err, ready.Load(), confirmed, snapshot) } if got := backoff.next(); got != 2*time.Second { t.Fatalf("failed attempt reset delay: %v", got) @@ -101,11 +106,3 @@ func TestTailnetFailedAttemptsCannotResetRetryEscalation(t *testing.T) { }) } } - -func observeTailnetRetryHealth(history *relayHealth, now time.Time, healthy bool, backoff *exponentialBackoff) { - observation := relayObservation{Connection: "unknown"} - if healthy { - observation = relayObservation{Connection: "connected", Forwards: true, Assigned: true} - } - history.observe(now, observation, backoff) -} From 4701667a57b9b06b2ec35fa259a92131c91cdf2b Mon Sep 17 00:00:00 2001 From: Baylem Date: Wed, 4 Nov 2026 22:37:51 -0500 Subject: [PATCH 18/18] test(operator): seed finished backups before forget manager startup Signed-off-by: Baylem --- .../controller/backup_forget_envtest_test.go | 35 ++++++++----------- 1 file changed, 15 insertions(+), 20 deletions(-) diff --git a/operator/internal/controller/backup_forget_envtest_test.go b/operator/internal/controller/backup_forget_envtest_test.go index 9dbf6bf8e..a358fda9b 100644 --- a/operator/internal/controller/backup_forget_envtest_test.go +++ b/operator/internal/controller/backup_forget_envtest_test.go @@ -38,15 +38,14 @@ func withForgetBackupReconciler(rec *events.FakeRecorder) setupReconciler { } } -// startForgetFixture boots a manager, seeds a GameServer and repo Secret, creates -// a restic Backup named name, waits for the reconciler to put the snapshot -// finalizer on it, then marks it Succeeded with snapshotID, the state a finished -// backup is in when someone deletes it. +// startForgetFixture seeds a finished restic Backup before starting its manager, +// then waits for the reconciler to add the snapshot finalizer. Starting with a +// Pending Backup races the copy Job's creation against the terminal status write; +// envtest has no garbage collector to finish deleting that unrelated copy Job. func startForgetFixture(t *testing.T, name, snapshotID string) (string, *events.FakeRecorder) { t.Helper() ns := newNamespace(t) rec := events.NewFakeRecorder(32) - startMgr(t, ns, withForgetBackupReconciler(rec)) seedGameServer(t, ns, "smp") if err := k8sClient.Create(context.Background(), buildResticRepoSecret(ns, "repo")); err != nil { @@ -55,6 +54,15 @@ func startForgetFixture(t *testing.T, name, snapshotID string) (string, *events. if err := k8sClient.Create(context.Background(), buildBackup(ns, name, "smp", "repo")); err != nil { t.Fatalf("create backup: %v", err) } + b := getBackup(t, ns, name) + now := metav1.Now() + b.Status.Phase = gameplanev1alpha1.BackupPhaseSucceeded + b.Status.SnapshotID = snapshotID + b.Status.CompletionTime = &now + if err := k8sClient.Status().Update(context.Background(), b); err != nil { + t.Fatalf("mark backup succeeded: %v", err) + } + startMgr(t, ns, withForgetBackupReconciler(rec)) eventually(t, func() (bool, string) { b := getBackup(t, ns, name) for _, f := range b.Finalizers { @@ -64,21 +72,8 @@ func startForgetFixture(t *testing.T, name, snapshotID string) (string, *events. } return false, "snapshot finalizer not yet added: " + strings.Join(b.Finalizers, ",") }) - - // The reconciler also writes status while the backup Job is Pending, so the - // update can conflict; retry on a fresh read. - if err := retry.RetryOnConflict(retry.DefaultRetry, func() error { - var b gameplanev1alpha1.Backup - if err := k8sClient.Get(context.Background(), types.NamespacedName{Namespace: ns, Name: name}, &b); err != nil { - return err - } - now := metav1.Now() - b.Status.Phase = gameplanev1alpha1.BackupPhaseSucceeded - b.Status.SnapshotID = snapshotID - b.Status.CompletionTime = &now - return k8sClient.Status().Update(context.Background(), &b) - }); err != nil { - t.Fatalf("mark backup succeeded: %v", err) + if _, exists := getJob(t, ns, name); exists { + t.Fatal("finished backup unexpectedly created a copy Job") } return ns, rec }