diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 876439572..285ff3a7e 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1194,6 +1194,8 @@ jobs: - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - uses: azure/setup-helm@9bc31f4ebc9c6b171d7bfbaa5d006ae7abdb4310 # v5.0.1 - run: helm lint charts/gameplane + - name: evaluate monitoring rules + run: bash charts/gameplane/tests/monitoring-rules.sh standalone-compose: name: standalone Compose smoke @@ -1257,10 +1259,10 @@ jobs: run: python3 charts/tests/check_gateway.py - name: standalone panel isolation and persistence chart renders run: python3 charts/tests/check_standalone.py - - name: strict network profile and sentinel policy selectors - run: python3 charts/tests/check_network_isolation.py - name: auxiliary resources and management placement chart renders run: python3 charts/tests/check_workloads.py + - name: strict network profile and sentinel policy selectors + run: python3 charts/tests/check_network_isolation.py - name: operator Secret namespace permissions and cache configuration run: python3 charts/tests/check_operator_secrets.py - name: sanity-check rendered manifests @@ -1347,10 +1349,16 @@ jobs: | grep -q 'app.kubernetes.io/name: gameplane-telemetry-receiver' awk '/kind: Service/,/^---$/' /tmp/telemetry-receiver-scrape.yaml \ | grep -q 'name: dashboard' - # scrapeNamespaceSelector unset (the chart default): neither - # exception renders, matching the pre-fix behavior. + # Missing scrape ingress must fail with actionable guidance. + if helm template gameplane charts/gameplane --set serviceMonitors.enabled=true > /tmp/monitoring-invalid.txt 2>&1; then + echo "::error::monitoring without a scrape selector unexpectedly rendered" + exit 1 + fi + grep -q 'serviceMonitors.scrapeNamespaceSelector' /tmp/monitoring-invalid.txt + # Explicit external policy ownership renders no chart exception. helm template gameplane charts/gameplane \ --set serviceMonitors.enabled=true \ + --set serviceMonitors.externallyManagedPolicy=true \ --show-only templates/networkpolicies.yaml > /tmp/netpol-default.yaml if grep -q 'allow-prometheus-to-agent' /tmp/netpol-default.yaml; then echo "::error::allow-prometheus-to-agent rendered without serviceMonitors.scrapeNamespaceSelector set" @@ -1527,7 +1535,8 @@ jobs: # keeps working without api.telemetry.enabled. helm template gameplane charts/gameplane \ --set api.telemetry.receiver.enabled=true \ - --set serviceMonitors.enabled=true > /tmp/rendered-reuse-values-receiver.yaml + --set serviceMonitors.enabled=true \ + --set serviceMonitors.externallyManagedPolicy=true > /tmp/rendered-reuse-values-receiver.yaml python3 - <<'PY' import yaml docs = [d for d in yaml.safe_load_all(open('/tmp/rendered-reuse-values-receiver.yaml')) if d] @@ -1824,9 +1833,13 @@ jobs: # (operator does zero logins and is wait-dominated, so it runs # wide; the api buckets stay within the login limiter's budget). # The arm64 job below uses the same steps and per-bucket settings. - bucket: [operator, api-auth, api-roles, api-rbac, api-agent, api-mods, telemetry, optional-components, network-isolation, management-placement] + bucket: [operator, api-auth, api-roles, api-rbac, api-agent, api-mods, telemetry, optional-components, management-placement, network-isolation, monitoring] arch: [amd64] include: + - bucket: monitoring + parallel: 1 + test_timeout: 20m + job_timeout: 75 - bucket: network-isolation parallel: 1 test_timeout: 15m @@ -1896,9 +1909,12 @@ jobs: - name: bring up cluster + helm install if: env.RUN_REGEX != '' || env.TAIL_REGEX != '' env: - GAMEPLANE_E2E_CNI: ${{ matrix.bucket == 'network-isolation' && 'cilium' || 'kindnet' }} GAMEPLANE_E2E_PROFILE: ${{ matrix.bucket == 'management-placement' && 'management' || 'default' }} + GAMEPLANE_E2E_CNI: ${{ (matrix.bucket == 'network-isolation' || matrix.bucket == 'monitoring') && 'cilium' || 'kindnet' }} run: ./deploy/kind/e2e.sh up "$CLUSTER" e2e "$GAMEPLANE_E2E_PROFILE" + - name: install real monitoring fixture + if: matrix.bucket == 'monitoring' && env.RUN_REGEX != '' + run: bash deploy/kind/monitoring.sh - name: run go e2e (${{ matrix.bucket }}) if: env.RUN_REGEX != '' run: | @@ -1922,7 +1938,7 @@ jobs: uses: ./.github/actions/dump-cluster-state with: context: ${{ env.CTX }} - namespaces: "gameplane-system,gameplane-games" + namespaces: "${{ matrix.bucket == 'monitoring' && 'gameplane-system,gameplane-games,monitoring' || 'gameplane-system,gameplane-games' }}" include-previous-logs: "true" e2e-go-arm64: @@ -2711,7 +2727,7 @@ jobs: 'go-e2e-unit': (n) => n === 'go (test/e2e)', 'api-postgres': (n) => n === 'api (postgres)', 'e2e-buckets': (n) => n === 'e2e bucket coverage', - 'e2e-go': (n) => /^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components|network-isolation|management-placement) \//.test(n) && n.includes('/ amd64 '), + 'e2e-go': (n) => /^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components|management-placement|network-isolation|monitoring) \//.test(n) && n.includes('/ amd64 '), 'e2e-go-arm64': (n) => /^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components) \//.test(n) && n.includes('/ arm64 '), 'e2e-multicluster': (n) => n.startsWith('e2e multicluster /') && n.includes('/ amd64 '), 'e2e-multicluster-arm64': (n) => n.startsWith('e2e multicluster /') && n.includes('/ arm64 '), @@ -2864,7 +2880,7 @@ jobs: const bucketSet = new Set(); for (const j of apiJobs) { const n = j.name || ''; - const m = n.match(/^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components|network-isolation|management-placement) \//); + const m = n.match(/^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components|management-placement|network-isolation|monitoring) \//); if (m) bucketSet.add(m[1]); if (/^e2e multicluster \//.test(n)) bucketSet.add('multicluster'); if (/^e2e upgrade \//.test(n)) bucketSet.add('upgrade'); diff --git a/agent/cmd/main.go b/agent/cmd/main.go index 8ce7f15f8..a27d217ef 100644 --- a/agent/cmd/main.go +++ b/agent/cmd/main.go @@ -16,6 +16,7 @@ import ( "github.com/go-chi/chi/v5" "github.com/go-chi/chi/v5/middleware" + "github.com/prometheus/client_golang/prometheus" "github.com/GameplanePanel/gameplane/agent/internal/actions" "github.com/GameplanePanel/gameplane/agent/internal/auth" @@ -178,6 +179,7 @@ func main() { // and isn't meant to be trusted with the console/files/RCON routes that // cert would also unlock. + statusSource := status.New(rconClient, statusSpec) mountProtected := func(protected chi.Router) { // Console/Logs open long-lived WebSockets and Mods installs can run @@ -198,7 +200,7 @@ func main() { quiesce.Mount(bounded, rconClient, gameName, quiesceSpec) lifecycle.Mount(bounded, rconClient, gameName, lifecycleSpec) actions.Mount(bounded, rconClient, gameName, actionSpecs) - status.Mount(bounded, rconClient, statusSpec) + statusSource.Mount(bounded) }) } r.Group(func(protected chi.Router) { @@ -230,6 +232,8 @@ func main() { // Prometheus metrics for per-server resource usage and player counts. // Registered with the default registry so promhttp.Handler() picks it up. m := metrics.New() + prometheus.MustRegister(statusSource) + go statusSource.Run(ctx) // Resource usage: in proc mode (the operator shares the pod's PID // namespace and sets GAMEPLANE_USAGE_PROC) the agent reports the game diff --git a/agent/internal/caps/caps.go b/agent/internal/caps/caps.go index 495cc2654..448501ff6 100644 --- a/agent/internal/caps/caps.go +++ b/agent/internal/caps/caps.go @@ -83,11 +83,19 @@ type ModInstall struct { // StatusMetric reads one live readout from an RCON command's output via a // named-group regex (group "value"). type StatusMetric struct { - ID string `json:"id"` - DisplayName string `json:"displayName,omitempty"` - Command string `json:"command"` - Regex string `json:"regex"` - Unit string `json:"unit,omitempty"` + ID string `json:"id"` + DisplayName string `json:"displayName,omitempty"` + Command string `json:"command"` + Regex string `json:"regex"` + Unit string `json:"unit,omitempty"` + Performance *PerformanceMetric `json:"performance,omitempty"` +} + +// PerformanceMetric mirrors the optional, explicitly numeric status contract. +type PerformanceMetric struct { + Signal string `json:"signal"` + Unit string `json:"unit"` + DegradedThreshold *string `json:"degradedThreshold,omitempty"` } // PlayerActions maps moderation actions to console command templates diff --git a/agent/internal/rcon/nuclearoption.go b/agent/internal/rcon/nuclearoption.go index 7b8f2b1f2..001135354 100644 --- a/agent/internal/rcon/nuclearoption.go +++ b/agent/internal/rcon/nuclearoption.go @@ -112,14 +112,21 @@ func (c *NuclearOption) Close() error { // If the response has no body (body length = 0), an empty string is returned. // Non-2000 status codes produce an error that includes the status code and, // if present, the error detail from the response body. +// One deadline bounds the complete exchange, including dialing and request writes. func (c *NuclearOption) Exec(cmd string) (string, error) { c.mu.Lock() defer c.mu.Unlock() + d := c.execDeadline + if d <= 0 { + d = defaultNuclearOptionExecDeadline + } + exchangeDeadline := time.Now().Add(d) + // Ensure connection is open. if c.conn == nil { d := net.Dialer{Timeout: c.dialTimeout} - ctx, cancel := context.WithTimeout(context.Background(), c.dialTimeout) + ctx, cancel := context.WithDeadline(context.Background(), exchangeDeadline) defer cancel() conn, err := d.DialContext(ctx, "tcp", net.JoinHostPort(c.host, fmt.Sprint(c.port))) if err != nil { @@ -144,6 +151,14 @@ func (c *NuclearOption) Exec(cmd string) (string, error) { return "", fmt.Errorf("nuclearoption rcon: marshal request: %w", err) } + // Bound writes as well as reads. Arm this before sending anything: a + // server that stops reading must not retain the shared client forever. + if err := c.conn.SetDeadline(exchangeDeadline); err != nil { + _ = c.conn.Close() + c.conn = nil + return "", fmt.Errorf("nuclearoption rcon: set deadline: %w", err) + } + // Write request: 4-byte LE length + body. reqLen := len(reqBody) if reqLen < 0 || reqLen > math.MaxUint32 { @@ -162,17 +177,6 @@ func (c *NuclearOption) Exec(cmd string) (string, error) { return "", fmt.Errorf("nuclearoption rcon: write body: %w", err) } - // Set deadline for the entire response read. - d := c.execDeadline - if d <= 0 { - d = defaultNuclearOptionExecDeadline - } - if err := c.conn.SetDeadline(time.Now().Add(d)); err != nil { - _ = c.conn.Close() - c.conn = nil - return "", fmt.Errorf("nuclearoption rcon: set deadline: %w", err) - } - // Read response: 4-byte status + 4-byte body length + body. statusBuf := make([]byte, 4) if _, err := io.ReadFull(c.conn, statusBuf); err != nil { diff --git a/agent/internal/rcon/nuclearoption_deadline_test.go b/agent/internal/rcon/nuclearoption_deadline_test.go new file mode 100644 index 000000000..9cae0e073 --- /dev/null +++ b/agent/internal/rcon/nuclearoption_deadline_test.go @@ -0,0 +1,46 @@ +package rcon + +import ( + "io" + "net" + "strings" + "testing" + "time" +) + +func TestNuclearOptionDeadlineIncludesRequestWrites(t *testing.T) { + t.Parallel() + for _, readHeader := range []bool{false, true} { + stage := "write length" + if readHeader { + stage = "write body" + } + t.Run(stage, func(t *testing.T) { + t.Parallel() + local, remote := net.Pipe() + defer local.Close() + defer remote.Close() + client := NewNuclearOption("ignored", 1, nil) + client.conn = local + client.execDeadline = 100 * time.Millisecond + if readHeader { + go func() { _, _ = io.CopyN(io.Discard, remote, 4) }() + } + done := make(chan error, 1) + go func() { _, err := client.Exec("get-server-id"); done <- err }() + select { + case err := <-done: + if err == nil || !isTimeout(err) || !strings.Contains(err.Error(), stage) { + t.Fatalf("stalled %s must time out: %v", stage, err) + } + case <-time.After(2 * time.Second): + _ = remote.Close() + <-done + t.Fatalf("stalled %s exceeded the command budget", stage) + } + if client.conn != nil { + t.Fatal("timed-out partial request must discard the connection") + } + }) + } +} diff --git a/agent/internal/rcon/nuclearoption_test.go b/agent/internal/rcon/nuclearoption_test.go index 9061d2c94..ccde83c53 100644 --- a/agent/internal/rcon/nuclearoption_test.go +++ b/agent/internal/rcon/nuclearoption_test.go @@ -740,7 +740,7 @@ func TestNuclearOptionWriteBodyError(t *testing.T) { } // TestNuclearOptionSetDeadlineError verifies that a failure setting the -// response read deadline (after the request was fully written) is reported +// exchange deadline (before sending the request) is reported // and the connection is reset. func TestNuclearOptionSetDeadlineError(t *testing.T) { local, remote := net.Pipe() diff --git a/agent/internal/rcon/rcon.go b/agent/internal/rcon/rcon.go index bbb0456d2..7dfcaa01e 100644 --- a/agent/internal/rcon/rcon.go +++ b/agent/internal/rcon/rcon.go @@ -56,6 +56,11 @@ const ( // byte length (F-104). minPacketSize = 10 maxPacketSize = 4*4096 + 10 // 16394 + + // sourceMaxResponseBytes permits large administrative listings while + // bounding the aggregate allocation across otherwise valid fragments. + // Status collection applies its separate, smaller 64-KiB acceptance cap. + sourceMaxResponseBytes = 8 * 1024 * 1024 ) // PassFn resolves the RCON password on demand (allowing rotation without restart). @@ -86,6 +91,8 @@ type Client struct { // execDeadline bounds a single command exchange (see Exec). execDeadline time.Duration + // authDeadline separately bounds authentication, including all preamble packets. + authDeadline time.Duration // authFailureCooldown bounds how long ensureLocked refuses to re-dial // after an explicit AUTH_RESPONSE rejection (id=-1). A struct field @@ -114,6 +121,8 @@ type Client struct { // (heartbeat, players, quiesce, console). const defaultExecDeadline = 30 * time.Second +const defaultSourceAuthDeadline = 10 * time.Second + // defaultAuthFailureCooldown bounds how long ensureLocked backs off after // an explicit auth rejection before it will dial again. Mirrors // websocket.go's defaultWebSocketAuthFailureCooldown — same value, same @@ -133,6 +142,7 @@ func New(host string, port int, pw PassFn) *Client { addr: net.JoinHostPort(host, fmt.Sprint(port)), password: pw, execDeadline: defaultExecDeadline, + authDeadline: defaultSourceAuthDeadline, authFailureCooldown: defaultAuthFailureCooldown, } } @@ -154,7 +164,8 @@ func (c *Client) Exec(cmd string) (string, error) { if d <= 0 { d = defaultExecDeadline } - _ = conn.SetDeadline(time.Now().Add(d)) + exchangeDeadline := time.Now().Add(d) + _ = conn.SetDeadline(exchangeDeadline) defer func() { _ = conn.SetDeadline(time.Time{}) }() reqID := c.allocID() @@ -172,12 +183,22 @@ func (c *Client) Exec(cmd string) (string, error) { // reqID) and bound the wait for any trailing fragments with a short grace // window: a single-packet reply returns as soon as the grace // elapses or the server closes; a fragmented Source reply keeps extending - // the window as each fragment arrives. + // the window as each fragment arrives, never beyond exchangeDeadline. var out bytes.Buffer gotResponse := false for { + if !time.Now().Before(exchangeDeadline) { + c.dropLocked() + return "", fmt.Errorf("rcon: command exchange deadline exceeded: %w", os.ErrDeadlineExceeded) + } id, _, body, err := c.readPacket() if err != nil { + if !time.Now().Before(exchangeDeadline) { + // Reaching the hard deadline is not evidence of a complete + // response. Discard its partial text and any queued fragments. + c.dropLocked() + return "", fmt.Errorf("rcon: command exchange deadline exceeded: %w", os.ErrDeadlineExceeded) + } if gotResponse { // We already hold the reply. A timeout means the grace window // elapsed on a still-healthy socket — keep it for reuse. Any @@ -196,9 +217,17 @@ func (c *Client) Exec(cmd string) (string, error) { // Source/Minecraft echo the request id in every reply fragment; ignore // anything else. if id == reqID { + if len(body) > sourceMaxResponseBytes-out.Len() { + c.dropLocked() + return "", fmt.Errorf("rcon: response exceeded limit of %d bytes", sourceMaxResponseBytes) + } out.WriteString(body) gotResponse = true - _ = conn.SetReadDeadline(time.Now().Add(responseGrace)) + readDeadline := time.Now().Add(responseGrace) + if readDeadline.After(exchangeDeadline) { + readDeadline = exchangeDeadline + } + _ = conn.SetReadDeadline(readDeadline) } } } @@ -247,7 +276,12 @@ func (c *Client) ensureLocked() error { return err } c.conn = conn - _ = conn.SetDeadline(time.Now().Add(10 * time.Second)) + authBudget := c.authDeadline + if authBudget <= 0 { + authBudget = defaultSourceAuthDeadline + } + authDeadline := time.Now().Add(authBudget) + _ = conn.SetDeadline(authDeadline) authID := c.allocID() if err := c.writePacket(authID, typeAuth, pw); err != nil { @@ -256,6 +290,10 @@ func (c *Client) ensureLocked() error { } // Server MAY send an empty RESPONSE_VALUE before AUTH_RESPONSE. for { + if !time.Now().Before(authDeadline) { + c.dropLocked() + return fmt.Errorf("rcon: authentication deadline exceeded: %w", os.ErrDeadlineExceeded) + } id, kind, _, err := c.readPacket() if err != nil { c.dropLocked() diff --git a/agent/internal/rcon/rcon_bounds_test.go b/agent/internal/rcon/rcon_bounds_test.go new file mode 100644 index 000000000..39038bd81 --- /dev/null +++ b/agent/internal/rcon/rcon_bounds_test.go @@ -0,0 +1,166 @@ +package rcon + +import ( + "net" + "strings" + "testing" + "time" +) + +func TestExecContinuousFragmentsRespectAbsoluteDeadline(t *testing.T) { + t.Parallel() + for _, unrelated := range []bool{false, true} { + name := "matching" + if unrelated { + name = "unrelated-after-response" + } + t.Run(name, func(t *testing.T) { + t.Parallel() + clientConn, serverConn := net.Pipe() + defer clientConn.Close() + defer serverConn.Close() + client := &Client{conn: clientConn, execDeadline: 100 * time.Millisecond} + server := &Client{conn: serverConn} + done := make(chan struct{}) + go func() { + defer close(done) + id, _, _, err := server.readPacket() + if err != nil { + return + } + if server.writePacket(id, typeRespValue, "partial") != nil { + return + } + if unrelated { + id++ + } + // Empty fragments cannot trigger the aggregate byte cap. A + // continuously readable socket must still reach its hard deadline. + for { + if server.writePacket(id, typeRespValue, "") != nil { + return + } + } + }() + finished := make(chan struct{}) + var output string + var execErr error + go func() { + output, execErr = client.Exec("status") + close(finished) + }() + select { + case <-finished: + case <-time.After(2 * time.Second): + // Unblock both goroutines even when testing the old read loop. + _ = serverConn.Close() + <-finished + t.Fatal("continuous fragments extended the absolute command deadline") + } + if execErr == nil || !isTimeout(execErr) || output != "" { + t.Fatalf("deadline must discard incomplete output: bytes=%d err=%v", len(output), execErr) + } + if client.conn != nil { + t.Fatal("deadline must drop the connection to discard trailing fragments") + } + <-done + }) + } +} + +func TestExecAggregateResponseBound(t *testing.T) { + t.Parallel() + for _, extra := range []int{0, 1} { + name := "exact-limit" + if extra > 0 { + name = "over-limit" + } + t.Run(name, func(t *testing.T) { + t.Parallel() + clientConn, serverConn := net.Pipe() + defer clientConn.Close() + defer serverConn.Close() + client := &Client{conn: clientConn, execDeadline: 10 * time.Second} + server := &Client{conn: serverConn} + done := make(chan struct{}) + go func() { + defer close(done) + defer serverConn.Close() + id, _, _, err := server.readPacket() + if err != nil { + return + } + fragment := strings.Repeat("x", 4000) + remaining := sourceMaxResponseBytes + extra + for remaining > 0 { + length := min(remaining, len(fragment)) + if server.writePacket(id, typeRespValue, fragment[:length]) != nil { + return + } + remaining -= length + } + }() + output, err := client.Exec("large-list") + <-done + if extra == 0 { + if err != nil || len(output) != sourceMaxResponseBytes || strings.Trim(output, "x") != "" { + t.Fatalf("legitimate response at the generous cap must succeed: bytes=%d err=%v", len(output), err) + } + return + } + if err == nil || !strings.Contains(err.Error(), "response exceeded limit") || output != "" { + t.Fatalf("oversized response must fail without a truncated result: bytes=%d err=%v", len(output), err) + } + if client.conn != nil { + t.Fatal("oversized response must drop the connection") + } + }) + } +} + +func TestAuthContinuousPreambleRespectsAbsoluteDeadline(t *testing.T) { + t.Parallel() + lc := &net.ListenConfig{} + listener, err := lc.Listen(t.Context(), "tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer listener.Close() + done := make(chan struct{}) + go func() { + defer close(done) + conn, err := listener.Accept() + if err != nil { + return + } + defer conn.Close() + // Bound the fixture itself even if the client's deadline regresses. + _ = conn.SetDeadline(time.Now().Add(3 * time.Second)) + server := &Client{conn: conn} + id, _, _, err := server.readPacket() + if err != nil { + return + } + for { + if server.writePacket(id, typeRespValue, "") != nil { + return + } + } + }() + host, port, err := net.SplitHostPort(listener.Addr().String()) + if err != nil { + t.Fatal(err) + } + client := New(host, mustAtoi(port), func() (string, error) { return "pw", nil }) + client.authDeadline = 100 * time.Millisecond + defer client.Close() + started := time.Now() + output, err := client.Exec("status") + if err == nil || !isTimeout(err) || output != "" || time.Since(started) > 2*time.Second { + t.Fatalf("continuous auth preamble must time out without output: bytes=%d err=%v elapsed=%v", len(output), err, time.Since(started)) + } + if client.conn != nil { + t.Fatal("authentication deadline must drop the connection") + } + <-done +} diff --git a/agent/internal/status/performance.go b/agent/internal/status/performance.go new file mode 100644 index 000000000..0dbda1979 --- /dev/null +++ b/agent/internal/status/performance.go @@ -0,0 +1,94 @@ +package status + +import ( + "math" + "regexp" + "strconv" + "time" + + "github.com/GameplanePanel/gameplane/agent/internal/caps" + "github.com/prometheus/client_golang/prometheus" +) + +type performanceSample struct { + value float64 + observed time.Time +} + +var decimalThreshold = regexp.MustCompile(`^(0|[1-9][0-9]{0,8})(\.[0-9]{1,6})?$`) + +func validPerformance(p *caps.PerformanceMetric) bool { + if p == nil { + return false + } + if p.DegradedThreshold != nil && !decimalThreshold.MatchString(*p.DegradedThreshold) { + return false + } + switch p.Signal { + case "tick_rate", "frame_rate": + return p.Unit == "per_second" + case "tick_duration": + return p.Unit == "seconds" || p.Unit == "milliseconds" + default: + return false + } +} + +func finite(n float64) bool { return !math.IsNaN(n) && !math.IsInf(n, 0) } +func baseValue(n float64, p *caps.PerformanceMetric) float64 { + if p.Unit == "milliseconds" { + return n / 1000 + } + return n +} +func parsePerformance(raw string, p *caps.PerformanceMetric) (performanceSample, bool) { + n, err := strconv.ParseFloat(raw, 64) + if err != nil || !finite(n) || n < 0 { + return performanceSample{}, false + } + return performanceSample{value: baseValue(n, p)}, true +} + +var performanceLabels = []string{"metric", "signal", "unit"} +var performanceValue = prometheus.NewDesc("gameplane_agent_performance_value", "Finite game performance measurement in the declared base unit; absent when unknown or stale.", performanceLabels, nil) +var performanceAvailable = prometheus.NewDesc("gameplane_agent_performance_available", "Whether the declared performance signal has a fresh valid sample.", performanceLabels, nil) +var performanceTimestamp = prometheus.NewDesc("gameplane_agent_performance_sample_timestamp_seconds", "Unix collection timestamp for a fresh valid game performance sample.", performanceLabels, nil) +var performanceThreshold = prometheus.NewDesc("gameplane_agent_performance_degraded_threshold", "Module-declared degradation threshold in the same base unit as the measurement.", append(append([]string{}, performanceLabels...), "direction"), nil) + +// Describe publishes the fixed descriptors for explicitly declared performance signals. +func (s *Source) Describe(ch chan<- *prometheus.Desc) { + ch <- performanceValue + ch <- performanceAvailable + ch <- performanceTimestamp + ch <- performanceThreshold +} + +// Collect exports cached performance samples and availability without issuing commands. +func (s *Source) Collect(ch chan<- prometheus.Metric) { + s.mu.Lock() + defer s.mu.Unlock() + for _, m := range s.metrics { + p := m.performance + if p == nil { + continue + } + unit, direction := "hertz", "below" + if p.Signal == "tick_duration" { + unit, direction = "seconds", "above" + } + labels := []string{m.id, p.Signal, unit} + sample, ok := s.samples[m.id] + fresh := !sample.observed.IsZero() && time.Since(sample.observed) < performanceFreshness + available := 0.0 + if fresh && ok { + available = 1 + ch <- prometheus.MustNewConstMetric(performanceValue, prometheus.GaugeValue, sample.value, labels...) + ch <- prometheus.MustNewConstMetric(performanceTimestamp, prometheus.GaugeValue, float64(sample.observed.UnixNano())/1e9, labels...) + } + ch <- prometheus.MustNewConstMetric(performanceAvailable, prometheus.GaugeValue, available, labels...) + if p.DegradedThreshold != nil { + threshold, _ := strconv.ParseFloat(*p.DegradedThreshold, 64) // validated bounded decimal + ch <- prometheus.MustNewConstMetric(performanceThreshold, prometheus.GaugeValue, baseValue(threshold, p), append(labels, direction)...) + } + } +} diff --git a/agent/internal/status/performance_test.go b/agent/internal/status/performance_test.go new file mode 100644 index 000000000..86e934242 --- /dev/null +++ b/agent/internal/status/performance_test.go @@ -0,0 +1,134 @@ +package status + +import ( + "context" + "net/http/httptest" + "sync/atomic" + "testing" + "time" + + "github.com/GameplanePanel/gameplane/agent/internal/caps" + "github.com/go-chi/chi/v5" + "github.com/prometheus/client_golang/prometheus" +) + +func performanceSpec() *caps.Status { + threshold := "18" + return &caps.Status{Metrics: []caps.StatusMetric{{ID: "tps", Command: "stats", Regex: `TPS: (?P\S+)`, Performance: &caps.PerformanceMetric{Signal: "tick_rate", Unit: "per_second", DegradedThreshold: &threshold}}, {ID: "tick", Command: "stats", Regex: `Tick: (?P\S+)`, Performance: &caps.PerformanceMetric{Signal: "tick_duration", Unit: "milliseconds"}}}} +} + +func TestPerformanceCacheZeroFreshnessAndSharedCommand(t *testing.T) { + t.Parallel() + rc := &fakeRcon{respond: func(string) (string, error) { return "TPS: 0 Tick: 50", nil }} + s := New(rc, performanceSpec()) + s.refresh(t.Context()) + if rc.count() != 1 { + t.Fatalf("shared command executed %d times", rc.count()) + } + reg := prometheus.NewRegistry() + reg.MustRegister(s) + router := chi.NewRouter() + router.Route("/legacy", s.Mount) + router.Route("/versioned", s.Mount) + for _, route := range []string{"/legacy/status", "/versioned/status"} { + response := httptest.NewRecorder() + router.ServeHTTP(response, httptest.NewRequestWithContext(t.Context(), "GET", route, nil)) + if response.Code != 200 { + t.Fatal("shared status cache unavailable") + } + } + for range 20 { + if _, err := reg.Gather(); err != nil { + t.Fatal(err) + } + } + if rc.count() != 1 { + t.Fatal("scrape executed RCON") + } + s.mu.Lock() + if s.samples["tps"].value != 0 || s.samples["tick"].value != .05 { + t.Fatal("zero or unit conversion lost") + } + s.lastFetch = time.Now().Add(-performanceFreshness - time.Second) + for id, sample := range s.samples { + sample.observed = s.lastFetch + s.samples[id] = sample + } + s.mu.Unlock() + families, err := reg.Gather() + if err != nil { + t.Fatal(err) + } + for _, family := range families { + if family.GetName() == "gameplane_agent_performance_value" { + t.Fatal("stale value emitted") + } + } +} + +func TestPerformanceRejectsNonFiniteAndInvalidDeclarations(t *testing.T) { + t.Parallel() + for _, value := range []string{"NaN", "+Inf", "-1", "no-value"} { + s := New(&fakeRcon{respond: func(string) (string, error) { return "TPS: " + value, nil }}, performanceSpec()) + s.refresh(t.Context()) + if len(s.samples) != 0 { + t.Fatalf("invalid %q emitted", value) + } + } + inf := "+Inf" + for _, p := range []*caps.PerformanceMetric{{Signal: "seed", Unit: "per_second"}, {Signal: "tick_rate", Unit: "milliseconds"}, {Signal: "tick_duration", Unit: "seconds", DegradedThreshold: &inf}} { + if validPerformance(p) { + t.Fatalf("accepted %+v", p) + } + } +} + +func TestPerformanceTimeoutBoundsOutstandingWorkAndDiscardsLateResult(t *testing.T) { + t.Parallel() + release := make(chan struct{}) + var calls atomic.Int32 + rc := &fakeRcon{respond: func(string) (string, error) { calls.Add(1); <-release; return "TPS: 20", nil }} + s := New(rc, performanceSpec()) + s.commandTimeout = 10 * time.Millisecond + s.refresh(t.Context()) + for range 10 { + s.refresh(t.Context()) + } + if calls.Load() != 1 { + t.Fatalf("unbounded workers: %d", calls.Load()) + } + close(release) + deadline := time.After(time.Second) + for len(s.execSlot) != 0 { + select { + case <-deadline: + t.Fatal("worker failed to finish") + default: + time.Sleep(time.Millisecond) + } + } + if len(s.samples) != 0 { + t.Fatal("late result became a fresh sample") + } + ctx, cancel := context.WithCancel(t.Context()) + cancel() + s.Run(ctx) + if calls.Load() != 1 { + t.Fatal("cancelled collector executed a command") + } +} + +func TestPerformanceBoundsDirectCapabilitiesAndDeduplicatesIDs(t *testing.T) { + t.Parallel() + spec := performanceSpec() + for range 30 { + spec.Metrics = append(spec.Metrics, spec.Metrics[0]) + } + if compiled := compile(spec.Metrics); len(compiled) != 2 { + t.Fatalf("duplicate IDs expanded cardinality: %d", len(compiled)) + } + spec.Metrics[0].Command = string(make([]byte, maxCommandBytes+1)) + if len(compile(spec.Metrics[:1])) != 0 { + t.Fatal("oversized command accepted") + } +} diff --git a/agent/internal/status/status.go b/agent/internal/status/status.go index 7ddc82bc8..1304d37ba 100644 --- a/agent/internal/status/status.go +++ b/agent/internal/status/status.go @@ -1,45 +1,45 @@ -// Package status reads module-declared live metrics over RCON for the -// dashboard's Overview tab. -// -// A GameTemplate declares metrics in spec.capabilities.status.metrics[]; -// the operator serializes them into GAMEPLANE_CAPABILITIES and the agent -// interprets them here. Each metric runs an RCON command and extracts a -// value via a named-group regex (group "value"), so modules surface -// game-specific readouts (TPS, world time, …) without an agent change. +// Package status shares module-declared RCON readouts between the dashboard +// and explicitly opted-in numeric performance monitoring. package status import ( + "context" "errors" - "log/slog" "net/http" "regexp" "strings" "sync" "time" - "github.com/go-chi/chi/v5" - "github.com/GameplanePanel/gameplane/agent/internal/caps" "github.com/GameplanePanel/gameplane/agent/internal/httpjson" "github.com/GameplanePanel/gameplane/agent/internal/rcon" + "github.com/go-chi/chi/v5" ) -// Rcon is the slice of *rcon.Client we use. An interface so tests can -// swap in a recording fake. +// Rcon executes module-declared commands through the configured game transport. type Rcon interface { Exec(cmd string) (string, error) } -// metric is one declared readout whose regex compiled at mount time. +const ( + cacheTTL = 5 * time.Second + performanceInterval = 30 * time.Second + performanceFreshness = 75 * time.Second + cycleTimeout = 10 * time.Second + maxCommandBytes = 1024 + maxRegexBytes = 4096 + maxResponseBytes = 64 * 1024 +) + type metric struct { id, displayName, command, unit string re *regexp.Regexp valueIdx int + performance *caps.PerformanceMetric } -// Result is one metric's current value (empty when the command did not -// match this cycle). DisplayName/Unit are echoed so the dashboard needs -// no second lookup. +// Result is one module-declared readout returned by the dashboard status route. type Result struct { ID string `json:"id"` DisplayName string `json:"displayName,omitempty"` @@ -47,85 +47,183 @@ type Result struct { Unit string `json:"unit,omitempty"` } -type handler struct { - rc Rcon - metrics []metric - - mu sync.Mutex - lastFetch time.Time - cached []Result +// Source is shared by both protected route mounts and the metrics collector. +// At most one legacy context-free Exec can remain in flight after a timeout. +type Source struct { + rc Rcon + metrics []metric + automatic bool + mu sync.Mutex + refreshMu sync.Mutex + lastFetch time.Time + cached []Result + samples map[string]performanceSample + execSlot chan struct{} + commandTimeout time.Duration } -const cacheTTL = 5 * time.Second - -// Mount registers GET /status. spec is the module's declared metrics -// (nil when the template declares none — the endpoint then returns []). -func Mount(r chi.Router, rc Rcon, spec *caps.Status) { - h := &handler{rc: rc} +// New validates the declared readouts and creates their shared cache and collector. +// Call Run to collect opted-in performance readings independently of requests. +func New(rc Rcon, spec *caps.Status) *Source { + s := &Source{rc: rc, execSlot: make(chan struct{}, 1), commandTimeout: 3 * time.Second, cached: []Result{}} if spec != nil { - h.metrics = compile(spec.Metrics) + s.metrics = compile(spec.Metrics) + } + for _, m := range s.metrics { + s.automatic = s.automatic || m.performance != nil } - r.Get("/status", h.serve) + return s } -// compile builds the metric list, dropping any whose regex is invalid or -// lacks the required (?P…) group (logged) so one bad declaration -// can't disable the whole panel. +// Mount retains the standalone handler API. Production constructs one Source. +func Mount(r chi.Router, rc Rcon, spec *caps.Status) { New(rc, spec).Mount(r) } + +// Mount registers the status route using this source's shared readout cache. +func (s *Source) Mount(r chi.Router) { r.Get("/status", s.serve) } + +var metricID = regexp.MustCompile(`^[a-z0-9]([a-z0-9-]*[a-z0-9])?$`) + func compile(specs []caps.StatusMetric) []metric { - out := make([]metric, 0, len(specs)) - for _, s := range specs { - if s.ID == "" || s.Command == "" || s.Regex == "" { + out := make([]metric, 0, min(len(specs), 16)) + seen := map[string]bool{} + for _, s := range specs[:min(len(specs), 16)] { + if seen[s.ID] || len(s.ID) > 63 || !metricID.MatchString(s.ID) || s.Command == "" || len(s.Command) > maxCommandBytes || s.Regex == "" || len(s.Regex) > maxRegexBytes { continue } re, err := regexp.Compile(s.Regex) - if err != nil { - slog.Warn("invalid status metric regex; metric disabled", "metric", s.ID, "err", err) + if err != nil || re.SubexpIndex("value") < 0 { continue } - idx := re.SubexpIndex("value") - if idx < 0 { - slog.Warn("status metric regex has no (?P…) group; metric disabled", "metric", s.ID) - continue + m := metric{id: s.ID, displayName: s.DisplayName, command: s.Command, unit: s.Unit, re: re, valueIdx: re.SubexpIndex("value")} + if validPerformance(s.Performance) { + p := *s.Performance + if p.DegradedThreshold != nil { + value := *p.DegradedThreshold + p.DegradedThreshold = &value + } + m.performance = &p } - out = append(out, metric{ - id: s.ID, displayName: s.DisplayName, command: s.Command, - unit: s.Unit, re: re, valueIdx: idx, - }) + seen[s.ID] = true + out = append(out, m) } return out } -func (h *handler) serve(w http.ResponseWriter, _ *http.Request) { - h.mu.Lock() - fresh := !h.lastFetch.IsZero() && time.Since(h.lastFetch) < cacheTTL - cached := h.cached - h.mu.Unlock() - if fresh { - httpjson.Write(w, http.StatusOK, cached) +// Run refreshes opted-in performance readouts until ctx is canceled. +// Sources without performance declarations retain on-demand collection. +func (s *Source) Run(ctx context.Context) { + if !s.automatic { return } - - results := make([]Result, 0, len(h.metrics)) - for _, m := range h.metrics { - raw, err := h.rc.Exec(m.command) - if errors.Is(err, rcon.ErrDisabled) { - // No RCON: no live metrics. Empty array so the dashboard can - // render the panel uniformly for every game. - httpjson.Write(w, http.StatusOK, []Result{}) + for ctx.Err() == nil { + s.refresh(ctx) + timer := time.NewTimer(performanceInterval) + select { + case <-ctx.Done(): + timer.Stop() return + case <-timer.C: + } + } +} + +type commandResult struct { + raw string + err error + completed time.Time +} + +var errUnavailable = errors.New("status source unavailable") + +func (s *Source) execute(ctx context.Context, command string) commandResult { + if ctx.Err() != nil { + return commandResult{err: ctx.Err()} + } + select { + case s.execSlot <- struct{}{}: + default: + return commandResult{err: errUnavailable} + } + result := make(chan commandResult, 1) + go func() { + raw, err := s.rc.Exec(command) + if len(raw) > maxResponseBytes { + raw, err = "", errUnavailable + } + <-s.execSlot + result <- commandResult{raw: raw, err: err, completed: time.Now()} + }() + queryCtx, cancel := context.WithTimeout(ctx, s.commandTimeout) + defer cancel() + select { + case <-queryCtx.Done(): + return commandResult{err: queryCtx.Err()} + case reply := <-result: + if queryCtx.Err() != nil { + return commandResult{err: queryCtx.Err()} + } + return reply + } +} + +func (s *Source) refresh(ctx context.Context) { + s.refreshMu.Lock() + defer s.refreshMu.Unlock() + s.mu.Lock() + fresh := !s.automatic && !s.lastFetch.IsZero() && time.Since(s.lastFetch) < cacheTTL + s.mu.Unlock() + if fresh { + return + } + ctx, cancel := context.WithTimeout(ctx, cycleTimeout) + defer cancel() + results := make([]Result, 0, len(s.metrics)) + samples := map[string]performanceSample{} + commands := map[string]commandResult{} + for _, m := range s.metrics { + reply, ok := commands[m.command] + if !ok { + reply = s.execute(ctx, m.command) + commands[m.command] = reply + } + if errors.Is(reply.err, rcon.ErrDisabled) { + results = []Result{} + samples = map[string]performanceSample{} + break } res := Result{ID: m.id, DisplayName: m.displayName, Unit: m.unit} - if err != nil { - slog.Warn("status metric rcon", "metric", m.id, "err", err) - } else if sub := m.re.FindStringSubmatch(strings.ReplaceAll(raw, "\r", "")); sub != nil { - res.Value = strings.TrimSpace(sub[m.valueIdx]) + if reply.err == nil { + if sub := m.re.FindStringSubmatch(strings.ReplaceAll(reply.raw, "\r", "")); sub != nil { + res.Value = strings.TrimSpace(sub[m.valueIdx]) + } + } + if m.performance != nil { + if sample, valid := parsePerformance(res.Value, m.performance); valid { + sample.observed = reply.completed + samples[m.id] = sample + } } results = append(results, res) } + s.mu.Lock() + s.lastFetch, s.cached, s.samples = time.Now(), results, samples + s.mu.Unlock() +} - h.mu.Lock() - h.lastFetch = time.Now() - h.cached = results - h.mu.Unlock() - httpjson.Write(w, http.StatusOK, results) +func (s *Source) serve(w http.ResponseWriter, r *http.Request) { + s.mu.Lock() + fresh := !s.lastFetch.IsZero() && time.Since(s.lastFetch) < cacheTTL + s.mu.Unlock() + if !s.automatic && !fresh { + s.refresh(r.Context()) + } + s.mu.Lock() + cached := append([]Result{}, s.cached...) + if s.automatic && time.Since(s.lastFetch) >= performanceFreshness { + for i := range cached { + cached[i].Value = "" + } + } + s.mu.Unlock() + httpjson.Write(w, http.StatusOK, cached) } diff --git a/agent/specs.md b/agent/specs.md index 13d03c989..059790c31 100644 --- a/agent/specs.md +++ b/agent/specs.md @@ -21,7 +21,7 @@ The agent is a per-pod HTTP/HTTPS sidecar that runs inside every game pod to exp - **Quiesce**: Pause auto-saves and flush in-flight state before snapshots; run module-declared sequences over RCON and handle games that don't support it gracefully. - **Lifecycle**: Run module-declared stop sequences over RCON before the operator scales the server to zero. - **Actions**: Execute module-declared operator actions (templated RCON commands with user parameters) without agent code changes. -- **Status metrics**: Query live game metrics (TPS, world time, etc.) via RCON and regex extraction; module-declared in `capabilities.status.metrics[]`. +- **Status metrics**: Query live game metrics (TPS, world time, etc.) via RCON and regex extraction; module-declared in `capabilities.status.metrics[]`. Optional `performance` declarations add closed numeric signal/unit semantics and decimal degradation thresholds. One shared source serves both protected route mounts and Prometheus: opted-in templates use a 30-second interval cache, 3-second command/10-second cycle budgets, one outstanding legacy RCON call, and 75-second sample freshness. Scrapes issue no commands; invalid or stale values are absent and valid zero is retained. See `docs/monitoring.md` for bounds and metric families. - **Mods**: List, install, and uninstall game mods from a registry; track per-volume metadata; guard downloads with strict SSRF egress validation. - **Heartbeat**: Periodically patch the owning GameServer's status with lastHeartbeat, playersOnline, playersMax, gameVersion, and resource usage (CPU, memory, disk). @@ -86,7 +86,7 @@ Per-package roles: - **`mods`**: Tracks installed mods in a per-volume manifest (`.gameplane-mods.json`); downloads from registry with strict egress validation via `netguard.IsPublic`. - **`players`**: Queries player count, names, ban lists, and runs moderation actions over RCON; a single template-driven `commander` renders each moderation command from the module's declared `capabilities.players` templates (no per-game Go implementations), and reports capabilities from what the template declares. - **`quiesce`**: Runs module-declared sequences (e.g., Minecraft's `save-off` + `save-all flush`) over RCON; responds `quiesced: false` + reason when unsupported (not an error). -- **`rcon`**: Factory pattern for wire-protocol clients (`Valve/Source`, `Telnet`, `WebSocket`, `BattlEye`, `Satisfactory`, `Palworld`, `NuclearOption`, `REST`, `CLI`, `Disabled`); `Exec(cmd) (string, error)` interface. RCON reply packets are accepted up to 16394 bytes (a 4096-character Minecraft chunk at its worst-case UTF-8 byte length, plus the 10-byte packet header) and rejected as malformed outside `[10, 16394]` (F-104). +- **`rcon`**: Factory pattern for wire-protocol clients (`Valve/Source`, `Telnet`, `WebSocket`, `BattlEye`, `Satisfactory`, `Palworld`, `NuclearOption`, `REST`, `CLI`, `Disabled`); `Exec(cmd) (string, error)` interface. RCON reply packets are accepted up to 16394 bytes (a 4096-character Minecraft chunk at its worst-case UTF-8 byte length, plus the 10-byte packet header) and rejected as malformed outside `[10, 16394]` (F-104). Source RCON also caps aggregate response text at 8 MiB and enforces an absolute 30-second command exchange deadline after connection/authentication; each 400-ms fragment idle window is clamped to that deadline. Exceeding the aggregate cap or hard deadline drops the connection and returns an error without partial output. The status collector separately accepts at most 64 KiB. - **`status`**: Runs module-declared metrics queries over RCON; each metric specifies a command and a regex with named group `"value"` for extraction. - **`usage`**: Reads CPU/memory from `/proc` (proc mode, default) or cgroup v2; disk via `statfs`; exposes `Sample` with `Known` flags so callers distinguish "unknown" from "zero". @@ -210,7 +210,7 @@ All endpoints on the `--addr` control mux, except `/healthz`, return `401 Unauth - **Resource usage is in-pod**: The `usage` package reads from `/proc` or cgroups; no external metrics pipeline required. Cgroup mode is a fallback for older clusters; proc mode (default in production) requires the operator to set `ShareProcessNamespace: true`. - **Module capabilities drive behavior**: Every game-specific handler (players, quiesce, lifecycle, status, actions) reads its config from `--capabilities` (JSON unmarshaled into `caps.Spec`). New games require no agent code change. - **RCON connection errors are graceful**: A lost RCON connection does not crash the agent. `console`, `players`, `quiesce`, and `lifecycle` handlers catch connection errors and return appropriate HTTP status (e.g., `502 Bad Gateway`). -- **Source RCON sends complete frames**: Each AUTH or command frame is submitted in one socket write, with its existing byte-size limit, request ID, type, body, and two NUL terminators preserved. This avoids separate header-only writes rejected by vanilla Minecraft's RCON reader. A short write is an error and drops the connection without replaying the command; response grace and healthy connection reuse remain unchanged. +- **Source RCON sends complete frames**: Each AUTH or command frame is submitted in one socket write, with its existing byte-size limit, request ID, type, body, and two NUL terminators preserved. This avoids separate header-only writes rejected by vanilla Minecraft's RCON reader. A short write is an error and drops the connection without replaying the command. Dialing has a five-second limit; authentication has an absolute ten-second limit including continuous preamble packets. Healthy idle completion permits connection reuse; aggregate response overflow or absolute command deadline exhaustion discards partial output and drops the connection. - **Log streams are tail-only**: The `logs` package does not support random-access reads. It streams from the current end (live mode) or from file start (backlog mode); clients must handle partial output and reconnection. ## Dependencies @@ -244,7 +244,7 @@ The agent and operator modules use `k8s.io/apimachinery`/`k8s.io/client-go` v0.3 - `battleye`: UDP-based CRC32 packet framing. - `satisfactory`: HTTPS JSON function-call API with session token. - `palworld`: HTTP Basic REST admin API. - - `nuclearoption`: Dedicated TCP stream for Nuclear Option remote command protocol. + - `nuclearoption`: Dedicated TCP stream for Nuclear Option remote command protocol. One 30-second deadline bounds dialing, request writes, and response reads; responses are capped at 10 MiB. A timed-out partial exchange closes the connection before the next command. - `rest`: Generic HTTP/JSON REST console client with lazy auth (`PassFn`), 1 MiB bounded body read, configurable timeouts, auth failure cooldown, and loopback TLS protection. Provides adapters for FiveM (txAdmin: `POST /fxserver/commands`, bearer/X-TxAdmin-Token, `{"action":"console","parameter":""}`), Farming Simulator 25 (`POST /api/console`, Basic auth, `{"command":""}`), and generic REST (`POST /api/command`). - `cli`: Container stdin/PTY / local execution client (Option A in OPEN-DECISIONS.md). Drives commands via named FIFO pipe (`GAMEPLANE_CLI_PIPE`) or local process shell execution without requiring remote TCP networking or password secrets, while web console continues to attach via pod-attach PTY. diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backups.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backups.yaml index 0fc5ae27d..2c964a9c7 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backups.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backups.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: c74f1d52f8257b6d1a091e0d84d4aa139aec6b83ef487004d31f59ef137f6170 + gameplane.local/crd-bundle-sha256: 519f827ad83dfea7450e7d4eb30f2433a7b3c04f111795fcef0496d81ee3a770 controller-gen.kubebuilder.io/version: v0.22.0 name: backups.gameplane.local spec: diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backupschedules.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backupschedules.yaml index 2bdad0023..f35faf587 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backupschedules.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backupschedules.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: c74f1d52f8257b6d1a091e0d84d4aa139aec6b83ef487004d31f59ef137f6170 + gameplane.local/crd-bundle-sha256: 519f827ad83dfea7450e7d4eb30f2433a7b3c04f111795fcef0496d81ee3a770 controller-gen.kubebuilder.io/version: v0.22.0 name: backupschedules.gameplane.local spec: diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_clusters.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_clusters.yaml index b2a11b475..a00a67448 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_clusters.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_clusters.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: c74f1d52f8257b6d1a091e0d84d4aa139aec6b83ef487004d31f59ef137f6170 + gameplane.local/crd-bundle-sha256: 519f827ad83dfea7450e7d4eb30f2433a7b3c04f111795fcef0496d81ee3a770 controller-gen.kubebuilder.io/version: v0.22.0 name: clusters.gameplane.local spec: diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gameservers.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gameservers.yaml index f768c5616..dcbc60df0 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gameservers.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gameservers.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: c74f1d52f8257b6d1a091e0d84d4aa139aec6b83ef487004d31f59ef137f6170 + gameplane.local/crd-bundle-sha256: 519f827ad83dfea7450e7d4eb30f2433a7b3c04f111795fcef0496d81ee3a770 controller-gen.kubebuilder.io/version: v0.22.0 name: gameservers.gameplane.local spec: diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gametemplates.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gametemplates.yaml index 7408d7b80..97d49d8bd 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gametemplates.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gametemplates.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: c74f1d52f8257b6d1a091e0d84d4aa139aec6b83ef487004d31f59ef137f6170 + gameplane.local/crd-bundle-sha256: 519f827ad83dfea7450e7d4eb30f2433a7b3c04f111795fcef0496d81ee3a770 controller-gen.kubebuilder.io/version: v0.22.0 name: gametemplates.gameplane.local spec: @@ -961,6 +961,41 @@ spec: maxLength: 63 pattern: ^[a-z0-9]([a-z0-9-]*[a-z0-9])?$ type: string + performance: + description: |- + Performance opts this existing readout into cached numeric monitoring. + Display-only strings are never inferred to be performance measurements. + properties: + degradedThreshold: + description: |- + DegradedThreshold is a finite nonnegative decimal string in the input + unit. Rates below it, or durations above it, are degraded. Omit to + disable alerts. A string keeps JSON/schema handling portable. + maxLength: 16 + pattern: ^(0|[1-9][0-9]{0,8})(\.[0-9]{1,6})?$ + type: string + signal: + enum: + - tick_rate + - tick_duration + - frame_rate + type: string + unit: + enum: + - per_second + - seconds + - milliseconds + type: string + required: + - signal + - unit + type: object + x-kubernetes-validations: + - message: duration requires seconds/milliseconds; rates + require per_second + rule: 'self.signal == ''tick_duration'' ? self.unit + in [''seconds'', ''milliseconds''] : self.unit == + ''per_second''' regex: description: |- Regex extracts the value via the named group "value", e.g. diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modules.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modules.yaml index 2fe842fde..099d0e3b0 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modules.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modules.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: c74f1d52f8257b6d1a091e0d84d4aa139aec6b83ef487004d31f59ef137f6170 + gameplane.local/crd-bundle-sha256: 519f827ad83dfea7450e7d4eb30f2433a7b3c04f111795fcef0496d81ee3a770 controller-gen.kubebuilder.io/version: v0.22.0 name: modules.gameplane.local spec: diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modulesources.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modulesources.yaml index 39450ee94..e31ebb242 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modulesources.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modulesources.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: c74f1d52f8257b6d1a091e0d84d4aa139aec6b83ef487004d31f59ef137f6170 + gameplane.local/crd-bundle-sha256: 519f827ad83dfea7450e7d4eb30f2433a7b3c04f111795fcef0496d81ee3a770 controller-gen.kubebuilder.io/version: v0.22.0 name: modulesources.gameplane.local spec: diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_networkcaptures.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_networkcaptures.yaml index c88fd000e..565e4fb47 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_networkcaptures.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_networkcaptures.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: c74f1d52f8257b6d1a091e0d84d4aa139aec6b83ef487004d31f59ef137f6170 + gameplane.local/crd-bundle-sha256: 519f827ad83dfea7450e7d4eb30f2433a7b3c04f111795fcef0496d81ee3a770 controller-gen.kubebuilder.io/version: v0.22.0 name: networkcaptures.gameplane.local spec: diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_restores.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_restores.yaml index a57f794c0..ab12e096d 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_restores.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_restores.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: c74f1d52f8257b6d1a091e0d84d4aa139aec6b83ef487004d31f59ef137f6170 + gameplane.local/crd-bundle-sha256: 519f827ad83dfea7450e7d4eb30f2433a7b3c04f111795fcef0496d81ee3a770 controller-gen.kubebuilder.io/version: v0.22.0 name: restores.gameplane.local spec: diff --git a/charts/gameplane/crd-manifests/gameplane.local_backups.yaml b/charts/gameplane/crd-manifests/gameplane.local_backups.yaml index 0fc5ae27d..2c964a9c7 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_backups.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_backups.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: c74f1d52f8257b6d1a091e0d84d4aa139aec6b83ef487004d31f59ef137f6170 + gameplane.local/crd-bundle-sha256: 519f827ad83dfea7450e7d4eb30f2433a7b3c04f111795fcef0496d81ee3a770 controller-gen.kubebuilder.io/version: v0.22.0 name: backups.gameplane.local spec: diff --git a/charts/gameplane/crd-manifests/gameplane.local_backupschedules.yaml b/charts/gameplane/crd-manifests/gameplane.local_backupschedules.yaml index 2bdad0023..f35faf587 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_backupschedules.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_backupschedules.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: c74f1d52f8257b6d1a091e0d84d4aa139aec6b83ef487004d31f59ef137f6170 + gameplane.local/crd-bundle-sha256: 519f827ad83dfea7450e7d4eb30f2433a7b3c04f111795fcef0496d81ee3a770 controller-gen.kubebuilder.io/version: v0.22.0 name: backupschedules.gameplane.local spec: diff --git a/charts/gameplane/crd-manifests/gameplane.local_clusters.yaml b/charts/gameplane/crd-manifests/gameplane.local_clusters.yaml index b2a11b475..a00a67448 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_clusters.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_clusters.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: c74f1d52f8257b6d1a091e0d84d4aa139aec6b83ef487004d31f59ef137f6170 + gameplane.local/crd-bundle-sha256: 519f827ad83dfea7450e7d4eb30f2433a7b3c04f111795fcef0496d81ee3a770 controller-gen.kubebuilder.io/version: v0.22.0 name: clusters.gameplane.local spec: diff --git a/charts/gameplane/crd-manifests/gameplane.local_gameservers.yaml b/charts/gameplane/crd-manifests/gameplane.local_gameservers.yaml index f768c5616..dcbc60df0 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_gameservers.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_gameservers.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: c74f1d52f8257b6d1a091e0d84d4aa139aec6b83ef487004d31f59ef137f6170 + gameplane.local/crd-bundle-sha256: 519f827ad83dfea7450e7d4eb30f2433a7b3c04f111795fcef0496d81ee3a770 controller-gen.kubebuilder.io/version: v0.22.0 name: gameservers.gameplane.local spec: diff --git a/charts/gameplane/crd-manifests/gameplane.local_gametemplates.yaml b/charts/gameplane/crd-manifests/gameplane.local_gametemplates.yaml index 7408d7b80..97d49d8bd 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_gametemplates.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_gametemplates.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: c74f1d52f8257b6d1a091e0d84d4aa139aec6b83ef487004d31f59ef137f6170 + gameplane.local/crd-bundle-sha256: 519f827ad83dfea7450e7d4eb30f2433a7b3c04f111795fcef0496d81ee3a770 controller-gen.kubebuilder.io/version: v0.22.0 name: gametemplates.gameplane.local spec: @@ -961,6 +961,41 @@ spec: maxLength: 63 pattern: ^[a-z0-9]([a-z0-9-]*[a-z0-9])?$ type: string + performance: + description: |- + Performance opts this existing readout into cached numeric monitoring. + Display-only strings are never inferred to be performance measurements. + properties: + degradedThreshold: + description: |- + DegradedThreshold is a finite nonnegative decimal string in the input + unit. Rates below it, or durations above it, are degraded. Omit to + disable alerts. A string keeps JSON/schema handling portable. + maxLength: 16 + pattern: ^(0|[1-9][0-9]{0,8})(\.[0-9]{1,6})?$ + type: string + signal: + enum: + - tick_rate + - tick_duration + - frame_rate + type: string + unit: + enum: + - per_second + - seconds + - milliseconds + type: string + required: + - signal + - unit + type: object + x-kubernetes-validations: + - message: duration requires seconds/milliseconds; rates + require per_second + rule: 'self.signal == ''tick_duration'' ? self.unit + in [''seconds'', ''milliseconds''] : self.unit == + ''per_second''' regex: description: |- Regex extracts the value via the named group "value", e.g. diff --git a/charts/gameplane/crd-manifests/gameplane.local_modules.yaml b/charts/gameplane/crd-manifests/gameplane.local_modules.yaml index 2fe842fde..099d0e3b0 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_modules.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_modules.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: c74f1d52f8257b6d1a091e0d84d4aa139aec6b83ef487004d31f59ef137f6170 + gameplane.local/crd-bundle-sha256: 519f827ad83dfea7450e7d4eb30f2433a7b3c04f111795fcef0496d81ee3a770 controller-gen.kubebuilder.io/version: v0.22.0 name: modules.gameplane.local spec: diff --git a/charts/gameplane/crd-manifests/gameplane.local_modulesources.yaml b/charts/gameplane/crd-manifests/gameplane.local_modulesources.yaml index 39450ee94..e31ebb242 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_modulesources.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_modulesources.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: c74f1d52f8257b6d1a091e0d84d4aa139aec6b83ef487004d31f59ef137f6170 + gameplane.local/crd-bundle-sha256: 519f827ad83dfea7450e7d4eb30f2433a7b3c04f111795fcef0496d81ee3a770 controller-gen.kubebuilder.io/version: v0.22.0 name: modulesources.gameplane.local spec: diff --git a/charts/gameplane/crd-manifests/gameplane.local_networkcaptures.yaml b/charts/gameplane/crd-manifests/gameplane.local_networkcaptures.yaml index c88fd000e..565e4fb47 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_networkcaptures.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_networkcaptures.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: c74f1d52f8257b6d1a091e0d84d4aa139aec6b83ef487004d31f59ef137f6170 + gameplane.local/crd-bundle-sha256: 519f827ad83dfea7450e7d4eb30f2433a7b3c04f111795fcef0496d81ee3a770 controller-gen.kubebuilder.io/version: v0.22.0 name: networkcaptures.gameplane.local spec: diff --git a/charts/gameplane/crd-manifests/gameplane.local_restores.yaml b/charts/gameplane/crd-manifests/gameplane.local_restores.yaml index a57f794c0..ab12e096d 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_restores.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_restores.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: c74f1d52f8257b6d1a091e0d84d4aa139aec6b83ef487004d31f59ef137f6170 + gameplane.local/crd-bundle-sha256: 519f827ad83dfea7450e7d4eb30f2433a7b3c04f111795fcef0496d81ee3a770 controller-gen.kubebuilder.io/version: v0.22.0 name: restores.gameplane.local spec: diff --git a/charts/gameplane/dashboards/gameplane-workloads.json b/charts/gameplane/dashboards/gameplane-workloads.json new file mode 100644 index 000000000..f0a978b26 --- /dev/null +++ b/charts/gameplane/dashboards/gameplane-workloads.json @@ -0,0 +1,345 @@ +{ + "title": "Gameplane Workloads", + "uid": "gameplane-workloads", + "tags": [ + "gameplane" + ], + "schemaVersion": 39, + "version": 1, + "editable": true, + "refresh": "30s", + "time": { + "from": "now-6h", + "to": "now" + }, + "templating": { + "list": [ + { + "name": "datasource", + "type": "datasource", + "query": "prometheus" + }, + { + "name": "namespace", + "type": "textbox", + "label": "Games namespace", + "current": { + "value": "gameplane-games", + "text": "gameplane-games" + } + } + ] + }, + "panels": [ + { + "id": 1, + "title": "Scrape health", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 0 + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "noValue": "Unknown", + "custom": { + "spanNulls": false + } + } + }, + "targets": [ + { + "refId": "A", + "expr": "max by (namespace,pod,gameplane_component) (up{gameplane_component=~\"agent|tunnel\",namespace=~\"$namespace\"})", + "legendFormat": "{{namespace}} / {{pod}} {{metric}}" + } + ] + }, + { + "id": 2, + "title": "Relay readiness", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 0 + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "noValue": "Unknown", + "custom": { + "spanNulls": false + } + } + }, + "targets": [ + { + "refId": "A", + "expr": "max by (namespace,pod) (gameplane_tunnel_ready{namespace=~\"$namespace\"})", + "legendFormat": "{{namespace}} / {{pod}} {{metric}}" + } + ] + }, + { + "id": 3, + "title": "Relay health known", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 8 + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "noValue": "Unknown", + "custom": { + "spanNulls": false + } + } + }, + "targets": [ + { + "refId": "A", + "expr": "max by (namespace,pod) (gameplane_tunnel_connection_known{namespace=~\"$namespace\"})", + "legendFormat": "{{namespace}} / {{pod}} {{metric}}" + } + ] + }, + { + "id": 4, + "title": "Game rates (declared signals)", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 8 + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "unit": "hertz", + "noValue": "Unknown", + "custom": { + "spanNulls": false + } + } + }, + "targets": [ + { + "refId": "A", + "expr": "gameplane_agent_performance_value{namespace=~\"$namespace\",unit=\"hertz\"}", + "legendFormat": "{{namespace}} / {{pod}} {{metric}}" + } + ] + }, + { + "id": 5, + "title": "Game tick duration", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 16 + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "noValue": "Unknown", + "custom": { + "spanNulls": false + } + } + }, + "targets": [ + { + "refId": "A", + "expr": "gameplane_agent_performance_value{namespace=~\"$namespace\",unit=\"seconds\"}", + "legendFormat": "{{namespace}} / {{pod}} {{metric}}" + } + ] + }, + { + "id": 6, + "title": "Performance available", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 16 + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "noValue": "Unknown", + "custom": { + "spanNulls": false + } + } + }, + "targets": [ + { + "refId": "A", + "expr": "gameplane_agent_performance_available{namespace=~\"$namespace\"}", + "legendFormat": "{{namespace}} / {{pod}} {{metric}}" + } + ] + }, + { + "id": 7, + "title": "Container CPU throttled periods", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 24 + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "noValue": "Unknown", + "custom": { + "spanNulls": false + } + } + }, + "targets": [ + { + "refId": "A", + "expr": "max by (namespace,pod) (rate(container_cpu_cfs_throttled_periods_total{namespace=~\"$namespace\",container=\"game\"}[5m]) / rate(container_cpu_cfs_periods_total{namespace=~\"$namespace\",container=\"game\"}[5m]))", + "legendFormat": "{{namespace}} / {{pod}} {{metric}}" + } + ] + }, + { + "id": 8, + "title": "Container working set / limit", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 24 + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "noValue": "Unknown", + "custom": { + "spanNulls": false + } + } + }, + "targets": [ + { + "refId": "A", + "expr": "max by (namespace,pod) (container_memory_working_set_bytes{namespace=~\"$namespace\",container=\"game\"} / (container_spec_memory_limit_bytes{namespace=~\"$namespace\",container=\"game\"} > 0))", + "legendFormat": "{{namespace}} / {{pod}} {{metric}}" + } + ] + }, + { + "id": 9, + "title": "Container restarts", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 32 + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "noValue": "Unknown", + "custom": { + "spanNulls": false + } + } + }, + "targets": [ + { + "refId": "A", + "expr": "max by (namespace,pod,container) (increase(kube_pod_container_status_restarts_total{namespace=~\"$namespace\"}[15m]))", + "legendFormat": "{{namespace}} / {{pod}} {{metric}}" + } + ] + }, + { + "id": 10, + "title": "Pending Pods", + "type": "timeseries", + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 32 + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "noValue": "Unknown", + "custom": { + "spanNulls": false + } + } + }, + "targets": [ + { + "refId": "A", + "expr": "max by (namespace,pod) (kube_pod_status_phase{namespace=~\"$namespace\",phase=\"Pending\"})", + "legendFormat": "{{namespace}} / {{pod}} {{metric}}" + } + ] + } + ] +} diff --git a/charts/gameplane/templates/NOTES.txt b/charts/gameplane/templates/NOTES.txt index bac02868a..63b9f80d1 100644 --- a/charts/gameplane/templates/NOTES.txt +++ b/charts/gameplane/templates/NOTES.txt @@ -10,6 +10,16 @@ Operator leader election is enabled automatically because operator.replicas is greater than one. The saved leaderElect=false value remains compatible with upgrades; all operator replicas coordinate through the leader-election Lease. {{- end }} +{{- if .Values.serviceMonitors.enabled }} + +Monitoring: configure Prometheus monitor label selectors AND discovery of the +release and games namespaces. Verify agent and tunnel targets have up=1. +See docs/monitoring.md and docs/examples/monitoring for a complete configuration. +{{- if .Values.serviceMonitors.externallyManagedPolicy }} +External scrape policies are selected: this setting opens no network access. +Ensure your policies allow the scraper to agent TCP 9090 and tunnel TCP 8091. +{{- end }} +{{- end }} {{- if .Values.api.standalone }} Standalone central panel: no local game cluster or Kubernetes API credentials. diff --git a/charts/gameplane/templates/grafana-dashboard.yaml b/charts/gameplane/templates/grafana-dashboard.yaml index de1052252..9c9664fb2 100644 --- a/charts/gameplane/templates/grafana-dashboard.yaml +++ b/charts/gameplane/templates/grafana-dashboard.yaml @@ -15,6 +15,8 @@ metadata: {{- toYaml . | nindent 4 }} {{- end }} data: + gameplane-workloads.json: |- + {{- .Files.Get "dashboards/gameplane-workloads.json" | nindent 4 }} gameplane-operator.json: |- {{- .Files.Get "dashboards/gameplane-operator.json" | nindent 4 }} {{- end }} diff --git a/charts/gameplane/templates/monitoring-rules.yaml b/charts/gameplane/templates/monitoring-rules.yaml new file mode 100644 index 000000000..98b383683 --- /dev/null +++ b/charts/gameplane/templates/monitoring-rules.yaml @@ -0,0 +1,72 @@ +{{- if and .Values.prometheusRules.enabled (eq (include "gameplane.operatorEnabled" .) "true") }} +apiVersion: monitoring.coreos.com/v1 +kind: PrometheusRule +metadata: + name: gameplane-workloads + namespace: {{ .Release.Namespace }} + labels: + {{- include "gameplane.labels" . | nindent 4 }} + {{- with .Values.prometheusRules.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + groups: + - name: gameplane.workloads + rules: + - alert: GameplaneScrapeUnavailable + expr: max by (namespace, pod, gameplane_component) (up{gameplane_component=~"agent|tunnel",namespace="{{ .Values.gamesNamespace }}"}) == 0 + for: 5m + labels: { severity: warning } + annotations: { summary: "Gameplane metrics target cannot be scraped" } + - alert: GameplaneRelayUnavailable + expr: (max by (namespace, pod) (gameplane_tunnel_ready{namespace="{{ .Values.gamesNamespace }}"}) == 0) and on (namespace, pod) (max by (namespace, pod) (gameplane_tunnel_connection_known{namespace="{{ .Values.gamesNamespace }}"}) == 1) + for: 2m + labels: { severity: warning } + annotations: { summary: "Gameplane relay connection or forwards are unavailable" } + - alert: GameplaneRelayHealthUnknown + expr: (max by (namespace, pod) (gameplane_tunnel_connection_known{namespace="{{ .Values.gamesNamespace }}"}) == 0) and on (namespace, pod) (max by (namespace, pod) (gameplane_tunnel_connection_supported{namespace="{{ .Values.gamesNamespace }}"}) == 1) + for: 5m + labels: { severity: info } + annotations: { summary: "Gameplane provider cannot confirm live relay health" } + - alert: GameplanePerformanceUnknown + expr: max by (namespace, pod, metric, signal, unit) (gameplane_agent_performance_available{namespace="{{ .Values.gamesNamespace }}"}) == 0 + for: 5m + labels: { severity: warning } + annotations: { summary: "Declared game performance measurement is unavailable" } + - alert: GameplanePerformanceDegraded + expr: | + (max by (namespace, pod, metric, signal, unit) (gameplane_agent_performance_value{namespace="{{ .Values.gamesNamespace }}"}) + < on (namespace, pod, metric, signal, unit) + max by (namespace, pod, metric, signal, unit) (gameplane_agent_performance_degraded_threshold{namespace="{{ .Values.gamesNamespace }}",direction="below"})) + or + (max by (namespace, pod, metric, signal, unit) (gameplane_agent_performance_value{namespace="{{ .Values.gamesNamespace }}"}) + > on (namespace, pod, metric, signal, unit) + max by (namespace, pod, metric, signal, unit) (gameplane_agent_performance_degraded_threshold{namespace="{{ .Values.gamesNamespace }}",direction="above"})) + for: 5m + labels: { severity: warning } + annotations: { summary: "Game performance crossed its module-declared threshold" } + - alert: GameplaneCPUThrottled + expr: max by (namespace, pod, container) (rate(container_cpu_cfs_throttled_periods_total{namespace="{{ .Values.gamesNamespace }}",container="game"}[5m]) / rate(container_cpu_cfs_periods_total{namespace="{{ .Values.gamesNamespace }}",container="game"}[5m])) > 0.25 + for: 10m + labels: { severity: warning } + annotations: { summary: "Game container CPU is throttled in more than 25% of periods" } + - alert: GameplaneMemoryPressure + expr: max by (namespace, pod, container) (container_memory_working_set_bytes{namespace="{{ .Values.gamesNamespace }}",container="game"} / (container_spec_memory_limit_bytes{namespace="{{ .Values.gamesNamespace }}",container="game"} > 0)) > 0.9 + for: 10m + labels: { severity: warning } + annotations: { summary: "Game container working set exceeds 90% of its memory limit" } + - alert: GameplaneContainerOOM + expr: (max by (namespace, pod, container) (increase(kube_pod_container_status_restarts_total{namespace="{{ .Values.gamesNamespace }}"}[10m])) > 0) and on (namespace, pod, container) (max by (namespace, pod, container) (kube_pod_container_status_last_terminated_reason{namespace="{{ .Values.gamesNamespace }}",reason="OOMKilled"}) == 1) + labels: { severity: warning } + annotations: { summary: "Gameplane container restarted after an OOM kill" } + - alert: GameplanePodPending + expr: max by (namespace, pod) (kube_pod_status_phase{namespace="{{ .Values.gamesNamespace }}",phase="Pending"}) == 1 + for: 10m + labels: { severity: warning } + annotations: { summary: "Gameplane Pod has remained Pending for ten minutes" } + - alert: GameplaneNodePressure + expr: (max by (node, condition) (kube_node_status_condition{condition=~"MemoryPressure|DiskPressure|PIDPressure",status="true"}) == 1) and on (node) max by (node) (kube_pod_info{namespace="{{ .Values.gamesNamespace }}"}) + for: 5m + labels: { severity: warning } + annotations: { summary: "A node hosting Gameplane Pods reports resource pressure" } +{{- end }} diff --git a/charts/gameplane/templates/monitoring-validation.yaml b/charts/gameplane/templates/monitoring-validation.yaml new file mode 100644 index 000000000..15fcacc60 --- /dev/null +++ b/charts/gameplane/templates/monitoring-validation.yaml @@ -0,0 +1,3 @@ +{{- if and (eq (include "gameplane.operatorEnabled" .) "true") .Values.serviceMonitors.enabled .Values.networkPolicies.enabled (not .Values.serviceMonitors.scrapeNamespaceSelector) (not .Values.serviceMonitors.externallyManagedPolicy) }} +{{- fail "serviceMonitors.enabled with networkPolicies.enabled requires serviceMonitors.scrapeNamespaceSelector (for example matchLabels: {kubernetes.io/metadata.name: monitoring}); alternatively set serviceMonitors.externallyManagedPolicy=true and supply your own scrape ingress policies" }} +{{- end }} diff --git a/charts/gameplane/templates/networkpolicies.yaml b/charts/gameplane/templates/networkpolicies.yaml index 612fc5071..d652bd58b 100644 --- a/charts/gameplane/templates/networkpolicies.yaml +++ b/charts/gameplane/templates/networkpolicies.yaml @@ -163,6 +163,24 @@ spec: {{- toYaml . | nindent 12 }} ports: - { protocol: TCP, port: 9090 } +--- +apiVersion: networking.k8s.io/v1 +kind: NetworkPolicy +metadata: + name: allow-prometheus-to-tunnel + namespace: {{ $.Values.gamesNamespace }} + labels: {{- include "gameplane.labels" $ | nindent 4 }} +spec: + podSelector: + matchLabels: + app.kubernetes.io/name: gameplane-tunnel + policyTypes: [Ingress] + ingress: + - from: + - namespaceSelector: + {{- toYaml . | nindent 12 }} + ports: + - { protocol: TCP, port: 8091 } {{- end }} {{- end }} --- diff --git a/charts/gameplane/templates/servicemonitors.yaml b/charts/gameplane/templates/servicemonitors.yaml index 4521db18e..2b5b6834b 100644 --- a/charts/gameplane/templates/servicemonitors.yaml +++ b/charts/gameplane/templates/servicemonitors.yaml @@ -107,6 +107,34 @@ spec: - port: metrics interval: 30s path: /metrics + relabelings: + - targetLabel: gameplane_component + replacement: agent + - sourceLabels: [__meta_kubernetes_pod_label_app_kubernetes_io_instance] + targetLabel: server +--- +apiVersion: monitoring.coreos.com/v1 +kind: PodMonitor +metadata: + name: gameplane-tunnel + namespace: {{ .Values.gamesNamespace }} + labels: + {{- include "gameplane.labels" . | nindent 4 }} + {{- with .Values.serviceMonitors.labels }} + {{- toYaml . | nindent 4 }} + {{- end }} +spec: + selector: + matchLabels: { app.kubernetes.io/name: gameplane-tunnel } + podMetricsEndpoints: + - port: relay-metrics + interval: 15s + path: /metrics + relabelings: + - targetLabel: gameplane_component + replacement: tunnel + - sourceLabels: [__meta_kubernetes_pod_label_app_kubernetes_io_instance] + targetLabel: server {{- end }} {{- end }} diff --git a/charts/gameplane/tests/monitoring-rules.sh b/charts/gameplane/tests/monitoring-rules.sh new file mode 100644 index 000000000..6e39efd8e --- /dev/null +++ b/charts/gameplane/tests/monitoring-rules.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# Render and evaluate shipped rules in CI, using the same Prometheus as E2E. +set -euo pipefail +cd "$(dirname "$0")/../../.." +scratch="$(mktemp -d)" +trap 'rm -rf "$scratch"' EXIT +helm template gameplane charts/gameplane -f docs/examples/monitoring/gameplane-values.yaml > "$scratch/rendered.yaml" +python3 - "$scratch" <<'PY' +import pathlib,sys,yaml +root=pathlib.Path(sys.argv[1]) +groups=[] +for document in yaml.safe_load_all((root/'rendered.yaml').read_text()): + if document and document.get('kind')=='PrometheusRule': + groups.extend(document['spec']['groups']) +(root/'rules.yaml').write_text(yaml.safe_dump({'groups':groups})) +PY +cp charts/gameplane/tests/monitoring-rules.test.yaml "$scratch/tests.yaml" +# mktemp creates a private directory, but promtool runs as the image's nobody +# user. These generated fixtures contain no credentials and are mounted read-only. +chmod 755 "$scratch" +chmod 644 "$scratch/rules.yaml" "$scratch/tests.yaml" +docker run --rm --volume "$scratch:/work:ro" --workdir /work --entrypoint /bin/promtool \ + prom/prometheus:v3.13.4 test rules tests.yaml diff --git a/charts/gameplane/tests/monitoring-rules.test.yaml b/charts/gameplane/tests/monitoring-rules.test.yaml new file mode 100644 index 000000000..2d285dedf --- /dev/null +++ b/charts/gameplane/tests/monitoring-rules.test.yaml @@ -0,0 +1,152 @@ +rule_files: [rules.yaml] +evaluation_interval: 1m +tests: + - interval: 1m + input_series: + - series: 'gameplane_agent_performance_value{namespace="gameplane-games",pod="duration-0",metric="tick",signal="tick_duration",unit="seconds"}' + values: '0.06x10 0.04x10' + - series: 'gameplane_agent_performance_degraded_threshold{namespace="gameplane-games",pod="duration-0",metric="tick",signal="tick_duration",unit="seconds",direction="above"}' + values: '0.05x20' + - series: 'container_memory_working_set_bytes{namespace="gameplane-games",pod="unlimited",container="game"}' + values: '1000x20' + - series: 'container_spec_memory_limit_bytes{namespace="gameplane-games",pod="unlimited",container="game"}' + values: '0x20' + alert_rule_test: + - eval_time: 6m + alertname: GameplanePerformanceDegraded + exp_alerts: + - exp_labels: {namespace: gameplane-games, pod: duration-0, metric: tick, signal: tick_duration, unit: seconds, severity: warning} + exp_annotations: {summary: "Game performance crossed its module-declared threshold"} + - eval_time: 12m + alertname: GameplanePerformanceDegraded + exp_alerts: [] + - eval_time: 16m + alertname: GameplaneMemoryPressure + exp_alerts: [] + - eval_time: 16m + alertname: GameplanePerformanceUnknown + exp_alerts: [] + - interval: 1m + input_series: + - series: 'up{namespace="gameplane-games",pod="game-0",gameplane_component="agent"}' + values: '0x10 1x10' + - series: 'gameplane_tunnel_ready{namespace="gameplane-games",pod="relay"}' + values: '0x10 1x10' + - series: 'gameplane_tunnel_connection_known{namespace="gameplane-games",pod="relay"}' + values: '1x20' + alert_rule_test: + - eval_time: 6m + alertname: GameplaneScrapeUnavailable + exp_alerts: + - exp_labels: {namespace: gameplane-games, pod: game-0, gameplane_component: agent, severity: warning} + exp_annotations: {summary: "Gameplane metrics target cannot be scraped"} + - eval_time: 6m + alertname: GameplaneRelayUnavailable + exp_alerts: + - exp_labels: {namespace: gameplane-games, pod: relay, severity: warning} + exp_annotations: {summary: "Gameplane relay connection or forwards are unavailable"} + - eval_time: 12m + alertname: GameplaneScrapeUnavailable + exp_alerts: [] + - eval_time: 12m + alertname: GameplaneRelayUnavailable + exp_alerts: [] + - interval: 1m + input_series: + - series: 'gameplane_agent_performance_value{namespace="gameplane-games",pod="game-0",metric="tps",signal="tick_rate",unit="hertz"}' + values: '0x10 stale _x10' + - series: 'gameplane_agent_performance_degraded_threshold{namespace="gameplane-games",pod="game-0",metric="tps",signal="tick_rate",unit="hertz",direction="below"}' + values: '18x20' + - series: 'gameplane_agent_performance_available{namespace="gameplane-games",pod="game-0",metric="tps",signal="tick_rate",unit="hertz"}' + values: '1x10 0x10' + - series: 'gameplane_tunnel_connection_known{namespace="gameplane-games",pod="playit"}' + values: '0x20' + - series: 'gameplane_tunnel_connection_supported{namespace="gameplane-games",pod="playit"}' + values: '0x20' + - series: 'gameplane_tunnel_connection_known{namespace="gameplane-games",pod="frp"}' + values: '0x10 1x10' + - series: 'gameplane_tunnel_connection_supported{namespace="gameplane-games",pod="frp"}' + values: '1x20' + - series: 'gameplane_tunnel_ready{namespace="gameplane-games",pod="playit"}' + values: '0x10 1x10' + alert_rule_test: + - eval_time: 6m + alertname: GameplanePerformanceDegraded + exp_alerts: + - exp_labels: {namespace: gameplane-games, pod: game-0, metric: tps, signal: tick_rate, unit: hertz, severity: warning} + exp_annotations: {summary: "Game performance crossed its module-declared threshold"} + - eval_time: 12m + alertname: GameplanePerformanceDegraded + exp_alerts: [] + - eval_time: 17m + alertname: GameplanePerformanceUnknown + exp_alerts: + - exp_labels: {namespace: gameplane-games, pod: game-0, metric: tps, signal: tick_rate, unit: hertz, severity: warning} + exp_annotations: {summary: "Declared game performance measurement is unavailable"} + - eval_time: 6m + alertname: GameplaneRelayHealthUnknown + exp_alerts: + - exp_labels: {namespace: gameplane-games, pod: frp, severity: info} + exp_annotations: {summary: "Gameplane provider cannot confirm live relay health"} + - eval_time: 6m + alertname: GameplaneRelayUnavailable + exp_alerts: [] + - eval_time: 12m + alertname: GameplaneRelayHealthUnknown + exp_alerts: [] + - interval: 1m + input_series: + - series: 'container_cpu_cfs_throttled_periods_total{namespace="gameplane-games",pod="game-0",container="game"}' + values: '0+50x25' + - series: 'container_cpu_cfs_periods_total{namespace="gameplane-games",pod="game-0",container="game"}' + values: '0+100x25' + - series: 'container_memory_working_set_bytes{namespace="gameplane-games",pod="game-0",container="game"}' + values: '95x25' + - series: 'container_spec_memory_limit_bytes{namespace="gameplane-games",pod="game-0",container="game"}' + values: '100x25' + - series: 'kube_pod_status_phase{namespace="gameplane-games",pod="pending",phase="Pending"}' + values: '1x25' + - series: 'kube_pod_container_status_restarts_total{namespace="gameplane-games",pod="game-0",container="game"}' + values: '0+1x25' + - series: 'kube_pod_container_status_last_terminated_reason{namespace="gameplane-games",pod="game-0",container="game",reason="OOMKilled"}' + values: '1x25' + - series: 'kube_node_status_condition{node="worker",condition="MemoryPressure",status="true"}' + values: '1x25' + - series: 'kube_pod_info{namespace="gameplane-games",pod="game-0",node="worker"}' + values: '1x25' + - series: 'gameplane_gameservers{phase="Failed",instance="operator-a"}' + values: '1x25' + - series: 'gameplane_gameservers{phase="Failed",instance="operator-b"}' + values: '1x25' + alert_rule_test: + - eval_time: 16m + alertname: GameplaneCPUThrottled + exp_alerts: + - exp_labels: {namespace: gameplane-games, pod: game-0, container: game, severity: warning} + exp_annotations: {summary: "Game container CPU is throttled in more than 25% of periods"} + - eval_time: 16m + alertname: GameplaneMemoryPressure + exp_alerts: + - exp_labels: {namespace: gameplane-games, pod: game-0, container: game, severity: warning} + exp_annotations: {summary: "Game container working set exceeds 90% of its memory limit"} + - eval_time: 16m + alertname: GameplanePodPending + exp_alerts: + - exp_labels: {namespace: gameplane-games, pod: pending, severity: warning} + exp_annotations: {summary: "Gameplane Pod has remained Pending for ten minutes"} + - eval_time: 16m + alertname: GameplaneContainerOOM + exp_alerts: + - exp_labels: {namespace: gameplane-games, pod: game-0, container: game, severity: warning} + exp_annotations: {summary: "Gameplane container restarted after an OOM kill"} + - eval_time: 16m + alertname: GameplaneNodePressure + exp_alerts: + - exp_labels: {node: worker, condition: MemoryPressure, severity: warning} + exp_annotations: {summary: "A node hosting Gameplane Pods reports resource pressure"} + promql_expr_test: + - expr: 'max(gameplane_gameservers{phase="Failed"})' + eval_time: 16m + exp_samples: + - labels: '{}' + value: 1 diff --git a/charts/gameplane/values.yaml b/charts/gameplane/values.yaml index 426dd9a77..f0aab3310 100644 --- a/charts/gameplane/values.yaml +++ b/charts/gameplane/values.yaml @@ -689,6 +689,9 @@ updates: channel: "" serviceMonitors: + # Explicit escape for independently managed scrape ingress policies. This + # does not open any ports; you must install and verify those policies. + externallyManagedPolicy: false enabled: false # when true, creates ServiceMonitors for operator/api/telemetry-receiver and a PodMonitor for agent sidecars on their separate plain metrics port 9090 (the control port 8090 stays mTLS-only) labels: {} # additional labels applied to ServiceMonitor/PodMonitor objects for Prometheus selector matching # Namespace selector (standard Kubernetes namespaceSelector label match @@ -700,9 +703,9 @@ serviceMonitors: # so the receiver ServiceMonitor also needs # api.telemetry.receiver.dashboard.tokenSecretRef.name) when # networkPolicies.enabled is also true. Required - # alongside enabled: true whenever networkPolicies.enabled is also true; - # left empty, both stay unreachable to Prometheus even though their - # ServiceMonitor/PodMonitor objects are rendered. Example: + # alongside enabled: true when the operator and networkPolicies are enabled; + # left empty, chart validation fails unless externallyManagedPolicy is true. + # Also permits tunnel health/metrics on port 8091. Example: # `{matchLabels: {kubernetes.io/metadata.name: monitoring}}`. scrapeNamespaceSelector: {} diff --git a/charts/tests/check_gateway.py b/charts/tests/check_gateway.py index 83f174ed6..819e46979 100644 --- a/charts/tests/check_gateway.py +++ b/charts/tests/check_gateway.py @@ -81,7 +81,9 @@ def main(): gateway["namespaces"] = ["games-one", "games-two"] objects = render({ "gateway": gateway, "api": {"enabled": False, "telemetry": {"receiver": {"enabled": True}}}, - "serviceMonitors": {"enabled": True}, + "serviceMonitors": {"enabled": True, "scrapeNamespaceSelector": { + "matchLabels": {"kubernetes.io/metadata.name": "monitoring"}, + }}, "clusterOps": {"enabled": True}, }, upgrade=True) assert not any(obj["metadata"]["name"] in {"gameplane-api", "gameplane-api-data", "gameplane-web", "gameplane-strategy-migrate", "gameplane-telemetry-receiver"} for obj in objects) diff --git a/charts/tests/check_standalone.py b/charts/tests/check_standalone.py index e30ea1211..b21e5fdc2 100644 --- a/charts/tests/check_standalone.py +++ b/charts/tests/check_standalone.py @@ -86,6 +86,8 @@ def main(): assert any(rule.get("resources") == ["clusters"] and "update" in rule["verbs"] for rule in api_rules) assert "--standalone" not in one(defaults, "Deployment", "gameplane-api")["spec"]["template"]["spec"]["containers"][0]["args"] + # With the operator disabled no chart NetworkPolicies render, so monitors + # must remain valid without a game-namespace scrape selector. values = { "operator": {"enabled": False}, "api": {"standalone": True}, "serviceMonitors": {"enabled": True}, "prometheusRules": {"enabled": True}, diff --git a/deploy/kind/monitoring.sh b/deploy/kind/monitoring.sh new file mode 100644 index 000000000..f5c2e4a4e --- /dev/null +++ b/deploy/kind/monitoring.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# CI-only real scrape fixture; never invoke as a local validation shortcut. +set -euo pipefail +cd "$(dirname "$0")/../.." +context="${CTX:-kind-gameplane-e2e}" +kubectl() { command kubectl --context "$context" "$@"; } +scratch="$(mktemp -d)" +trap 'rm -rf "$scratch"' EXIT +# Retry transient upstream failures without relaxing the pinned checksum below. +curl --fail --silent --show-error --location \ + --connect-timeout 10 --max-time 30 --retry 5 --retry-max-time 120 \ + https://raw.githubusercontent.com/prometheus-operator/prometheus-operator/v0.94.1/bundle.yaml \ + -o "$scratch/bundle.yaml" +echo "7ab177610b271c70c57c7d6fc550a1e934eff4f707a5d8b61dfaa3845d52deea $scratch/bundle.yaml" | sha256sum --check +kubectl create namespace monitoring --dry-run=client -o yaml | kubectl apply -f - +# The official bundle's namespaced objects and binding subjects use default. +sed 's/namespace: default/namespace: monitoring/g' "$scratch/bundle.yaml" > "$scratch/monitoring.yaml" +kubectl apply --server-side -f "$scratch/monitoring.yaml" +kubectl wait --for=condition=Established --timeout=120s crd/prometheuses.monitoring.coreos.com crd/podmonitors.monitoring.coreos.com crd/servicemonitors.monitoring.coreos.com crd/prometheusrules.monitoring.coreos.com +kubectl rollout status -n monitoring deployment/prometheus-operator --timeout=180s +helm --kube-context "$context" upgrade gameplane charts/gameplane --namespace gameplane-system --reuse-values \ + -f docs/examples/monitoring/gameplane-values.yaml --wait --timeout 5m +kubectl apply -f docs/examples/monitoring/prometheus.yaml +kubectl wait -n monitoring --for=create statefulset/prometheus-gameplane --timeout=120s +kubectl rollout status -n monitoring statefulset/prometheus-gameplane --timeout=180s diff --git a/docs/examples/monitoring/gameplane-values.yaml b/docs/examples/monitoring/gameplane-values.yaml new file mode 100644 index 000000000..8b51fe3cb --- /dev/null +++ b/docs/examples/monitoring/gameplane-values.yaml @@ -0,0 +1,19 @@ +# Apply to the Gameplane Helm release. See docs/monitoring.md for node-specific +# strict NetworkPolicy settings and the companion Prometheus resource. +networkPolicies: + enabled: true +serviceMonitors: + enabled: true + labels: + monitoring: gameplane + scrapeNamespaceSelector: + matchLabels: + kubernetes.io/metadata.name: monitoring +prometheusRules: + enabled: true + labels: + monitoring: gameplane +grafanaDashboards: + enabled: true + labels: + grafana_dashboard: "1" diff --git a/docs/examples/monitoring/prometheus.yaml b/docs/examples/monitoring/prometheus.yaml new file mode 100644 index 000000000..11793ace4 --- /dev/null +++ b/docs/examples/monitoring/prometheus.yaml @@ -0,0 +1,71 @@ +# Requires the Prometheus Operator and its CRDs. The monitoring namespace must +# match serviceMonitors.scrapeNamespaceSelector in gameplane-values.yaml. +apiVersion: v1 +kind: Namespace +metadata: + name: monitoring +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: gameplane-prometheus + namespace: monitoring +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: gameplane-prometheus-discovery +rules: + - apiGroups: [""] + resources: [nodes, nodes/metrics, services, endpoints, pods] + verbs: [get, list, watch] + - apiGroups: [discovery.k8s.io] + resources: [endpointslices] + verbs: [get, list, watch] + - nonResourceURLs: [/metrics] + verbs: [get] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: gameplane-prometheus-discovery +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: gameplane-prometheus-discovery +subjects: + - kind: ServiceAccount + name: gameplane-prometheus + namespace: monitoring +--- +apiVersion: monitoring.coreos.com/v1 +kind: Prometheus +metadata: + name: gameplane + namespace: monitoring +spec: + version: v3.13.4 + serviceAccountName: gameplane-prometheus + replicas: 1 + retention: 24h + resources: + requests: { cpu: 100m, memory: 256Mi } + limits: { memory: 1Gi } + serviceMonitorSelector: + matchLabels: { monitoring: gameplane } + serviceMonitorNamespaceSelector: + matchExpressions: + - key: kubernetes.io/metadata.name + operator: In + values: [gameplane-system, gameplane-games] + podMonitorSelector: + matchLabels: { monitoring: gameplane } + podMonitorNamespaceSelector: + matchExpressions: + - key: kubernetes.io/metadata.name + operator: In + values: [gameplane-system, gameplane-games] + ruleSelector: + matchLabels: { monitoring: gameplane } + ruleNamespaceSelector: + matchLabels: { kubernetes.io/metadata.name: gameplane-system } diff --git a/docs/monitoring.md b/docs/monitoring.md new file mode 100644 index 000000000..83a22b300 --- /dev/null +++ b/docs/monitoring.md @@ -0,0 +1,150 @@ +# Monitoring games and relays + +Gameplane exports the existing agent CPU, process memory, disk and player gauges, +optional game performance measurements, and relay health. Prometheus and Grafana +are optional. A monitor resource alone does not prove discovery or network access. + +## Complete setup + +Install the Prometheus Operator and its CRDs first. Apply +[`examples/monitoring/gameplane-values.yaml`](examples/monitoring/gameplane-values.yaml) +to the Gameplane release and deploy +[`examples/monitoring/prometheus.yaml`](examples/monitoring/prometheus.yaml), or +copy that resource's selectors into your existing Prometheus configuration. + +The examples agree on the `monitoring: gameplane` monitor/rule label, discover +both `gameplane-system` and `gameplane-games`, and place Prometheus in `monitoring`. +Change all three files/settings together when renaming namespaces. With +kube-prometheus-stack, configure `prometheus.prometheusSpec.serviceMonitorSelector`, +`podMonitorSelector`, their namespace selectors and `ruleSelector` to match the +example. Do not leave the chart's default release-label selector pointing at a +different Helm release. The Grafana sidecar must watch ConfigMaps labeled +`grafana_dashboard: "1"` in the Gameplane release namespace. + +The chart enables rules and both Grafana dashboards in this example. Agent +metrics use plain HTTP port 9090; tunnel metrics share the health listener on +8091. Neither monitor uses the agent's mTLS control port or a client certificate. +Check Prometheus **Status → Targets** for agent and tunnel targets with `up=1`, +then query `gameplane_agent_players_online` and `gameplane_tunnel_ready`. +Unknown player counts are absent even when scraping works. + +With the operator and its NetworkPolicies enabled, the chart requires a nonempty +`serviceMonitors.scrapeNamespaceSelector`. Missing configuration fails with a +message naming the setting. If another system owns the policies, set +`serviceMonitors.externallyManagedPolicy: true`; that flag opens no traffic. +When upgrading an installation that already enabled monitors without a selector, +set the selector or the explicit external-policy option in the same Helm upgrade. +Supply your own ingress policies selecting `app.kubernetes.io/name: gameplane-game` +on TCP 9090 and `app.kubernetes.io/name: gameplane-tunnel` on TCP 8091, with a +namespace selector for the scraper. Restricted Prometheus egress policies must +also allow those destinations, DNS and API discovery. + +For enforced isolation, configure `networkPolicies.strict: true`, actual +`kubeletCIDRs` and `apiServerCIDRs`, and the game modules' exact `probePorts`. +The legacy broad kubelet allowance can otherwise admit other private-address +sources on metrics ports. Keep metrics ports out of the game probe allowance; +the dedicated scrape policies grant their access. The CI monitoring bucket +uses Cilium and actual node/API addresses, tests a scraper in a separate +namespace, revokes/restores the scrape policy, and checks an unapproved namespace +cannot reach agent metrics. + +## Declaring numeric game measurements + +Reuse a module's existing `capabilities.status.metrics[]` entry. This opt-in +example describes an output such as `TPS: 19.5`: + +```yaml +capabilities: + status: + metrics: + - id: tps + displayName: TPS + command: stats + regex: 'TPS: (?P[0-9.]+)' + unit: TPS # presentation only + performance: + signal: tick_rate + unit: per_second # machine-readable input unit + degradedThreshold: "18" +``` + +`signal` is one of `tick_rate`, `tick_duration`, or `frame_rate`. Rates require +`per_second`; durations accept `seconds` or `milliseconds`. Values and thresholds +are converted to hertz or seconds. The optional threshold is a finite nonnegative +decimal string, up to nine integer and six fractional digits, for portable CRD +handling. Rates below the threshold or durations above it trigger the degradation +rule after five minutes. No default threshold or game-specific tick rate is +invented. Modules must choose a read-only command and a meaningful threshold. +No external module is opted in by this change. + +The source polls immediately and then 30 seconds after each completed cycle. +There are at most 16 declared entries, 16 distinct commands per cycle, 1024 bytes +per command, 4096 bytes per regex and 64 KiB of accepted response text. Identical +commands are executed once and parsed for each reading. Each command waits at +most three seconds and the whole cycle at most ten seconds. The legacy RCON +interface cannot cancel every protocol operation: one outstanding call may +finish later, but it blocks further background calls until it returns, its late +result is discarded, and no queue or growing set of workers is created. +Underlying transports retain their own deadlines and response limits. Source +RCON bounds dialing to five seconds, authentication to ten seconds (including +all preamble packets), and a command exchange to 30 seconds after authentication, +including continuously arriving fragments, and accepts at most 8 MiB of +aggregate response text for large administrative commands. Its 400-ms fragment +idle window never extends the absolute exchange deadline. Exceeding either +bound discards the incomplete response and closes the connection. This transport +cap is separate from the status collector's smaller 64-KiB acceptance limit. + +Both protected `/status` routes and Prometheus share the same source. Templates +with valid performance declarations use the interval cache for all their status +readouts; templates without them retain the five-second on-demand cache. Scrapes +never execute RCON or wait on an RCON command. Samples expire 75 seconds after +the command completed. Failed queries, invalid/negative/nonfinite values and +stale samples have no numeric value series. A genuine zero is retained. + +The metric families are `gameplane_agent_performance_value`, +`gameplane_agent_performance_available`, +`gameplane_agent_performance_sample_timestamp_seconds`, and optional +`gameplane_agent_performance_degraded_threshold`. Labels are the bounded metric +ID, closed signal and base unit; threshold adds `direction=below|above`. +PodMonitor adds namespace, Pod, server and component identity. Display strings, +commands, responses and errors never become labels. Unsupported declarations +produce no performance series. Failed supported declarations emit availability +zero; value and timestamp remain absent. + +## Relays and infrastructure + +Tunnel gauges read the same snapshot as readiness: `gameplane_tunnel_ready`, +`gameplane_tunnel_connection_supported`, `gameplane_tunnel_connection_known`, `gameplane_tunnel_connected`, +`gameplane_tunnel_forwards_registered`, `gameplane_tunnel_endpoint_assigned`, +and `gameplane_tunnel_observation_timestamp_seconds`. `connected` and observation +timestamp are absent when their evidence is unknown or stale. Readiness and +registration flags mean *confirmed*, so zero does not invent a disconnected +provider. The rules distinguish an unavailable relay from unknown health. +Pinned Playit cannot confirm live connectivity: its support gauge is zero and +the informational unknown-health alert excludes it. Its known/connected metrics +still represent unknown evidence, without claiming a healthy relay. Supported +providers raise the info alert when their observations become unknown or stale; +an observed tsnet failure is unavailable and raises the warning-level outage +alert. Between Tailscale attempts and while a replacement joins, the supervisor +knows that no forwarding attempt is serving, so that unavailable state remains +known throughout backoff. It carries no provider observation timestamp. Fresh +observations resume when forwarding is established and retain their normal +staleness limit. See [relay-health.md](relay-health.md). +Tailscale, frp and Playit all use the same metrics path. Metrics contain no +credentials, login URLs, proxy names or provider error messages. + +Container CPU throttling and working-set pressure require kubelet/cAdvisor +scrapes. OOM/restart, Pending Pod and node pressure rules require kube-state-metrics. +Install/configure those collectors (kube-prometheus-stack normally provides +them); the minimal Prometheus example demonstrates Gameplane discovery and +does not deploy them or Grafana. A missing upstream collector leaves those +series absent; it does not mean zero pressure. Agent process RSS is not total +container memory. These rules filter the games namespace and use `max` to +deduplicate replicated sources. The existing fleet rules likewise collapse +duplicate operator replicas instead of adding their cache-derived counts. + +The Workloads dashboard shows scrape health, relay readiness/known state, +supported game measurements, container pressure, restarts and Pending Pods. +Missing measurements remain gaps. CI renders and evaluates the shipped alert +rules with promtool, including outage/recovery, supported zero versus unknown, +pressure, throttling, OOM, Pending/node pressure and duplicate operator readings. diff --git a/docs/tunnels.md b/docs/tunnels.md index 504d98d96..1ba4c9a57 100644 --- a/docs/tunnels.md +++ b/docs/tunnels.md @@ -1,5 +1,8 @@ # Tunnels +For Prometheus discovery, relay health alerts and the complete scrape-policy +configuration, see [Monitoring games and relays](monitoring.md). + A GameServer can optionally route players through an external relay — the server dials outbound to the relay and players connect to the relay's address. This exists for installs with no public IP and no ability to port-forward (homelab k3s behind diff --git a/hack/ci_scope.py b/hack/ci_scope.py index e887463dd..d6a001b99 100644 --- a/hack/ci_scope.py +++ b/hack/ci_scope.py @@ -29,8 +29,8 @@ FORCE_ALL_PREFIXES = (".github/actions/",) # Mirrors the e2e-go matrix buckets in .github/workflows/ci.yaml. E2E_BUCKETS = ["operator", "api-auth", "api-roles", "api-rbac", "api-agent", "api-mods", - "telemetry", "optional-components", "network-isolation", "management-placement"] -E2E_AMD64_ONLY = {"network-isolation", "management-placement"} + "telemetry", "optional-components", "management-placement", "network-isolation", "monitoring"] +E2E_AMD64_ONLY = {"management-placement", "network-isolation", "monitoring"} SUITES_FILE = "test/e2e/suite-components.json" # Paths no Go e2e test reads. Their own jobs keep their paths-filter gates. NO_E2E_FILES = { diff --git a/hack/test_ci_scope.py b/hack/test_ci_scope.py index 03984e022..6b4a23f11 100644 --- a/hack/test_ci_scope.py +++ b/hack/test_ci_scope.py @@ -419,6 +419,16 @@ def test_audit_network_isolation_has_a_run_pattern(self): self.assertIn("TestNetworkIsolation_EnforcedAndUpgrade", values["e2e-run"].get("network-isolation", "")) + def test_audit_monitoring_has_a_run_pattern(self): + """Metric producers and the owning test file select the real scrape fixture.""" + for path in ("operator/internal/controller/gameserver_controller.go", + "api/cmd/main.go", "agent/internal/status/performance.go", + "tunnel/relay_health.go", "test/e2e/monitoring_e2e_test.go"): + with self.subTest(path=path): + values = self.select(path) + self.assertIn("TestMonitoring_DiscoveryPerformanceAndPolicy", + values["e2e-run"].get("monitoring", "")) + def test_audit_management_placement_has_a_run_pattern(self): """Affected components and the owning test file select the audit bucket.""" for path in ("operator/cmd/main.go", diff --git a/operator/api/v1alpha1/gametemplate_types.go b/operator/api/v1alpha1/gametemplate_types.go index 38a906125..092b48acf 100644 --- a/operator/api/v1alpha1/gametemplate_types.go +++ b/operator/api/v1alpha1/gametemplate_types.go @@ -741,6 +741,28 @@ type StatusMetricSpec struct { // Unit is an optional suffix shown after the value (e.g. "ms", "TPS"). // +optional Unit string `json:"unit,omitempty"` + + // Performance opts this existing readout into cached numeric monitoring. + // Display-only strings are never inferred to be performance measurements. + // +optional + Performance *PerformanceMetricSpec `json:"performance,omitempty"` +} + +// PerformanceMetricSpec gives a status readout explicit numeric semantics. +// Rate signals use per_second; duration accepts seconds or milliseconds. +// +kubebuilder:validation:XValidation:rule="self.signal == 'tick_duration' ? self.unit in ['seconds', 'milliseconds'] : self.unit == 'per_second'",message="duration requires seconds/milliseconds; rates require per_second" +type PerformanceMetricSpec struct { + // +kubebuilder:validation:Enum=tick_rate;tick_duration;frame_rate + Signal string `json:"signal"` + // +kubebuilder:validation:Enum=per_second;seconds;milliseconds + Unit string `json:"unit"` + // DegradedThreshold is a finite nonnegative decimal string in the input + // unit. Rates below it, or durations above it, are degraded. Omit to + // disable alerts. A string keeps JSON/schema handling portable. + // +kubebuilder:validation:Pattern=`^(0|[1-9][0-9]{0,8})(\.[0-9]{1,6})?$` + // +kubebuilder:validation:MaxLength=16 + // +optional + DegradedThreshold *string `json:"degradedThreshold,omitempty"` } // PlayerActionsSpec maps moderation actions to console commands. Each diff --git a/operator/api/v1alpha1/zz_generated.deepcopy.go b/operator/api/v1alpha1/zz_generated.deepcopy.go index 05926fd58..20421d118 100644 --- a/operator/api/v1alpha1/zz_generated.deepcopy.go +++ b/operator/api/v1alpha1/zz_generated.deepcopy.go @@ -2215,6 +2215,26 @@ func (in *OCISourceSpec) DeepCopy() *OCISourceSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *PerformanceMetricSpec) DeepCopyInto(out *PerformanceMetricSpec) { + *out = *in + if in.DegradedThreshold != nil { + in, out := &in.DegradedThreshold, &out.DegradedThreshold + *out = new(string) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PerformanceMetricSpec. +func (in *PerformanceMetricSpec) DeepCopy() *PerformanceMetricSpec { + if in == nil { + return nil + } + out := new(PerformanceMetricSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *PlayerActionsSpec) DeepCopyInto(out *PlayerActionsSpec) { *out = *in @@ -2521,6 +2541,11 @@ func (in *ServerActionSpec) DeepCopy() *ServerActionSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *StatusMetricSpec) DeepCopyInto(out *StatusMetricSpec) { *out = *in + if in.Performance != nil { + in, out := &in.Performance, &out.Performance + *out = new(PerformanceMetricSpec) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StatusMetricSpec. @@ -2539,7 +2564,9 @@ func (in *StatusSpec) DeepCopyInto(out *StatusSpec) { if in.Metrics != nil { in, out := &in.Metrics, &out.Metrics *out = make([]StatusMetricSpec, len(*in)) - copy(*out, *in) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } } } diff --git a/operator/config/crd/gameplane.local_gametemplates.yaml b/operator/config/crd/gameplane.local_gametemplates.yaml index 9cefe0a11..1bab7b727 100644 --- a/operator/config/crd/gameplane.local_gametemplates.yaml +++ b/operator/config/crd/gameplane.local_gametemplates.yaml @@ -960,6 +960,41 @@ spec: maxLength: 63 pattern: ^[a-z0-9]([a-z0-9-]*[a-z0-9])?$ type: string + performance: + description: |- + Performance opts this existing readout into cached numeric monitoring. + Display-only strings are never inferred to be performance measurements. + properties: + degradedThreshold: + description: |- + DegradedThreshold is a finite nonnegative decimal string in the input + unit. Rates below it, or durations above it, are degraded. Omit to + disable alerts. A string keeps JSON/schema handling portable. + maxLength: 16 + pattern: ^(0|[1-9][0-9]{0,8})(\.[0-9]{1,6})?$ + type: string + signal: + enum: + - tick_rate + - tick_duration + - frame_rate + type: string + unit: + enum: + - per_second + - seconds + - milliseconds + type: string + required: + - signal + - unit + type: object + x-kubernetes-validations: + - message: duration requires seconds/milliseconds; rates + require per_second + rule: 'self.signal == ''tick_duration'' ? self.unit + in [''seconds'', ''milliseconds''] : self.unit == + ''per_second''' regex: description: |- Regex extracts the value via the named group "value", e.g. diff --git a/operator/internal/controller/backup_forget_envtest_test.go b/operator/internal/controller/backup_forget_envtest_test.go index 9dbf6bf8e..a358fda9b 100644 --- a/operator/internal/controller/backup_forget_envtest_test.go +++ b/operator/internal/controller/backup_forget_envtest_test.go @@ -38,15 +38,14 @@ func withForgetBackupReconciler(rec *events.FakeRecorder) setupReconciler { } } -// startForgetFixture boots a manager, seeds a GameServer and repo Secret, creates -// a restic Backup named name, waits for the reconciler to put the snapshot -// finalizer on it, then marks it Succeeded with snapshotID, the state a finished -// backup is in when someone deletes it. +// startForgetFixture seeds a finished restic Backup before starting its manager, +// then waits for the reconciler to add the snapshot finalizer. Starting with a +// Pending Backup races the copy Job's creation against the terminal status write; +// envtest has no garbage collector to finish deleting that unrelated copy Job. func startForgetFixture(t *testing.T, name, snapshotID string) (string, *events.FakeRecorder) { t.Helper() ns := newNamespace(t) rec := events.NewFakeRecorder(32) - startMgr(t, ns, withForgetBackupReconciler(rec)) seedGameServer(t, ns, "smp") if err := k8sClient.Create(context.Background(), buildResticRepoSecret(ns, "repo")); err != nil { @@ -55,6 +54,15 @@ func startForgetFixture(t *testing.T, name, snapshotID string) (string, *events. if err := k8sClient.Create(context.Background(), buildBackup(ns, name, "smp", "repo")); err != nil { t.Fatalf("create backup: %v", err) } + b := getBackup(t, ns, name) + now := metav1.Now() + b.Status.Phase = gameplanev1alpha1.BackupPhaseSucceeded + b.Status.SnapshotID = snapshotID + b.Status.CompletionTime = &now + if err := k8sClient.Status().Update(context.Background(), b); err != nil { + t.Fatalf("mark backup succeeded: %v", err) + } + startMgr(t, ns, withForgetBackupReconciler(rec)) eventually(t, func() (bool, string) { b := getBackup(t, ns, name) for _, f := range b.Finalizers { @@ -64,21 +72,8 @@ func startForgetFixture(t *testing.T, name, snapshotID string) (string, *events. } return false, "snapshot finalizer not yet added: " + strings.Join(b.Finalizers, ",") }) - - // The reconciler also writes status while the backup Job is Pending, so the - // update can conflict; retry on a fresh read. - if err := retry.RetryOnConflict(retry.DefaultRetry, func() error { - var b gameplanev1alpha1.Backup - if err := k8sClient.Get(context.Background(), types.NamespacedName{Namespace: ns, Name: name}, &b); err != nil { - return err - } - now := metav1.Now() - b.Status.Phase = gameplanev1alpha1.BackupPhaseSucceeded - b.Status.SnapshotID = snapshotID - b.Status.CompletionTime = &now - return k8sClient.Status().Update(context.Background(), &b) - }); err != nil { - t.Fatalf("mark backup succeeded: %v", err) + if _, exists := getJob(t, ns, name); exists { + t.Fatal("finished backup unexpectedly created a copy Job") } return ns, rec } diff --git a/operator/internal/controller/gameserver_relay_health_test.go b/operator/internal/controller/gameserver_relay_health_test.go index 9b48f40ae..9e9628789 100644 --- a/operator/internal/controller/gameserver_relay_health_test.go +++ b/operator/internal/controller/gameserver_relay_health_test.go @@ -37,6 +37,15 @@ func TestRelayDeploymentIdentityProbesAndResourceRemoval(t *testing.T) { t.Fatal("relay permits overlapping identities") } container := dep.Spec.Template.Spec.Containers[0] + metricsPorts := 0 + for _, port := range container.Ports { + if port.Name == "relay-metrics" && port.ContainerPort == 8091 { + metricsPorts++ + } + } + if metricsPorts != 1 { + t.Fatal("tunnel metrics port missing or duplicated") + } if container.StartupProbe == nil || container.StartupProbe.HTTPGet == nil || container.StartupProbe.HTTPGet.Path != "/healthz" || !hasRelayReadinessProbe(&dep) || container.LivenessProbe != nil { t.Fatal("provider outage must affect readiness without restarting the supervisor") } diff --git a/operator/internal/controller/gameserver_tunnel.go b/operator/internal/controller/gameserver_tunnel.go index b2a19ef2d..7764240c8 100644 --- a/operator/internal/controller/gameserver_tunnel.go +++ b/operator/internal/controller/gameserver_tunnel.go @@ -258,6 +258,12 @@ func (r *GameServerReconciler) reconcileTunnel( ContainerPort: p.ContainerPort, Protocol: p.Protocol, } + // The generated health port owns this discovery name. Forwarded + // port metadata is optional; the provider still uses the unchanged + // template/API name in its actual mapping. + if cp.Name == "relay-metrics" { + cp.Name = "" + } ports = append(ports, cp) } @@ -431,6 +437,7 @@ func (r *GameServerReconciler) reconcileTunnel( readiness = &corev1.Probe{ProbeHandler: corev1.ProbeHandler{HTTPGet: &corev1.HTTPGetAction{Path: "/readyz", Port: intstr.FromInt(8091)}}, PeriodSeconds: 5, TimeoutSeconds: 2, FailureThreshold: 1} } dep.Spec.Template.Spec.Volumes = volumes + ports = append(ports, corev1.ContainerPort{Name: "relay-metrics", ContainerPort: 8091, Protocol: corev1.ProtocolTCP}) dep.Spec.Template.Spec.Containers = []corev1.Container{{ Name: "tunnel", Image: image, diff --git a/operator/internal/controller/gametemplate_performance_envtest_test.go b/operator/internal/controller/gametemplate_performance_envtest_test.go new file mode 100644 index 000000000..ee0a0c96a --- /dev/null +++ b/operator/internal/controller/gametemplate_performance_envtest_test.go @@ -0,0 +1,46 @@ +//go:build envtest + +package controller + +import ( + "testing" + + gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" + apierrors "k8s.io/apimachinery/pkg/api/errors" +) + +func TestTemplatePerformanceSchema(t *testing.T) { + for _, tc := range []struct { + signal, unit, threshold string + valid bool + }{ + {"tick_rate", "per_second", "18", true}, + {"tick_duration", "milliseconds", "50.5", true}, + {"frame_rate", "per_second", "0", true}, + {"seed", "per_second", "1", false}, + {"tick_rate", "seconds", "1", false}, + {"tick_duration", "per_second", "1", false}, + {"tick_rate", "per_second", "NaN", false}, + {"tick_rate", "per_second", "-1", false}, + {"tick_rate", "per_second", "1e999", false}, + } { + t.Run(tc.signal+"-"+tc.unit+"-"+tc.threshold, func(t *testing.T) { + tmpl := buildGameTemplate(uniqueName("performance")) + tmpl.Spec.RCON = &gameplanev1alpha1.RCONSpec{Protocol: "source", Port: 25575} + if tmpl.Spec.Capabilities == nil { + tmpl.Spec.Capabilities = &gameplanev1alpha1.CapabilitiesSpec{} + } + threshold := tc.threshold + tmpl.Spec.Capabilities.Status = &gameplanev1alpha1.StatusSpec{Metrics: []gameplanev1alpha1.StatusMetricSpec{{ID: "performance", DisplayName: "Performance", Command: "stats", Regex: `(?P\S+)`, Performance: &gameplanev1alpha1.PerformanceMetricSpec{Signal: tc.signal, Unit: tc.unit, DegradedThreshold: &threshold}}}} + err := k8sClient.Create(t.Context(), tmpl) + if tc.valid { + if err != nil { + t.Fatal(err) + } + deleteCleanup(t, tmpl) + } else if !apierrors.IsInvalid(err) { + t.Fatalf("wanted schema refusal, got %v", err) + } + }) + } +} diff --git a/operator/specs.md b/operator/specs.md index 6cba0f995..29c829144 100644 --- a/operator/specs.md +++ b/operator/specs.md @@ -20,6 +20,8 @@ finalizer until those references are gone. These explicit waits use Existing restore fence-write requeues and writer-drain polling remain unchanged, and every resumed restore pass still verifies the workload fence and live writers. +- Preserve explicit optional numeric performance declarations in template capabilities passed to the agent; CRD validation restricts signal/unit pairs and finite decimal thresholds. Tunnel containers name their existing 8091 listener `relay-metrics` for PodMonitor discovery; health, resources and provider identity policies remain unchanged. + - **Relay health:** frp identities use GameServer UID without changing advertised port names. frp/Playit Deployments use one replica, Recreate, a wrapper startup probe and a provider readiness probe on TCP 8091. `TunnelReady` requires the current observed generation and updated/ready replicas with the new probe; legacy unprobed Pods cannot report success. Playit v1.0.10's cached IPC assignment cannot confirm a relay connection, so it reports unknown/not ready rather than optimistic success. Preserve `TunnelResources` defaults and override removal. See `docs/relay-health.md` for health bounds, migration and the provider limitation. - **GameServer lifecycle:** reconcile GameServer CRs into StatefulSets, Services, PVCs, and pod-injected agent sidecars; manage pod affinity, RCON/console integration, file I/O, and graceful stop sequences. diff --git a/test/e2e/buckets.sh b/test/e2e/buckets.sh index 88fc7225f..8f6af22dd 100755 --- a/test/e2e/buckets.sh +++ b/test/e2e/buckets.sh @@ -421,12 +421,17 @@ EOF unbucketed() { :; } +bucket_monitoring() { cat <<'EOF' +TestMonitoring_DiscoveryPerformanceAndPolicy +EOF +} + bucket_network_isolation() { printf '%s\n' TestNetworkIsolation_EnforcedAndUpgrade TestNetworkIsolation_ProbeOutput } bucket_names() { - printf '%s\n' operator api-auth api-roles api-rbac api-agent api-mods ratelimit bot-fast bot-heavy multicluster upgrade postgres-availability telemetry capture-overhead optional-components network-isolation management-placement + printf '%s\n' operator api-auth api-roles api-rbac api-agent api-mods ratelimit bot-fast bot-heavy multicluster upgrade postgres-availability telemetry capture-overhead optional-components management-placement network-isolation monitoring } list_bucket() { @@ -446,8 +451,9 @@ list_bucket() { telemetry) bucket_telemetry ;; capture-overhead) bucket_capture_overhead ;; optional-components) bucket_optional_components ;; - network-isolation) bucket_network_isolation ;; management-placement) bucket_management_placement ;; + network-isolation) bucket_network_isolation ;; + monitoring) bucket_monitoring ;; *) echo "unknown bucket: $1 (known: $(bucket_names | tr '\n' ' '))" >&2 exit 2 diff --git a/test/e2e/ci_selection_e2e_test.go b/test/e2e/ci_selection_e2e_test.go index 8cccdf589..061bd82dc 100644 --- a/test/e2e/ci_selection_e2e_test.go +++ b/test/e2e/ci_selection_e2e_test.go @@ -22,7 +22,7 @@ import ( // ciSelectionBuckets mirrors E2E_BUCKETS in hack/ci_scope.py (the e2e-go matrix). var ciSelectionBuckets = []string{ "operator", "api-auth", "api-roles", "api-rbac", "api-agent", "api-mods", - "telemetry", "optional-components", "network-isolation", "management-placement", + "telemetry", "optional-components", "management-placement", "network-isolation", "monitoring", } // ciSelection is the decoded output of one `ci_scope.py e2e` run. diff --git a/test/e2e/monitoring_e2e_test.go b/test/e2e/monitoring_e2e_test.go new file mode 100644 index 000000000..00bdb02ae --- /dev/null +++ b/test/e2e/monitoring_e2e_test.go @@ -0,0 +1,221 @@ +//go:build e2e + +package e2e + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "strconv" + "strings" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/types" +) + +// Dedicated serial bucket: real Prometheus Operator discovery and Cilium policy +// enforcement, with deterministic game readouts instead of an external module. +func TestMonitoring_DiscoveryPerformanceAndPolicy(t *testing.T) { + ctx := t.Context() + if _, err := envInstance.K8s.AppsV1().DaemonSets("kube-system").Get(ctx, "cilium", metav1.GetOptions{}); err != nil { + t.Fatalf("monitoring requires enforcing Cilium: %v", err) + } + port, stop := envInstance.PortForward(t, "monitoring", "svc/prometheus-operated", 9090) + defer stop() + client := &http.Client{Timeout: 10 * time.Second} + query := func(expression string) (string, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, fmt.Sprintf("http://127.0.0.1:%d/api/v1/query?query=%s", port, url.QueryEscape(expression)), nil) + if err != nil { + return "", err + } + response, err := client.Do(req) + if err != nil { + return "", err + } + defer response.Body.Close() + var result struct { + Status string `json:"status"` + Data struct { + Result []struct { + Value []json.RawMessage `json:"value"` + } `json:"result"` + } `json:"data"` + } + if err := json.NewDecoder(io.LimitReader(response.Body, 1<<20)).Decode(&result); err != nil { + return "", err + } + if response.StatusCode != 200 || result.Status != "success" || len(result.Data.Result) != 1 || len(result.Data.Result[0].Value) != 2 { + return "", fmt.Errorf("query %q returned status %s and %d results", expression, result.Status, len(result.Data.Result)) + } + var value string + err = json.Unmarshal(result.Data.Result[0].Value[1], &value) + return value, err + } + waitQuery := func(expression, want string) { + envInstance.Eventually(t, 3*time.Minute, func() (bool, string) { + got, err := query(expression) + return err == nil && got == want, fmt.Sprintf("%s: got %s want %s (%v)", expression, got, want, err) + }) + } + envInstance.ApplyYAML(t, "frps.yaml") + stamp := time.Now().UnixNano() + name := fmt.Sprintf("monitoring-%d", stamp) + const gameCode = `from http.server import BaseHTTPRequestHandler,ThreadingHTTPServer +import json,pathlib,threading +value=pathlib.Path('/tmp/performance') +count=pathlib.Path('/tmp/performance-queries') +value.write_text('20') +count.write_text('0') +class Handler(BaseHTTPRequestHandler): + def log_message(self,*args): pass + def do_GET(self): + self.send_response(200); self.end_headers(); self.wfile.write(b'ok') + def do_POST(self): + command=json.loads(self.rfile.read(int(self.headers.get('Content-Length','0')))).get('command','') + if command=='stats': + count.write_text(str(int(count.read_text())+1)) + output='TPS: '+value.read_text()+' Tick: 50' + else: output='0 players online' + self.send_response(200); self.end_headers(); self.wfile.write(json.dumps({'output':output}).encode()) +threading.Thread(target=ThreadingHTTPServer(('127.0.0.1',25575),Handler).serve_forever,daemon=True).start() +ThreadingHTTPServer(('0.0.0.0',25566),Handler).serve_forever() +` + tmpl := &unstructured.Unstructured{Object: map[string]any{"apiVersion": "gameplane.local/v1alpha1", "kind": "GameTemplate", "metadata": map[string]any{"name": name}, "spec": map[string]any{ + "displayName": "Monitoring fixture", "game": "noop", "version": "1", "image": "python:3.14-alpine", "consoleMode": "rcon", "command": []any{"python", "-u", "-c"}, "args": []any{gameCode}, "sentinelIsolation": true, + "ports": []any{map[string]any{"name": "game", "containerPort": int64(25566), "protocol": "TCP", "advertise": true}}, "rcon": map[string]any{"protocol": "rest", "port": int64(25575)}, + "capabilities": map[string]any{"status": map[string]any{"metrics": []any{ + map[string]any{"id": "tps", "displayName": "TPS", "command": "stats", "regex": `TPS: (?P\S+)`, "performance": map[string]any{"signal": "tick_rate", "unit": "per_second", "degradedThreshold": "18"}}, + map[string]any{"id": "tick", "displayName": "Tick", "command": "stats", "regex": `Tick: (?P\S+)`, "performance": map[string]any{"signal": "tick_duration", "unit": "milliseconds"}}, + map[string]any{"id": "display", "displayName": "Display only", "command": "stats", "regex": `TPS: (?P\S+)`}, + }}}, + }}} + if _, err := envInstance.Dyn.Resource(gameTemplateGVR).Create(ctx, tmpl, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + _ = envInstance.Dyn.Resource(gameTemplateGVR).Delete(context.Background(), name, metav1.DeleteOptions{}) + }) + gs := &unstructured.Unstructured{Object: map[string]any{"apiVersion": "gameplane.local/v1alpha1", "kind": "GameServer", "metadata": map[string]any{"name": name, "namespace": ocGamesNS}, "spec": map[string]any{"templateRef": map[string]any{"name": name}, "networking": map[string]any{"expose": "ClusterIP", "tunnel": map[string]any{"enabled": false, "provider": "frp", "frp": map[string]any{"serverAddr": ocFrpsAddr, "remotePorts": []any{map[string]any{"name": "game", "remotePort": int64(30020)}}}}}}}} + gs, err := envInstance.Dyn.Resource(gameServerGVR).Namespace(ocGamesNS).Create(ctx, gs, metav1.CreateOptions{}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + _ = envInstance.Dyn.Resource(gameServerGVR).Namespace(ocGamesNS).Delete(context.Background(), name, metav1.DeleteOptions{}) + }) + secret := &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: name + "-auth", Namespace: ocGamesNS, OwnerReferences: []metav1.OwnerReference{{APIVersion: "gameplane.local/v1alpha1", Kind: "GameServer", Name: name, UID: gs.GetUID()}}}, StringData: map[string]string{"token": ocFrpsToken}} + if _, err := envInstance.K8s.CoreV1().Secrets(ocGamesNS).Create(ctx, secret, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + patch, _ := json.Marshal(map[string]any{"spec": map[string]any{"networking": map[string]any{"tunnel": map[string]any{"enabled": true, "credentialsSecretRef": map[string]any{"name": secret.Name}}}}}) + if _, err := envInstance.Dyn.Resource(gameServerGVR).Namespace(ocGamesNS).Patch(ctx, name, types.MergePatchType, patch, metav1.PatchOptions{}); err != nil { + t.Fatal(err) + } + selector := fmt.Sprintf(`namespace="gameplane-games",server="%s"`, name) + waitQuery(`max(up{gameplane_component="agent",`+selector+`})`, "1") + waitQuery(`max(up{gameplane_component="tunnel",`+selector+`})`, "1") + waitQuery(`max(gameplane_tunnel_ready{`+selector+`})`, "1") + waitQuery(`max(gameplane_agent_performance_value{metric="tps",`+selector+`})`, "20") + waitQuery(`max(gameplane_agent_performance_value{metric="tick",`+selector+`})`, "0.05") + waitQuery(`count(gameplane_agent_performance_value{metric="display",`+selector+`}) or vector(0)`, "0") + readCount := func() int { + out, err := envInstance.Kubectl(ctx, "exec", "-n", ocGamesNS, name+"-0", "-c", "game", "--", "python", "-c", `from pathlib import Path; print(Path('/tmp/performance-queries').read_text())`) + if err != nil { + t.Fatal(err) + } + n, err := strconv.Atoi(strings.TrimSpace(out)) + if err != nil { + t.Fatal(err) + } + return n + } + pod, err := envInstance.K8s.CoreV1().Pods(ocGamesNS).Get(ctx, name+"-0", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + approvedProbe := name + "-approved" + monitoringProbe(t, "monitoring", approvedProbe) + before := readCount() + scrapeStart := time.Now() + const scrapeMany = `import urllib.request,sys +for _ in range(20): + with urllib.request.urlopen(sys.argv[1],timeout=2) as response: + assert b'gameplane_agent_performance_value' in response.read() +` + if out, err := envInstance.Kubectl(ctx, "exec", "-n", "monitoring", approvedProbe, "--", "python", "-c", scrapeMany, "http://"+pod.Status.PodIP+":9090/metrics"); err != nil { + t.Fatalf("repeated direct scrape failed: %v %s", err, out) + } + if after := readCount(); after-before > int(time.Since(scrapeStart)/(30*time.Second))+1 { + t.Fatalf("scrapes triggered RCON: %d -> %d", before, after) + } + for _, value := range []string{"0", "NaN", "20"} { + if out, err := envInstance.Kubectl(ctx, "exec", "-n", ocGamesNS, name+"-0", "-c", "game", "--", "python", "-c", `from pathlib import Path; import sys; Path('/tmp/performance').write_text(sys.argv[1])`, value); err != nil { + t.Fatalf("set measurement: %v %s", err, out) + } + if value == "NaN" { + waitQuery(`max(gameplane_agent_performance_available{metric="tps",`+selector+`})`, "0") + waitQuery(`count(gameplane_agent_performance_value{metric="tps",`+selector+`}) or vector(0)`, "0") + } else { + waitQuery(`max(gameplane_agent_performance_value{metric="tps",`+selector+`})`, value) + } + } + // A successful Prometheus query is not proof of policy enforcement. Revoke + // exactly the chart's scrape exception, observe scrape loss, then restore. + policy, err := envInstance.K8s.NetworkingV1().NetworkPolicies(ocGamesNS).Get(ctx, "allow-prometheus-to-agent", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if err := envInstance.K8s.NetworkingV1().NetworkPolicies(ocGamesNS).Delete(ctx, policy.Name, metav1.DeleteOptions{}); err != nil { + t.Fatal(err) + } + policy.ResourceVersion = "" + policy.UID = "" + t.Cleanup(func() { + _, _ = envInstance.K8s.NetworkingV1().NetworkPolicies(ocGamesNS).Create(context.Background(), policy, metav1.CreateOptions{}) + }) + waitQuery(`max(up{gameplane_component="agent",`+selector+`})`, "0") + if _, err := envInstance.K8s.NetworkingV1().NetworkPolicies(ocGamesNS).Create(ctx, policy, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + waitQuery(`max(up{gameplane_component="agent",`+selector+`})`, "1") + // The system namespace is deliberately outside the monitoring selector. + probeName := name + "-probe" + monitoringProbe(t, ocNS, probeName) + const denied = `import urllib.request,sys +try: + urllib.request.urlopen(sys.argv[1],timeout=2).read() + print('ALLOWED') +except (OSError,TimeoutError): + print('DENIED') +` + out, err := envInstance.Kubectl(ctx, "exec", "-n", ocNS, probeName, "--", "python", "-c", denied, "http://"+pod.Status.PodIP+":9090/metrics") + if err != nil || strings.TrimSpace(out) != "DENIED" { + t.Fatalf("unapproved namespace reached metrics, or probe failed: %v %s", err, out) + } +} + +func monitoringProbe(t *testing.T, namespace, name string) { + t.Helper() + probe := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: namespace}, Spec: corev1.PodSpec{RestartPolicy: corev1.RestartPolicyNever, Containers: []corev1.Container{{Name: "probe", Image: "python:3.14-alpine", Command: []string{"python", "-c", "import time; time.sleep(900)"}}}}} + nonRoot, no, uid := true, false, int64(65532) + probe.Spec.AutomountServiceAccountToken = &no + probe.Spec.SecurityContext = &corev1.PodSecurityContext{RunAsNonRoot: &nonRoot, RunAsUser: &uid, SeccompProfile: &corev1.SeccompProfile{Type: corev1.SeccompProfileTypeRuntimeDefault}} + probe.Spec.Containers[0].SecurityContext = &corev1.SecurityContext{AllowPrivilegeEscalation: &no, Capabilities: &corev1.Capabilities{Drop: []corev1.Capability{"ALL"}}} + if _, err := envInstance.K8s.CoreV1().Pods(namespace).Create(t.Context(), probe, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + _ = envInstance.K8s.CoreV1().Pods(namespace).Delete(context.Background(), name, metav1.DeleteOptions{}) + }) + envInstance.Eventually(t, time.Minute, func() (bool, string) { + ready, err := envInstance.PodIsReady(t.Context(), namespace, name) + return ready, fmt.Sprint(err) + }) +} diff --git a/test/e2e/specs.md b/test/e2e/specs.md index be4272594..9aabb6354 100644 --- a/test/e2e/specs.md +++ b/test/e2e/specs.md @@ -20,6 +20,8 @@ It was one of three `go.work` modules without CI lint coverage until this featur ## Responsibilities - Keep the lint workflow, devcontainer and `lint-gate-verify.sh` on the exact `v2.14.0` linter pin, whose importer supports Go 1.27.3 export data. The verifier's valid fixtures use the same pin; regression fixtures reject `latest` and a stale fixed version in any lint step. +- The serial `monitoring` bucket installs a checksum-pinned Prometheus Operator and Prometheus in a distinct namespace on the Cilium profile. It discovers real agent/tunnel targets through the documented chart values, verifies cached numeric samples, valid zero, invalid-value omission, no scrape-driven commands, policy revocation/recovery and refusal from an unapproved namespace. It performs no API login and does not opt in external modules. + - The optional-components lifecycle's `frp-identity-health` subtest runs two same-template TCP/UDP echo GameServers through one real frps. It asserts backend identity, registration rejection, replacement isolation, invalid credentials, relay outage/recovery and secret-free logs. It shares the lifecycle's controlled fixture and serial bucket; no extra top-level test or login budget is introduced. - Boot and tear down (or reuse) a kind cluster, install the working-tree Helm chart, and wait for every component to come up healthy. diff --git a/test/e2e/suite-components.json b/test/e2e/suite-components.json index de4a034e7..afded8a7a 100644 --- a/test/e2e/suite-components.json +++ b/test/e2e/suite-components.json @@ -17,6 +17,7 @@ ], "suites": [ { "tests": "^TestNetworkIsolation_", "components": ["operator", "api", "agent"] }, + { "tests": "^TestMonitoring_DiscoveryPerformanceAndPolicy$", "components": ["operator", "api", "agent", "tunnel"] }, { "tests": "^TestManagementPlacementAndDisruption$", "components": ["operator", "api", "agent"] }, { "tests": "^TestHelmInstall_", "components": ["*"] }, { "tests": "^TestCRD_Validation_", "components": ["operator"] }, diff --git a/tunnel/main.go b/tunnel/main.go index a9cb48140..1c8844122 100644 --- a/tunnel/main.go +++ b/tunnel/main.go @@ -231,7 +231,7 @@ func run(ctx context.Context, cfg Config) error { }() } - h := &relayHealth{} + h := &relayHealth{connectionUnsupported: cfg.TunnelType == "playit"} if ctx.Err() != nil { return nil } diff --git a/tunnel/relay_health.go b/tunnel/relay_health.go index b664a0f94..15a0b7984 100644 --- a/tunnel/relay_health.go +++ b/tunnel/relay_health.go @@ -34,11 +34,15 @@ func (o relayObservation) ready() bool { type relaySnapshot struct { relayObservation - Ready bool `json:"ready"` + Ready bool `json:"ready"` + checked time.Time } type relayHealth struct { + // Immutable provider capability, independent of fresh observation state. + connectionUnsupported bool mu sync.Mutex + supervisorUnavailable bool observation relayObservation checked, healthySince time.Time } @@ -46,10 +50,21 @@ type relayHealth struct { func (h *relayHealth) clear() { h.mu.Lock() defer h.mu.Unlock() + h.supervisorUnavailable = false h.observation = relayObservation{Connection: "unknown"} h.checked, h.healthySince = time.Time{}, time.Time{} } +// markUnavailable records that no forwarding attempt is serving. Unlike a +// provider observation, this local fact remains true throughout retry/backoff. +func (h *relayHealth) markUnavailable() { + h.mu.Lock() + defer h.mu.Unlock() + h.supervisorUnavailable = true + h.observation = relayObservation{Connection: "unavailable"} + h.checked, h.healthySince = time.Time{}, time.Time{} +} + func (h *relayHealth) observe(now time.Time, observation relayObservation, backoff *exponentialBackoff) { h.mu.Lock() defer h.mu.Unlock() @@ -57,6 +72,7 @@ func (h *relayHealth) observe(now time.Time, observation relayObservation, backo } func (h *relayHealth) observeLocked(now time.Time, observation relayObservation, backoff *exponentialBackoff) { + h.supervisorUnavailable = false if !observation.ready() { h.healthySince = time.Time{} } else { @@ -73,14 +89,18 @@ func (h *relayHealth) observeLocked(now time.Time, observation relayObservation, func (h *relayHealth) snapshot(now time.Time) relaySnapshot { h.mu.Lock() defer h.mu.Unlock() + if h.supervisorUnavailable { + return relaySnapshot{relayObservation: relayObservation{Connection: "unavailable"}} + } if h.checked.IsZero() || now.Sub(h.checked) >= healthStaleAfter { return relaySnapshot{relayObservation: relayObservation{Connection: "unknown"}} } - return relaySnapshot{relayObservation: h.observation, Ready: h.observation.ready()} + return relaySnapshot{relayObservation: h.observation, Ready: h.observation.ready(), checked: h.checked} } func relayHealthHandler(h *relayHealth) http.Handler { mux := http.NewServeMux() + mux.Handle("/metrics", relayMetrics(h)) mux.HandleFunc("/healthz", func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) }) mux.HandleFunc("/readyz", func(w http.ResponseWriter, _ *http.Request) { if !h.snapshot(time.Now()).Ready { diff --git a/tunnel/relay_metrics.go b/tunnel/relay_metrics.go new file mode 100644 index 000000000..aee7bf024 --- /dev/null +++ b/tunnel/relay_metrics.go @@ -0,0 +1,45 @@ +package main + +import ( + "fmt" + "net/http" + "time" +) + +// Fixed metric names and a closed state enum make the small exposition bounded. +// Scraping never polls a provider. Unknown connection is not encoded as down. +func relayMetrics(h *relayHealth) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) { + w.Header().Set("Content-Type", "text/plain; version=0.0.4; charset=utf-8") + now := time.Now() + snapshot := h.snapshot(now) + checked := snapshot.checked + known := snapshot.Connection == "connected" || snapshot.Connection == "unavailable" + gauges := []struct { + name, help string + value bool + }{ + {"gameplane_tunnel_connection_supported", "Whether the provider can report live connection health.", !h.connectionUnsupported}, + {"gameplane_tunnel_ready", "Whether the relay and all configured forwards are confirmed ready.", snapshot.Ready}, + {"gameplane_tunnel_connection_known", "Whether current provider or supervisor evidence confirms the connection state.", known}, + {"gameplane_tunnel_forwards_registered", "Whether every configured forward is confirmed registered.", snapshot.Forwards}, + {"gameplane_tunnel_endpoint_assigned", "Whether the configured endpoint is assigned.", snapshot.Assigned}, + } + if known { + gauges = append(gauges, struct { + name, help string + value bool + }{"gameplane_tunnel_connected", "Whether the relay is connected; absent when unknown.", snapshot.Connection == "connected"}) + } + for _, g := range gauges { + n := 0 + if g.value { + n = 1 + } + _, _ = fmt.Fprintf(w, "# HELP %s %s\n# TYPE %s gauge\n%s %d\n", g.name, g.help, g.name, g.name, n) + } + if !checked.IsZero() && now.Sub(checked) < healthStaleAfter { + _, _ = fmt.Fprintf(w, "# HELP gameplane_tunnel_observation_timestamp_seconds Unix time of the latest fresh provider observation.\n# TYPE gameplane_tunnel_observation_timestamp_seconds gauge\ngameplane_tunnel_observation_timestamp_seconds %.3f\n", float64(checked.UnixNano())/1e9) + } + }) +} diff --git a/tunnel/relay_metrics_test.go b/tunnel/relay_metrics_test.go new file mode 100644 index 000000000..1a0bcf306 --- /dev/null +++ b/tunnel/relay_metrics_test.go @@ -0,0 +1,53 @@ +package main + +import ( + "net/http/httptest" + "strings" + "testing" + "time" +) + +func TestRelayMetricsUnknownStaleAndRecovery(t *testing.T) { + h := &relayHealth{} + for _, tc := range []struct { + state relayObservation + age time.Duration + connected, ready string + }{ + {relayObservation{Connection: "unknown", Assigned: true}, 0, "", "0"}, + {relayObservation{Connection: "connected", Assigned: true, Forwards: true}, 0, "1", "1"}, + {relayObservation{Connection: "connected", Assigned: true, Forwards: true}, healthStaleAfter + time.Second, "", "0"}, + {relayObservation{Connection: "unavailable"}, 0, "0", "0"}, + } { + h.observe(time.Now().Add(-tc.age), tc.state, nil) + w := httptest.NewRecorder() + relayHealthHandler(h).ServeHTTP(w, httptest.NewRequestWithContext(t.Context(), "GET", "/metrics", nil)) + body := w.Body.String() + if !strings.Contains(body, "gameplane_tunnel_ready "+tc.ready+"\n") { + t.Fatal(body) + } + if tc.connected == "" { + if strings.Contains(body, "gameplane_tunnel_connected ") { + t.Fatal("unknown fabricated as connected/down") + } + } else if !strings.Contains(body, "gameplane_tunnel_connected "+tc.connected+"\n") { + t.Fatal(body) + } + } +} + +func TestRelayMetricsDistinguishUnsupportedHealth(t *testing.T) { + for _, unsupported := range []bool{false, true} { + h := &relayHealth{connectionUnsupported: unsupported} + h.observe(time.Now(), relayObservation{Connection: "unknown", Assigned: true}, nil) + w := httptest.NewRecorder() + relayHealthHandler(h).ServeHTTP(w, httptest.NewRequestWithContext(t.Context(), "GET", "/metrics", nil)) + want := "gameplane_tunnel_connection_supported 1\n" + if unsupported { + want = "gameplane_tunnel_connection_supported 0\n" + } + if !strings.Contains(w.Body.String(), want) || !strings.Contains(w.Body.String(), "gameplane_tunnel_connection_known 0\n") { + t.Fatal(w.Body.String()) + } + } +} diff --git a/tunnel/specs.md b/tunnel/specs.md index c27a505eb..687d5b285 100644 --- a/tunnel/specs.md +++ b/tunnel/specs.md @@ -10,6 +10,8 @@ Relay client supervisor that configures and supervises frp/playit child processe ## Responsibilities +- Expose cached provider evidence as bounded Prometheus gauges on `/metrics` at the existing health port 8091, including known/unknown connection, confirmed forwards/assignment and observation timestamp. Scraping never queries the provider; unknown or stale connection evidence has no `connected` value. Tailscale uses the same snapshot for readiness and metrics. See `docs/monitoring.md`. + 1. Parse environment variables to determine the tunnel provider (frp, tailscale, or playit), backing service address, and provider-specific config. 2. Read provider-specific credentials from the mounted Secret at `/etc/gameplane/tunnel-auth` (read-only). 3. Render frp TOML or playit credentials at fixed paths, with secrets excluded from argv. @@ -299,6 +301,14 @@ two-second deadline and two-second cadence. Observations expire after ten seconds. All health responses use closed-set state; raw provider errors and credentials are not exposed. +Tailscale reports observed failures as unavailable. While no forwarding attempt +is serving, including retry backoff and replacement registration, the supervisor +retains that known unavailable state without a provider-observation timestamp. +This local lifecycle fact does not expire. Once forwarding is established, fresh +provider observations replace it and retain the ten-second staleness limit. +The served health handler and metrics are checked after a real listener failure, +during a blocked replacement join, and after recovery in the regression source. + frpc's authenticated loopback API `/api/status` on 127.0.0.1:7400 must confirm every configured proxy is running without error, with the expected protocol. The generated `user` is the GameServer UID, preserving advertised proxy names diff --git a/tunnel/tailscale.go b/tunnel/tailscale.go index 7c938cf79..76a9b0ad0 100644 --- a/tunnel/tailscale.go +++ b/tunnel/tailscale.go @@ -80,11 +80,12 @@ func runTailscale(ctx context.Context, cfg Config, authKey string) error { return err } var ready atomic.Bool + healthHistory := &relayHealth{} listener, err := (&net.ListenConfig{}).Listen(ctx, "tcp", ":8091") if err != nil { return fmt.Errorf("tailscale health listener: %w", err) } - health := &http.Server{Handler: tailscaleHealth(&ready), ReadHeaderTimeout: 2 * time.Second, ReadTimeout: 5 * time.Second, WriteTimeout: 5 * time.Second, IdleTimeout: 10 * time.Second} + health := &http.Server{Handler: relayHealthHandler(healthHistory), ReadHeaderTimeout: 2 * time.Second, ReadTimeout: 5 * time.Second, WriteTimeout: 5 * time.Second, IdleTimeout: 10 * time.Second} healthDone := make(chan error, 1) go func() { healthDone <- health.Serve(listener) }() defer func() { _ = health.Close() }() @@ -94,8 +95,7 @@ func runTailscale(ctx context.Context, cfg Config, authKey string) error { if err != nil { return err } - healthHistory := &relayHealth{} - err = serveTailnetObserved(ctx, embeddedTailnet{server}, tags, ports, &ready, 2*time.Second, tailnetRetryObserver(healthHistory, backoff, time.Now)) + err = serveTailnetWithHealth(ctx, embeddedTailnet{server}, tags, ports, &ready, 2*time.Second, healthHistory, backoff) ready.Store(false) _ = server.Close() if ctx.Err() != nil { @@ -116,16 +116,6 @@ func runTailscale(ctx context.Context, cfg Config, authKey string) error { return nil } -func tailnetRetryObserver(history *relayHealth, backoff *exponentialBackoff, now func() time.Time) func(bool) { - return func(healthy bool) { - observation := relayObservation{Connection: "unknown"} - if healthy { - observation = relayObservation{Connection: "connected", Forwards: true, Assigned: true} - } - history.observe(now(), observation, backoff) - } -} - func waitTailnetRetry(ctx context.Context, healthDone <-chan error, delay time.Duration) error { timer := time.NewTimer(delay) defer timer.Stop() @@ -139,17 +129,27 @@ func waitTailnetRetry(ctx context.Context, healthDone <-chan error, delay time.D } } -func tailscaleHealth(ready *atomic.Bool) http.Handler { - mux := http.NewServeMux() - mux.HandleFunc("/healthz", func(w http.ResponseWriter, _ *http.Request) { w.WriteHeader(http.StatusOK) }) - mux.HandleFunc("/readyz", func(w http.ResponseWriter, _ *http.Request) { - if !ready.Load() { - http.Error(w, "tailnet or forwarding listeners unavailable", http.StatusServiceUnavailable) - return - } - w.WriteHeader(http.StatusOK) - }) - return mux +// serveTailnetWithHealth preserves a known outage while a replacement joins +// and after an attempt exits, until serving forwards provide fresh evidence. +func serveTailnetWithHealth(ctx context.Context, node tailnetNode, tags []string, ports []forwardPort, ready *atomic.Bool, poll time.Duration, history *relayHealth, backoff *exponentialBackoff) error { + history.markUnavailable() + defer history.markUnavailable() + return serveTailnetObserved(ctx, node, tags, ports, ready, poll, tailnetRetryObserver(history, backoff, time.Now)) +} + +func tailnetRetryObserver(history *relayHealth, backoff *exponentialBackoff, now func() time.Time) func(bool) { + return func(healthy bool) { + history.observe(now(), tailnetHealthObservation(healthy), backoff) + } +} + +func tailnetHealthObservation(healthy bool) relayObservation { + if healthy { + return relayObservation{Connection: "connected", Forwards: true, Assigned: true} + } + // Unlike Playit's cached assignment, tsnet provides an observed failure. + // Keep it known so an outage raises the unavailable alert. + return relayObservation{Connection: "unavailable"} } func tailnetReady(status *ipnstate.Status, tags []string) bool { diff --git a/tunnel/tailscale_test.go b/tunnel/tailscale_test.go index bceec3a0b..baedab707 100644 --- a/tunnel/tailscale_test.go +++ b/tunnel/tailscale_test.go @@ -7,6 +7,7 @@ import ( "net/http" "net/http/httptest" "net/netip" + "strings" "sync" "sync/atomic" "testing" @@ -223,21 +224,128 @@ func TestTailnetListenerFailureClearsReady(t *testing.T) { } } -func TestTailnetHealthNeedsForwardingReadiness(t *testing.T) { +type gatedTailnetNode struct { + *fakeTailnetNode + started chan struct{} + release chan struct{} +} + +func (n *gatedTailnetNode) Up(ctx context.Context) (*ipnstate.Status, error) { + close(n.started) + select { + case <-ctx.Done(): + return nil, ctx.Err() + case <-n.release: + return n.fakeTailnetNode.Up(ctx) + } +} + +func TestTailnetFailedAttemptRemainsKnownDuringRetry(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + history := &relayHealth{} + handler := relayHealthHandler(history) + ports := []forwardPort{{network: "tcp", port: "25565", target: "127.0.0.1:1"}} + node := &fakeTailnetNode{status: onlineTailnet()} var ready atomic.Bool - handler := tailscaleHealth(&ready) + done := make(chan error, 1) + go func() { + done <- serveTailnetWithHealth(ctx, node, nil, ports, &ready, time.Millisecond, history, nil) + }() + awaitTailnetReady(t, &ready, true) + if err := node.listeners[0].Close(); err != nil { + t.Fatal(err) + } + select { + case err := <-done: + if err == nil { + t.Fatal("failed listener was not reported") + } + case <-time.After(time.Second): + t.Fatal("failed attempt did not finish") + } + assertUnavailable := func() { + t.Helper() + // Advance beyond the maximum retry delay without sleeping: absence of + // a serving attempt is a local fact, not a stale provider observation. + if got := history.snapshot(time.Now().Add(6 * time.Minute)); got.Connection != "unavailable" || got.Ready { + t.Fatalf("retry forgot the known outage: %+v", got) + } + rec := httptest.NewRecorder() + handler.ServeHTTP(rec, httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/readyz", nil)) + if rec.Code != http.StatusServiceUnavailable { + t.Fatalf("failed attempt readyz=%d", rec.Code) + } + metrics := httptest.NewRecorder() + handler.ServeHTTP(metrics, httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/metrics", nil)) + for _, want := range []string{"gameplane_tunnel_connection_known 1\n", "gameplane_tunnel_connected 0\n", "gameplane_tunnel_ready 0\n"} { + if !strings.Contains(metrics.Body.String(), want) { + t.Fatalf("failed attempt missing %q: %s", want, metrics.Body.String()) + } + } + if strings.Contains(metrics.Body.String(), "gameplane_tunnel_observation_timestamp_seconds") { + t.Fatalf("supervisor outage invented a provider observation timestamp: %s", metrics.Body.String()) + } + } + assertUnavailable() + + // A new registration attempt must retain that state until forwarding is + // actually established, even if its Up call takes longer than the lease. + retry := &gatedTailnetNode{fakeTailnetNode: &fakeTailnetNode{status: onlineTailnet()}, started: make(chan struct{}), release: make(chan struct{})} + go func() { + done <- serveTailnetWithHealth(ctx, retry, nil, ports, &ready, time.Millisecond, history, nil) + }() + select { + case <-retry.started: + case <-time.After(time.Second): + t.Fatal("retry did not start") + } + assertUnavailable() + close(retry.release) + awaitTailnetReady(t, &ready, true) + deadline := time.Now().Add(time.Second) + for !history.snapshot(time.Now()).Ready && time.Now().Before(deadline) { + time.Sleep(time.Millisecond) + } + if got := history.snapshot(time.Now()); got.Connection != "connected" || !got.Ready { + t.Fatalf("recovered forwarding did not restore health: %+v", got) + } + if got := history.snapshot(time.Now().Add(2 * healthStaleAfter)); got.Connection != "unknown" || got.Ready { + t.Fatalf("live provider observations no longer expire: %+v", got) + } + cancel() + select { + case err := <-done: + if err != nil { + t.Fatal(err) + } + case <-time.After(time.Second): + t.Fatal("retry shutdown stalled") + } +} + +func TestTailnetHealthNeedsForwardingReadiness(t *testing.T) { + history := &relayHealth{} + handler := relayHealthHandler(history) for _, check := range []struct { path string ready bool want int }{{"/readyz", false, 503}, {"/healthz", false, 200}, {"/readyz", true, 200}, {"/readyz", false, 503}} { - ready.Store(check.ready) + history.observe(time.Now(), tailnetHealthObservation(check.ready), nil) rec := httptest.NewRecorder() handler.ServeHTTP(rec, httptest.NewRequestWithContext(context.Background(), http.MethodGet, check.path, nil)) if rec.Code != check.want { t.Fatalf("%s=%d want%d", check.path, rec.Code, check.want) } } + metrics := httptest.NewRecorder() + handler.ServeHTTP(metrics, httptest.NewRequestWithContext(t.Context(), http.MethodGet, "/metrics", nil)) + for _, want := range []string{"gameplane_tunnel_connection_known 1\n", "gameplane_tunnel_connected 0\n", "gameplane_tunnel_ready 0\n"} { + if !strings.Contains(metrics.Body.String(), want) { + t.Fatalf("confirmed tailnet failure missing %q: %s", want, metrics.Body.String()) + } + } } func TestTailnetConfirmedObservationsResetRetryEscalation(t *testing.T) { diff --git a/web/src/types.ts b/web/src/types.ts index 1832222bc..ac43d8827 100644 --- a/web/src/types.ts +++ b/web/src/types.ts @@ -47,6 +47,11 @@ export interface StatusMetricDecl { command?: string; regex?: string; unit?: string; + performance?: { + signal: "tick_rate" | "tick_duration" | "frame_rate"; + unit: "per_second" | "seconds" | "milliseconds"; + degradedThreshold?: string; + }; } // Install policy for the mods capability (spec.capabilities.mods.install).