diff --git a/api/internal/handlers/config_redact.go b/api/internal/handlers/config_redact.go index 5d37b99d5..f801424f2 100644 --- a/api/internal/handlers/config_redact.go +++ b/api/internal/handlers/config_redact.go @@ -2,9 +2,9 @@ package handlers import ( "context" + "errors" "sync" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "github.com/GameplanePanel/gameplane/api/internal/kube" @@ -58,32 +58,43 @@ func rulesFromTemplate(tmpl *unstructured.Unstructured) configRules { type configRuleCache struct { k *kube.Client mu sync.Mutex - entries map[string]configRules + entries map[configRuleKey]configRules +} + +// A revision is content-addressed, but snapshot ownership belongs to one +// server. Never reuse a verified snapshot for a replacement or another server. +type configRuleKey struct { + template, namespace, name, uid, revision string } func newConfigRuleCache(k *kube.Client) *configRuleCache { - return &configRuleCache{k: k, entries: map[string]configRules{}} + return &configRuleCache{k: k, entries: map[configRuleKey]configRules{}} } -// rulesFor returns the rules for the named template, failing closed when the -// template cannot be read. -func (c *configRuleCache) rulesFor(ctx context.Context, templateName string) configRules { - if templateName == "" || c == nil || c.k == nil || c.k.Dynamic == nil { +// rulesFor uses the server's admitted template, failing closed when that +// template cannot be read or its pinned snapshot cannot be verified. +func (c *configRuleCache) rulesFor(ctx context.Context, gs *unstructured.Unstructured) configRules { + if gs == nil || c == nil || c.k == nil { return configRules{failClosed: true} } + templateName, _, _ := unstructured.NestedString(gs.Object, "spec", "templateRef", "name") + key := configRuleKey{template: templateName, namespace: gs.GetNamespace(), name: gs.GetName(), uid: string(gs.GetUID()), revision: gs.GetAnnotations()[kube.EffectiveTemplateRevisionAnnotation]} c.mu.Lock() - cached, ok := c.entries[templateName] + cached, ok := c.entries[key] c.mu.Unlock() if ok { return cached } - tmpl, err := c.k.Dynamic.Resource(kube.GVRs["templates"]).Get(ctx, templateName, metav1.GetOptions{}) + tmpl, err := c.k.EffectiveGameTemplate(ctx, gs) if err != nil { return configRules{failClosed: true} } rules := rulesFromTemplate(tmpl) + if rules.failClosed { + return rules + } c.mu.Lock() - c.entries[templateName] = rules + c.entries[key] = rules c.mu.Unlock() return rules } @@ -107,8 +118,7 @@ func (c *configRuleCache) redact(ctx context.Context, gs *unstructured.Unstructu if len(cfg) == 0 { return } - name, _, _ := unstructured.NestedString(gs.Object, "spec", "templateRef", "name") - rules := c.rulesFor(ctx, name) + rules := c.rulesFor(ctx, gs) for key, val := range cfg { if !rules.redacts(key) { continue @@ -127,6 +137,32 @@ func (c *configRuleCache) redactList(ctx context.Context, list *unstructured.Uns } } +// validateCloneConfig prevents secrets copied from a pinned source from +// becoming ordinary config on an unpinned clone. Redacting only the POST reply +// would still expose them on the clone's subsequent GET/list/event responses. +func (c *configRuleCache) validateCloneConfig(ctx context.Context, source, clone *unstructured.Unstructured) error { + if source.GetAnnotations()[kube.EffectiveTemplateRevisionAnnotation] == "" { + return nil + } + cfg, _, err := unstructured.NestedMap(source.Object, "spec", "config") + if err != nil { + return errors.New("cannot clone: source config is malformed") + } + if len(cfg) == 0 { + return nil + } + sourceRules, targetRules := c.rulesFor(ctx, source), c.rulesFor(ctx, clone) + for key, value := range cfg { + if s, ok := value.(string); ok && s == "" { + continue + } + if sourceRules.redacts(key) && (targetRules.failClosed || !targetRules.password[key]) { + return errors.New("cannot clone: the current template does not protect copied passwords; restore its password field types or create a new server with fresh configuration") + } + } + return nil +} + // restoreRedactedConfig rewrites desired.spec.config before a PUT so redacted // values the client sent back do not overwrite stored passwords. live is the // stored object (nil when it does not exist). The rules are documented in @@ -146,8 +182,7 @@ func restoreRedactedConfig(ctx context.Context, c *configRuleCache, desired, liv return } if live != nil { - name, _, _ := unstructured.NestedString(live.Object, "spec", "templateRef", "name") - rules = c.rulesFor(ctx, name) + rules = c.rulesFor(ctx, live) } out := make(map[string]any, len(incoming)) for key, val := range incoming { diff --git a/api/internal/handlers/config_redact_revision_test.go b/api/internal/handlers/config_redact_revision_test.go new file mode 100644 index 000000000..82c8c709c --- /dev/null +++ b/api/internal/handlers/config_redact_revision_test.go @@ -0,0 +1,318 @@ +package handlers + +import ( + "context" + "crypto/sha256" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "reflect" + "strings" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/watch" + dynamicfake "k8s.io/client-go/dynamic/fake" + kubefake "k8s.io/client-go/kubernetes/fake" + clienttesting "k8s.io/client-go/testing" + + "github.com/GameplanePanel/gameplane/api/internal/auth" + "github.com/GameplanePanel/gameplane/api/internal/kube" + "github.com/GameplanePanel/gameplane/api/internal/scope" +) + +// Mirror the operator's immutable snapshot wire format, without fetching the +// current catalog template when preparing a pinned server fixture. +func pinRedactionTemplate(t *testing.T, k *kube.Client, gs, tmpl *unstructured.Unstructured) *corev1.ConfigMap { + t.Helper() + if gs.GetUID() == "" { + gs.SetUID(types.UID("uid-" + gs.GetName())) + } + data, err := json.Marshal(map[string]any{"metadata": map[string]any{"name": tmpl.GetName()}, "spec": tmpl.Object["spec"]}) + if err != nil { + t.Fatal(err) + } + revision := fmt.Sprintf("%x", sha256.Sum256(data)) + identity := sha256.Sum256([]byte(string(gs.GetUID()) + "/" + gs.GetNamespace() + "/" + gs.GetName())) + immutable := true + cm := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{ + Name: fmt.Sprintf("gpt-%x-%s", identity[:8], revision[:32]), Namespace: gs.GetNamespace(), + OwnerReferences: []metav1.OwnerReference{{APIVersion: "gameplane.local/v1alpha1", Kind: "GameServer", Name: gs.GetName(), UID: gs.GetUID(), Controller: &immutable}}, + }, Immutable: &immutable, Data: map[string]string{"template.json": string(data)}} + if _, err := k.Typed.CoreV1().ConfigMaps(gs.GetNamespace()).Create(t.Context(), cm, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + annotations := gs.GetAnnotations() + if annotations == nil { + annotations = map[string]string{} + } + annotations[kube.EffectiveTemplateRevisionAnnotation] = revision + gs.SetAnnotations(annotations) + return cm +} + +func catalogWithoutPasswords() *unstructured.Unstructured { + tmpl := redactTemplate() + tmpl.Object["spec"] = map[string]any{"configSchema": []any{ + map[string]any{"name": "MOTD", "type": "string"}, + map[string]any{"name": "RCON_PASSWORD", "type": "string"}, + }} + return tmpl +} + +func TestRedact_PinnedTemplateSurvivesCatalogPasswordRemoval(t *testing.T) { + k := fakeKubeClient(catalogWithoutPasswords()) + gs := serverWithConfig("alpha", plainConfig()) + pinRedactionTemplate(t, k, gs, redactTemplate()) + if _, err := k.Dynamic.Resource(kube.GVRs["servers"]).Namespace(redactNS).Create(t.Context(), gs, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + router := mountResourcesRouter(k) + for _, path := range []string{"/servers/alpha", "/servers/"} { + rr := do(t, router, http.MethodGet, path, nil) + if rr.Code != http.StatusOK { + t.Fatalf("%s: %d %s", path, rr.Code, rr.Body) + } + assertNoSecretLeak(t, rr.Body.String()) + if !strings.Contains(rr.Body.String(), "hello") { + t.Fatalf("%s failed closed despite a valid snapshot: %s", path, rr.Body) + } + } + for _, incoming := range []map[string]any{ + {"MOTD": "hello", "RCON_PASSWORD": configRedactedMarker, "ADMIN_PASSWORD": ""}, + {"MOTD": "hello", "RCON_PASSWORD": configRedactedMarker}, + } { + body := updateBody(incoming) + body["metadata"].(map[string]any)["uid"] = string(gs.GetUID()) + rr := do(t, router, http.MethodPut, "/servers/alpha", body) + if rr.Code != http.StatusOK { + t.Fatalf("put: %d %s", rr.Code, rr.Body) + } + assertNoSecretLeak(t, rr.Body.String()) + if got := storedConfig(t, k, "alpha"); !reflect.DeepEqual(got, plainConfig()) { + t.Fatalf("PUT lost the pinned template's required password: %v", got) + } + } +} + +func TestRedact_PinnedTemplateCacheSeparatesServersAndRevisions(t *testing.T) { + k := fakeKubeClient(catalogWithoutPasswords()) + old := serverWithConfig("old", plainConfig()) + pinRedactionTemplate(t, k, old, redactTemplate()) + current := serverWithConfig("current", plainConfig()) + pinRedactionTemplate(t, k, current, catalogWithoutPasswords()) + cache := newConfigRuleCache(k) + cache.redact(t.Context(), current) + cache.redact(t.Context(), old) + if cfg := specConfig(t, old.Object); cfg["ADMIN_PASSWORD"] != configRedactedMarker || cfg["RCON_PASSWORD"] != configRedactedMarker { + t.Fatalf("current schema escaped into the older server: %v", cfg) + } + // The same server adopting another revision within a request must use its + // new schema, while a same-name replacement must re-prove snapshot ownership. + next := serverWithConfig("current", plainConfig()) + next.SetUID(current.GetUID()) + pinRedactionTemplate(t, k, next, redactTemplate()) + cache.redact(t.Context(), next) + if specConfig(t, next.Object)["RCON_PASSWORD"] != configRedactedMarker { + t.Fatal("revision change reused the old cache entry") + } + replacement := current.DeepCopy() + replacement.SetUID("replacement-uid") + cache.redact(t.Context(), replacement) + if specConfig(t, replacement.Object)["MOTD"] != configRedactedMarker { + t.Fatal("replacement server reused another UID's verified snapshot") + } + otherNamespace := current.DeepCopy() + otherNamespace.SetNamespace("another-namespace") + cache.redact(t.Context(), otherNamespace) + if specConfig(t, otherNamespace.Object)["MOTD"] != configRedactedMarker { + t.Fatal("another namespace reused a verified snapshot") + } +} + +func TestRedact_InvalidPinnedTemplateFailsClosedWithoutCatalogFallback(t *testing.T) { + for _, mode := range []string{"missing", "malformed-revision", "uppercase-revision", "missing-uid", "mutable", "foreign-owner", "wrong-owner-name", "wrong-owner-kind", "wrong-owner-api", "deleting", "corrupt-content", "wrong-template", "wrong-namespace", "malformed-schema"} { + t.Run(mode, func(t *testing.T) { + k := fakeKubeClient(catalogWithoutPasswords()) + gs := serverWithConfig("alpha", plainConfig()) + snapshotTemplate := redactTemplate() + if mode == "wrong-template" { + snapshotTemplate.SetName("other-template") + } + if mode == "malformed-schema" { + snapshotTemplate.Object["spec"].(map[string]any)["configSchema"] = "invalid" + } + cm := pinRedactionTemplate(t, k, gs, snapshotTemplate) + switch mode { + case "missing": + if err := k.Typed.CoreV1().ConfigMaps(cm.Namespace).Delete(t.Context(), cm.Name, metav1.DeleteOptions{}); err != nil { + t.Fatal(err) + } + case "malformed-revision": + gs.SetAnnotations(map[string]string{kube.EffectiveTemplateRevisionAnnotation: "../invalid"}) + case "uppercase-revision": + gs.SetAnnotations(map[string]string{kube.EffectiveTemplateRevisionAnnotation: strings.ToUpper(gs.GetAnnotations()[kube.EffectiveTemplateRevisionAnnotation])}) + case "missing-uid": + gs.SetUID("") + case "mutable": + cm.Immutable = nil + case "foreign-owner": + cm.OwnerReferences[0].UID = "other-server" + case "wrong-owner-name": + cm.OwnerReferences[0].Name = "other-server" + case "wrong-owner-kind": + cm.OwnerReferences[0].Kind = "Backup" + case "wrong-owner-api": + cm.OwnerReferences[0].APIVersion = "foreign.example/v1alpha1" + case "deleting": + now := metav1.Now() + cm.DeletionTimestamp = &now + case "corrupt-content": + cm.Data["template.json"] = `{"metadata":{"name":"minecraft"},"spec":{}}` + case "wrong-namespace": + gs.SetNamespace("another-namespace") + } + if mode != "missing" { + if _, err := k.Typed.CoreV1().ConfigMaps(cm.Namespace).Update(t.Context(), cm, metav1.UpdateOptions{}); err != nil { + t.Fatal(err) + } + } + newConfigRuleCache(k).redact(t.Context(), gs) + for key, value := range specConfig(t, gs.Object) { + if value != configRedactedMarker { + t.Fatalf("%s leaked despite invalid snapshot: %v", key, value) + } + } + if len(k.Dynamic.(*dynamicfake.FakeDynamicClient).Actions()) != 0 { + t.Fatal("invalid pin fell back to current catalog") + } + }) + } +} + +func TestRedact_PinnedSnapshotFailureIsNotCached(t *testing.T) { + k := fakeKubeClient(catalogWithoutPasswords()) + gs := serverWithConfig("alpha", plainConfig()) + cm := pinRedactionTemplate(t, k, gs, redactTemplate()) + if err := k.Typed.CoreV1().ConfigMaps(cm.Namespace).Delete(t.Context(), cm.Name, metav1.DeleteOptions{}); err != nil { + t.Fatal(err) + } + cache := newConfigRuleCache(k) + missing := gs.DeepCopy() + cache.redact(t.Context(), missing) + if specConfig(t, missing.Object)["MOTD"] != configRedactedMarker { + t.Fatal("missing snapshot did not fail closed") + } + if _, err := k.Typed.CoreV1().ConfigMaps(cm.Namespace).Create(t.Context(), cm, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + cache.redact(t.Context(), gs) + if cfg := specConfig(t, gs.Object); cfg["MOTD"] != "hello" || cfg["RCON_PASSWORD"] != configRedactedMarker { + t.Fatalf("recovered snapshot did not supply its schema: %v", cfg) + } +} + +func TestRedact_PinnedCloneRejectsPasswordDowngradeBeforeCreate(t *testing.T) { + for _, mode := range []string{"removed", "retyped", "missing-target", "missing-source", "unchanged-passwords", "empty-passwords"} { + t.Run(mode, func(t *testing.T) { + catalog := redactTemplate() + fields := catalog.Object["spec"].(map[string]any)["configSchema"].([]any) + switch mode { + case "removed": + catalog.Object["spec"].(map[string]any)["configSchema"] = fields[:2] + case "retyped": + fields[1].(map[string]any)["type"] = "string" + case "empty-passwords": + catalog = catalogWithoutPasswords() + } + k := fakeKubeClient(catalog) + gs := serverWithConfig("alpha", plainConfig()) + if mode == "empty-passwords" { + gs.Object["spec"].(map[string]any)["config"] = map[string]any{"MOTD": "hello", "RCON_PASSWORD": "", "ADMIN_PASSWORD": ""} + } + cm := pinRedactionTemplate(t, k, gs, redactTemplate()) + if mode == "missing-target" { + if err := k.Dynamic.Resource(kube.GVRs["templates"]).Delete(t.Context(), catalog.GetName(), metav1.DeleteOptions{}); err != nil { + t.Fatal(err) + } + } + if mode == "missing-source" { + if err := k.Typed.CoreV1().ConfigMaps(cm.Namespace).Delete(t.Context(), cm.Name, metav1.DeleteOptions{}); err != nil { + t.Fatal(err) + } + } + if _, err := k.Dynamic.Resource(kube.GVRs["servers"]).Namespace(redactNS).Create(t.Context(), gs, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + rr := do(t, mountLifecycleRouter(k), http.MethodPost, "/servers/alpha:clone", map[string]any{"newName": "beta"}) + assertNoSecretLeak(t, rr.Body.String()) + if mode == "unchanged-passwords" || mode == "empty-passwords" { + if rr.Code != http.StatusOK { + t.Fatalf("safe clone: %d %s", rr.Code, rr.Body) + } + if !reflect.DeepEqual(storedConfig(t, k, "beta"), specConfig(t, gs.Object)) { + t.Fatal("safe clone changed config") + } + return + } + if rr.Code != http.StatusConflict || !strings.Contains(rr.Body.String(), "current template") { + t.Fatalf("unsafe clone: %d %s", rr.Code, rr.Body) + } + if _, err := k.Dynamic.Resource(kube.GVRs["servers"]).Namespace(redactNS).Get(t.Context(), "beta", metav1.GetOptions{}); !apierrors.IsNotFound(err) { + t.Fatalf("rejected clone persisted: %v", err) + } + }) + } +} + +func TestRedact_PinnedFleetAndEventsUseSelectedClusterSnapshot(t *testing.T) { + previous := scope.AllowedNamespaces + scope.AllowedNamespaces = []string{scope.DefaultNamespace} + t.Cleanup(func() { scope.AllowedNamespaces = previous }) + k := fleetTestClient(catalogWithoutPasswords()) + gs := serverWithConfig("alpha", plainConfig()) + gs.SetNamespace(scope.DefaultNamespace) + pinRedactionTemplate(t, k, gs, redactTemplate()) + if _, err := k.Dynamic.Resource(kube.GVRs["servers"]).Namespace(gs.GetNamespace()).Create(t.Context(), gs, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + home := fleetTestClient(catalogWithoutPasswords()) + reg := clusterTestRegistry(home) + reg.Set("remote", k) + rr := inventoryRequest(t, fleetRouter(reg), inventoryUser("remote", gs.GetNamespace(), "servers:read"), http.MethodGet, "/fleet/servers?cluster=remote") + if rr.Code != http.StatusOK || !strings.Contains(rr.Body.String(), "hello") { + t.Fatalf("fleet: %d %s", rr.Code, rr.Body) + } + assertNoSecretLeak(t, rr.Body.String()) + if len(home.Typed.(*kubefake.Clientset).Actions()) != 0 { + t.Fatal("remote redaction consulted the home cluster") + } + fw := watch.NewFake() + k.Dynamic.(*dynamicfake.FakeDynamicClient).PrependWatchReactor("gameservers", func(clienttesting.Action) (bool, watch.Interface, error) { return true, fw, nil }) + ctx, cancel := context.WithCancel(auth.WithUser(t.Context(), testAdminUser())) + defer cancel() + req := httptest.NewRequestWithContext(ctx, http.MethodGet, "/events?cluster=remote", nil) + w := &lockedBuffer{header: http.Header{}} + done := make(chan struct{}) + go func() { defer close(done); eventsHandler(reg)(w, req) }() + go fw.Add(gs) + deadline := time.Now().Add(5 * time.Second) + for !strings.Contains(w.String(), "data:") && time.Now().Before(deadline) { + time.Sleep(10 * time.Millisecond) + } + cancel() + <-done + if out := w.String(); !strings.Contains(out, "data:") || !strings.Contains(out, "hello") { + t.Fatalf("missing pinned event: %s", out) + } + assertNoSecretLeak(t, w.String()) + if specConfig(t, gs.Object)["ADMIN_PASSWORD"] != "adm1n" { + t.Fatal("SSE redacted the shared watch object in place") + } +} diff --git a/api/internal/handlers/lifecycle.go b/api/internal/handlers/lifecycle.go index 2622f318c..9d46758fa 100644 --- a/api/internal/handlers/lifecycle.go +++ b/api/internal/handlers/lifecycle.go @@ -233,6 +233,11 @@ func cloneHandler(reg *kube.Registry) http.HandlerFunc { // never grants host-level diagnostic privileges to the new server. unstructured.RemoveNestedField(clone.Object, "spec", "capture", "nodeDiagnostic") stampOwner(clone, req) + configRules := newConfigRuleCache(k) + if err := configRules.validateCloneConfig(req.Context(), src, clone); err != nil { + httperr.WriteCode(w, req, http.StatusConflict, err) + return + } cl := scope.RequestedCluster(req) if err := validateAndProtectGameServer(req.Context(), k, cl, ns, body.NewName, clone, nil); err != nil { @@ -249,7 +254,7 @@ func cloneHandler(reg *kube.Registry) http.HandlerFunc { } // The clone stores the source's config verbatim; only the response is redacted. created = created.DeepCopy() - newConfigRuleCache(k).redact(req.Context(), created) + configRules.redact(req.Context(), created) writeOrErr(w, req, created, nil) } } diff --git a/api/internal/kube/template_revision.go b/api/internal/kube/template_revision.go new file mode 100644 index 000000000..1ef1eba2c --- /dev/null +++ b/api/internal/kube/template_revision.go @@ -0,0 +1,77 @@ +package kube + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +// EffectiveTemplateRevisionAnnotation records the template admitted by the +// operator. Once set, consumers must not fall back to the current catalog. +const EffectiveTemplateRevisionAnnotation = "gameplane.local/effective-template-revision" + +// EffectiveGameTemplate reads the schema used by this server, including an +// OnRequest server that has not adopted the latest catalog revision. Snapshot +// identity and content checks match the operator's snapshot contract; reads use +// this cluster's client and the server's namespace, never a caller-supplied path. +func (c *Client) EffectiveGameTemplate(ctx context.Context, gs *unstructured.Unstructured) (*unstructured.Unstructured, error) { + if c == nil || gs == nil { + return nil, fmt.Errorf("template client or server is missing") + } + name, _, err := unstructured.NestedString(gs.Object, "spec", "templateRef", "name") + if err != nil || name == "" { + return nil, fmt.Errorf("server template name is missing or malformed") + } + revision := gs.GetAnnotations()[EffectiveTemplateRevisionAnnotation] + if revision == "" { + if c.Dynamic == nil { + return nil, fmt.Errorf("template client is missing") + } + return c.Dynamic.Resource(GVRs["templates"]).Get(ctx, name, metav1.GetOptions{}) + } + decoded, err := hex.DecodeString(revision) + if err != nil || len(decoded) != sha256.Size || hex.EncodeToString(decoded) != revision { + return nil, fmt.Errorf("invalid effective template revision") + } + if c.Typed == nil || gs.GetUID() == "" || gs.GetNamespace() == "" || gs.GetName() == "" { + return nil, fmt.Errorf("snapshot client or server identity is missing") + } + identity := sha256.Sum256([]byte(string(gs.GetUID()) + "/" + gs.GetNamespace() + "/" + gs.GetName())) + snapshotName := fmt.Sprintf("gpt-%x-%s", identity[:8], revision[:32]) + cm, err := c.Typed.CoreV1().ConfigMaps(gs.GetNamespace()).Get(ctx, snapshotName, metav1.GetOptions{}) + if err != nil { + return nil, fmt.Errorf("read pinned template snapshot: %w", err) + } + owner := metav1.GetControllerOf(cm) + if cm.Name != snapshotName || cm.Namespace != gs.GetNamespace() || owner == nil || + owner.UID != gs.GetUID() || owner.Name != gs.GetName() || owner.Kind != "GameServer" || owner.APIVersion != "gameplane.local/v1alpha1" || + cm.Immutable == nil || !*cm.Immutable || !cm.DeletionTimestamp.IsZero() { + return nil, fmt.Errorf("template snapshot is foreign, mutable or deleting") + } + data := []byte(cm.Data["template.json"]) + if len(data) > 900*1024 { + return nil, fmt.Errorf("template snapshot exceeds 900 KiB") + } + // The operator stores the exact JSON bytes it hashes. Hash those bytes + // directly so the API does not need a second copy of the typed CRD schema. + // Noncanonical rewrites are rejected even if they decode to the same spec. + digest := sha256.Sum256(data) + if hex.EncodeToString(digest[:]) != revision { + return nil, fmt.Errorf("template snapshot content does not match pinned revision") + } + var tmpl unstructured.Unstructured + // The stored snapshot deliberately omits kind/apiVersion. Decode its map + // directly rather than invoking Unstructured's GVK-requiring decoder. + if err := json.Unmarshal(data, &tmpl.Object); err != nil { + return nil, fmt.Errorf("decode template snapshot: %w", err) + } + if tmpl.GetName() != name { + return nil, fmt.Errorf("template snapshot name does not match server") + } + return &tmpl, nil +} diff --git a/api/specs.md b/api/specs.md index 1a202ed09..822387d95 100644 --- a/api/specs.md +++ b/api/specs.md @@ -655,11 +655,12 @@ Foreign keys are enforced only on Postgres (modernc-sqlite runs with FK OFF); th ### GameServer config password redaction - Every covered response omits `metadata.annotations["kubectl.kubernetes.io/last-applied-configuration"]`: this historical manifest can contain passwords absent from the current config or template. Omission applies even when current config is absent, empty, or malformed; stored Kubernetes metadata and other annotations remain unchanged. `GameServer.spec.config` stores wizard values in clear, including those of `type: password` fields in the template's `configSchema`. Anyone holding `servers:read` (or the owner/collaborator fallback) could therefore read every password. The API now redacts them; implementation and tests are in `api/internal/handlers/config_redact.go` and `config_redact_test.go`. -- **Marker:** `__gameplane_redacted__`. On reads, each non-empty password-type value is replaced by the marker (an empty value stays empty so clients can show "not set"). The password-field set comes from the server's `GameTemplate.spec.configSchema` (read live and memoised only within one request, or within one SSE event, so a template edit applies to the very next response and event). +- **Marker:** `__gameplane_redacted__`. On reads, each non-empty password-type value is replaced by the marker (an empty value stays empty so clients can show "not set"). The password-field set comes from the server's effective `GameTemplate.spec.configSchema`. When `gameplane.local/effective-template-revision` is set, the API reads the server's immutable snapshot from the selected cluster and namespace, verifies its controller owner, identity, digest and template name, and rejects deleting snapshots. The exact stored JSON must match the pinned SHA-256 digest. An OnRequest server keeps its admitted password rules even when the catalog removes or retypes those fields. Servers without a pin use the current catalog. Rules are memoised only within one request or SSE event, keyed by template, namespace, server name, UID and revision. - **Covered paths (one helper):** GET/POST/PUT `/servers` and `/servers/{name}` (including the create and update replies), `POST /servers/{name}:clone` reply (the clone itself stores the source's values verbatim), `GET /users/me/servers`, `GET /fleet/servers`, and the `/events` SSE stream (each watch event is redacted on a copy). Share links expose only name/status/address/player count; capture, mod, tunnel-credential, notification and WebSocket paths never carry `spec.config`; the audit log and its sinks record actor/method/path/target/status/ip/reason only and never request bodies. -- **Fail closed:** when the template cannot be read (no `templateRef`, not found, API error, malformed `configSchema`) every non-empty config value is redacted. +- **Fail closed:** when the template cannot be read (no `templateRef`, not found, API error, malformed `configSchema`), or a pinned snapshot is missing, unreadable or invalid, every non-empty config value is redacted. A failed pin lookup never falls back to the latest catalog. The API needs `get` on ConfigMaps in each allowed game namespace; the chart grants it through the games Role, and remote kubeconfigs need the same permission. Failed lookups are not cached. PUT secret preservation uses this same effective schema. - **Update rule (PUT `/servers/{name}`)**, applied before the write (markers: every key; the other rules: per password-type key, every key when failing closed): a value equal to the marker keeps the stored value for ANY key, whether or not the current template marks it a password and whatever the stored value's type (the marker is API-emitted, and a fail-closed read marks every key); if the key is absent from the stored object the marker is dropped, never stored; a different non-empty value replaces it; an explicit empty string clears it only when the field is optional, for a `required` field (or when failing closed, where optionality is unknown) the stored value is kept; a key omitted from the body is removed as in any PUT, except a `required` password field, which is restored. This is what makes the dashboard's whole-`spec` PUT (`mergeDraftOntoLatest`) safe. On POST, marker values are dropped. - **Not covered:** Kubernetes API readers (kubectl, GitOps, the optional read-only `mcp-server`) see the stored object; protect it with cluster RBAC. +- **Pinned source clones:** cloning drops the source's server-specific pin. Before creating the clone, the API checks that its current catalog schema still marks every copied, non-empty source password as a password. If those fields were removed/retyped, or their protection cannot be verified, cloning returns `409` without creating anything. Restore the template's password types or create a new server with fresh configuration. Safe clones continue to copy config verbatim. ### Audit - **Scope:** every mutating request (POST/PATCH/DELETE); reads excluded diff --git a/assets/design-export/MANIFEST.md b/assets/design-export/MANIFEST.md index f8656ad32..60128ae85 100644 --- a/assets/design-export/MANIFEST.md +++ b/assets/design-export/MANIFEST.md @@ -2810,3 +2810,14 @@ Merging master after PR #815 conflicted on `design.pen`. The file was taken from The operator has supported `spec.idle.wakeOnConnect` since v0.2.0-beta.8, but the dashboard had no control for it. No reusable component changed and Row 26 keeps its height, so no other node moved. `check-layout.js`: `NO ISSUES`. `pen-componentize` `audit.js`: nothing new on the two frames; the repeated "Field Auto-restart" and probe "row1" structures it lists predate this change. **Exports:** JSON from `Get(id, {depth: 80, includePathGeometry: true})`, pretty-printed (zero `"..."` elisions, `python3 -m json.tool` passes). PNG from `Export(ids, "png", …, {scale: 2})`: both 2880×3240. Headless `pen interactive` session (pen.dev CLI 0.3.14), with the lucide icon font served from the `lucide-static@0.563.0` npm tarball because unpkg.com is blocked in that sandbox. Body-text checks: `"Wake the server when a player connects"` only in `i1bLR.json` and `A8vGwM.json`; `"unable to wake the server"` only in `A8vGwM.json`. + +## Incremental export 2026-11-04 — Backup consistency and volume coverage (PR #769, inherited by #772) + +| Node | Name | Change | +| --- | --- | --- | +| `zhLZN` | Gameplane/Backup Detail Drawer | Added achieved consistency and legacy volume-coverage fields after Server. The legacy state explicitly says capture metadata is unavailable instead of inferring coverage from current configuration. The drawer remains 440×760. | +| `bHJy6` | Gameplane/Backup Volume Coverage — Recorded | Added a 396px-wide coverage state with capture time, the single-primary-claim scope, and included data, excluded rebuildable, and excluded unclassified claims. Rebuildable is explicitly a declaration rather than proof of recoverability. The width includes the drawer body's scrollbar allowance; card padding includes the 1px inside border. | + +Source: `web/src/components/backups/BackupDetailDrawer.tsx`. The recorded state uses the matching screenshot fixture in `web/src/test/handlers.ts`; `web/e2e/screenshots/slice-3.spec.ts` captures both node IDs and checks their distinct content. + +**Exports:** JSON from Pencil `Get(id, {depth: 30, includePathGeometry: true})`, with no truncation markers; PNG from `Export(ids, "png", …, {scale: 2})`. Both nodes were visually inspected in Pencil, and the recorded state was checked manually in Chrome. The user saved `design.pen` in Pencil after the final sizing adjustment. TypeScript application and screenshot source compilation passed; automated visual execution remains CI-only. The drawer reference crop was remeasured as `(120,96)-(1000,1616)` in the new PNG, retaining the 880×1520 panel. diff --git a/assets/design-export/json/bHJy6.json b/assets/design-export/json/bHJy6.json new file mode 100644 index 000000000..adabe84d3 --- /dev/null +++ b/assets/design-export/json/bHJy6.json @@ -0,0 +1,273 @@ +{ + "type": "frame", + "id": "bHJy6", + "x": 2276, + "y": 4461, + "name": "Gameplane/Backup Volume Coverage — Recorded", + "reusable": true, + "width": 396, + "fill": "$surface/surface", + "layout": "vertical", + "gap": 4, + "children": [ + { + "type": "text", + "id": "r8Owu", + "name": "Coverage label", + "fill": "$muted", + "content": "VOLUME COVERAGE", + "lineHeight": 1.333333, + "fontFamily": "$typography/font-sans", + "fontSize": 12, + "fontWeight": "normal", + "letterSpacing": 0.5 + }, + { + "type": "frame", + "id": "mfHN5", + "name": "Recorded capture manifest", + "width": "fill_container", + "layout": "vertical", + "gap": 12, + "children": [ + { + "type": "text", + "id": "Z5aM17", + "name": "Capture recorded at", + "fill": "$muted", + "textGrowth": "fixed-width", + "width": "fill_container", + "content": "Recorded: 2026-07-13T00:12:04Z", + "lineHeight": 1.333333, + "fontFamily": "$typography/font-sans", + "fontSize": 12, + "fontWeight": "normal" + }, + { + "type": "text", + "id": "etbdz", + "name": "Coverage scope", + "fill": "$muted", + "textGrowth": "fixed-width", + "width": "fill_container", + "content": "Only the primary data claim is copied. Rebuildable is a declaration, not proof that recovery can reconstruct its contents.", + "lineHeight": 1.333333, + "fontFamily": "$typography/font-sans", + "fontSize": 12, + "fontWeight": "normal" + }, + { + "type": "frame", + "id": "zLDqu", + "name": "Captured volume inventory", + "width": "fill_container", + "layout": "vertical", + "gap": 12, + "children": [ + { + "type": "frame", + "id": "jgV6n", + "name": "Volume game-data", + "width": "fill_container", + "cornerRadius": 4, + "stroke": "$border/border", + "strokeWidth": 1, + "strokeAlignment": "inner", + "layout": "vertical", + "gap": 4, + "padding": 9, + "children": [ + { + "type": "text", + "id": "F0Qiz", + "name": "Claim name", + "fill": "$foreground/foreground", + "textGrowth": "fixed-width", + "width": "fill_container", + "content": "game-data", + "lineHeight": 1.333333, + "fontFamily": "$typography/font-mono", + "fontSize": 12, + "fontWeight": "normal" + }, + { + "type": "text", + "id": "b4aDFu", + "name": "Coverage result", + "fill": "$foreground/foreground", + "textGrowth": "fixed-width", + "width": "fill_container", + "content": "Included", + "lineHeight": 1.428571, + "fontFamily": "$typography/font-sans", + "fontSize": 14, + "fontWeight": "normal" + }, + { + "type": "text", + "id": "pq4ts", + "name": "Volume classification", + "fill": "$muted", + "textGrowth": "fixed-width", + "width": "fill_container", + "content": "Data", + "lineHeight": 1.333333, + "fontFamily": "$typography/font-sans", + "fontSize": 12, + "fontWeight": "normal" + }, + { + "type": "text", + "id": "fM4kj", + "name": "Mount path", + "fill": "$foreground/foreground", + "textGrowth": "fixed-width", + "width": "fill_container", + "content": "/world", + "lineHeight": 1.333333, + "fontFamily": "$typography/font-mono", + "fontSize": 12, + "fontWeight": "normal" + } + ] + }, + { + "type": "frame", + "id": "DTQiV", + "name": "Volume game-extra-install", + "width": "fill_container", + "cornerRadius": 4, + "stroke": "$border/border", + "strokeWidth": 1, + "strokeAlignment": "inner", + "layout": "vertical", + "gap": 4, + "padding": 9, + "children": [ + { + "type": "text", + "id": "MiYbU", + "name": "Claim name", + "fill": "$foreground/foreground", + "textGrowth": "fixed-width", + "width": "fill_container", + "content": "game-extra-install", + "lineHeight": 1.333333, + "fontFamily": "$typography/font-mono", + "fontSize": 12, + "fontWeight": "normal" + }, + { + "type": "text", + "id": "GhJX5", + "name": "Coverage result", + "fill": "$foreground/foreground", + "textGrowth": "fixed-width", + "width": "fill_container", + "content": "Not covered", + "lineHeight": 1.428571, + "fontFamily": "$typography/font-sans", + "fontSize": 14, + "fontWeight": "normal" + }, + { + "type": "text", + "id": "ektuq", + "name": "Volume classification", + "fill": "$muted", + "textGrowth": "fixed-width", + "width": "fill_container", + "content": "Rebuildable", + "lineHeight": 1.333333, + "fontFamily": "$typography/font-sans", + "fontSize": 12, + "fontWeight": "normal" + }, + { + "type": "text", + "id": "HGTOq", + "name": "Mount path", + "fill": "$foreground/foreground", + "textGrowth": "fixed-width", + "width": "fill_container", + "content": "/serverfiles", + "lineHeight": 1.333333, + "fontFamily": "$typography/font-mono", + "fontSize": 12, + "fontWeight": "normal" + } + ] + }, + { + "type": "frame", + "id": "VdVDP", + "name": "Volume game-mods", + "width": "fill_container", + "cornerRadius": 4, + "stroke": "$border/border", + "strokeWidth": 1, + "strokeAlignment": "inner", + "layout": "vertical", + "gap": 4, + "padding": 9, + "children": [ + { + "type": "text", + "id": "Xox0R", + "name": "Claim name", + "fill": "$foreground/foreground", + "textGrowth": "fixed-width", + "width": "fill_container", + "content": "game-mods", + "lineHeight": 1.333333, + "fontFamily": "$typography/font-mono", + "fontSize": 12, + "fontWeight": "normal" + }, + { + "type": "text", + "id": "Iarhd", + "name": "Coverage result", + "fill": "$foreground/foreground", + "textGrowth": "fixed-width", + "width": "fill_container", + "content": "Not covered", + "lineHeight": 1.428571, + "fontFamily": "$typography/font-sans", + "fontSize": 14, + "fontWeight": "normal" + }, + { + "type": "text", + "id": "xtjIY", + "name": "Volume classification", + "fill": "$muted", + "textGrowth": "fixed-width", + "width": "fill_container", + "content": "Unclassified", + "lineHeight": 1.333333, + "fontFamily": "$typography/font-sans", + "fontSize": 12, + "fontWeight": "normal" + }, + { + "type": "text", + "id": "o9BwkI", + "name": "Mount path", + "fill": "$foreground/foreground", + "textGrowth": "fixed-width", + "width": "fill_container", + "content": "/world/mods", + "lineHeight": 1.333333, + "fontFamily": "$typography/font-mono", + "fontSize": 12, + "fontWeight": "normal" + } + ] + } + ] + } + ] + } + ] +} diff --git a/assets/design-export/json/zhLZN.json b/assets/design-export/json/zhLZN.json index a61847465..741ccfd75 100644 --- a/assets/design-export/json/zhLZN.json +++ b/assets/design-export/json/zhLZN.json @@ -22,8 +22,7 @@ "x": -12, "y": 0 }, - "blur": 32, - "spread": -8 + "blur": 32 }, "layout": "vertical", "children": [ @@ -167,6 +166,76 @@ } ] }, + { + "type": "frame", + "id": "T8CP1C", + "name": "Achieved consistency", + "width": "fill_container", + "layout": "vertical", + "gap": 4, + "children": [ + { + "type": "text", + "id": "No2fi", + "name": "Consistency label", + "fill": "$muted", + "content": "ACHIEVED CONSISTENCY", + "lineHeight": 1.333333, + "fontFamily": "$typography/font-sans", + "fontSize": 12, + "fontWeight": "normal", + "letterSpacing": 0.5 + }, + { + "type": "text", + "id": "f04lR", + "name": "Unknown consistency", + "fill": "$foreground/foreground", + "textGrowth": "fixed-width", + "width": "fill_container", + "content": "Consistency has not been reported.", + "lineHeight": 1.428571, + "fontFamily": "$typography/font-sans", + "fontSize": 14, + "fontWeight": "normal" + } + ] + }, + { + "type": "frame", + "id": "W6b7K", + "name": "Volume coverage — Legacy unknown", + "width": "fill_container", + "layout": "vertical", + "gap": 4, + "children": [ + { + "type": "text", + "id": "J3Xx6", + "name": "Coverage label", + "fill": "$muted", + "content": "VOLUME COVERAGE", + "lineHeight": 1.333333, + "fontFamily": "$typography/font-sans", + "fontSize": 12, + "fontWeight": "normal", + "letterSpacing": 0.5 + }, + { + "type": "text", + "id": "YFaQj", + "name": "Unknown capture coverage", + "fill": "$foreground/foreground", + "textGrowth": "fixed-width", + "width": "fill_container", + "content": "Capture-time volume metadata is unavailable. Inspect the original backup and module configuration before recovery.", + "lineHeight": 1.428571, + "fontFamily": "$typography/font-sans", + "fontSize": 14, + "fontWeight": "normal" + } + ] + }, { "type": "frame", "id": "llkFe", diff --git a/assets/design-export/screenshots/bHJy6.png b/assets/design-export/screenshots/bHJy6.png new file mode 100644 index 000000000..4685c56b3 Binary files /dev/null and b/assets/design-export/screenshots/bHJy6.png differ diff --git a/assets/design-export/screenshots/zhLZN.png b/assets/design-export/screenshots/zhLZN.png index eb4d6d22b..c1bdf5403 100644 Binary files a/assets/design-export/screenshots/zhLZN.png and b/assets/design-export/screenshots/zhLZN.png differ diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backups.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backups.yaml index 2df2ff0ca..d62bfcc61 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backups.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backups.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 5bb7d2029f782e9791e29305f35a91883aa3344fdeb81cb4c28969152578109e + gameplane.local/crd-bundle-sha256: 3839b91bcfc726f69c1df7b23e71af742432cb2dd9b955737f3a274ce3ebb809 controller-gen.kubebuilder.io/version: v0.22.0 name: backups.gameplane.local spec: @@ -129,6 +129,75 @@ spec: status: description: BackupStatus is the observed state of a Backup. properties: + captureManifest: + description: |- + CaptureManifest is recorded once, before the first capture worker starts. + It describes persistent-volume coverage independently of copy success. + Missing on pre-upgrade backups; never reconstruct it from current layout. + properties: + layoutSource: + description: LayoutSource identifies the frozen workload used + for mount paths. + type: string + quiesceOutcome: + description: |- + QuiesceOutcome is confirmed, already-stopped, unsupported, not-requested, + agent-unavailable, or unknown, independent of subsequent copy success. + type: string + recordedAt: + format: date-time + type: string + serverUID: + type: string + state: + description: |- + State is Recorded for new captures, LegacyUnknown for an existing worker + whose original layout was never recorded. + enum: + - Recorded + - LegacyUnknown + type: string + strategy: + type: string + volumes: + items: + description: |- + BackupVolumeCoverage describes one persistent claim. Extra and loader/mod + claims remain uncovered even when mounted beneath the primary data directory. + properties: + claimName: + type: string + claimUID: + description: ClaimUID is empty if the claim did not yet + exist at admission. + type: string + classification: + description: |- + Classification records the module/server declaration at admission; + it does not prove that the volume can actually be reconstructed. + enum: + - data + - rebuildable + - unclassified + type: string + included: + type: boolean + mountPaths: + items: + type: string + type: array + reason: + type: string + required: + - claimName + - classification + - included + - reason + type: object + type: array + required: + - state + type: object completionTime: format: date-time type: string diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backupschedules.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backupschedules.yaml index 231cdb10a..91db42032 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backupschedules.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_backupschedules.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 5bb7d2029f782e9791e29305f35a91883aa3344fdeb81cb4c28969152578109e + gameplane.local/crd-bundle-sha256: 3839b91bcfc726f69c1df7b23e71af742432cb2dd9b955737f3a274ce3ebb809 controller-gen.kubebuilder.io/version: v0.22.0 name: backupschedules.gameplane.local spec: diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_clusters.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_clusters.yaml index 42e6b615d..7d5cca499 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_clusters.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_clusters.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 5bb7d2029f782e9791e29305f35a91883aa3344fdeb81cb4c28969152578109e + gameplane.local/crd-bundle-sha256: 3839b91bcfc726f69c1df7b23e71af742432cb2dd9b955737f3a274ce3ebb809 controller-gen.kubebuilder.io/version: v0.22.0 name: clusters.gameplane.local spec: diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gameservers.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gameservers.yaml index 31c1e9b34..aaa82dabb 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gameservers.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gameservers.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 5bb7d2029f782e9791e29305f35a91883aa3344fdeb81cb4c28969152578109e + gameplane.local/crd-bundle-sha256: 3839b91bcfc726f69c1df7b23e71af742432cb2dd9b955737f3a274ce3ebb809 controller-gen.kubebuilder.io/version: v0.22.0 name: gameservers.gameplane.local spec: @@ -2220,6 +2220,16 @@ spec: primary Storage.MountPath, since these exist precisely for directories that share no safe common parent with it. properties: + backupRole: + description: |- + BackupRole describes recovery requirements without adding this volume + to a backup. "data" needs a separate copy; "rebuildable" may be recreated. + Unset remains unclassified and not covered. Both backup strategies still + capture only the primary data claim. + enum: + - data + - rebuildable + type: string mountPath: description: MountPath is the absolute in-container path this volume is mounted at. @@ -2291,6 +2301,27 @@ spec: - message: templateRef is immutable; delete and recreate the GameServer to switch templates rule: self == oldSelf + templateRollout: + description: |- + TemplateRollout controls adoption of changes to the shared GameTemplate. + Omitted preserves Automatic adoption. Per-server overrides remain live. + properties: + policy: + default: Automatic + description: Policy defaults to Automatic. OnRequest requires + the exact desired digest. + enum: + - Automatic + - OnRequest + type: string + requestedRevision: + description: |- + RequestedRevision authorizes this SHA256 only, never future revisions. + Read status.templateRevision.desired and copy it here to admit an update. + Initial provisioning always adopts the installed template. + pattern: ^[a-f0-9]{64}$ + type: string + type: object tolerations: items: description: |- @@ -2626,6 +2657,25 @@ spec: observed as Ready. format: date-time type: string + templateRevision: + description: |- + TemplateRevision is a read model; operator-owned annotations and immutable + ConfigMaps retain admission state across status resets and leader changes. + properties: + desired: + description: Desired is the installed template's SHA256; empty + if it is unavailable. + type: string + effective: + description: Effective is the immutable content used for all derived + resources. + type: string + running: + description: |- + Running comes from an owned Pod with a running game container. Empty + means no running revision is known, including pre-upgrade legacy Pods. + type: string + type: object tunnelEndpoints: description: |- TunnelEndpoints are the per-port addresses reported by the tunnel pod. diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gametemplates.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gametemplates.yaml index 7705c2b02..c039b349b 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gametemplates.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_gametemplates.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 5bb7d2029f782e9791e29305f35a91883aa3344fdeb81cb4c28969152578109e + gameplane.local/crd-bundle-sha256: 3839b91bcfc726f69c1df7b23e71af742432cb2dd9b955737f3a274ce3ebb809 controller-gen.kubebuilder.io/version: v0.22.0 name: gametemplates.gameplane.local spec: @@ -2166,6 +2166,16 @@ spec: primary Storage.MountPath, since these exist precisely for directories that share no safe common parent with it. properties: + backupRole: + description: |- + BackupRole describes recovery requirements without adding this volume + to a backup. "data" needs a separate copy; "rebuildable" may be recreated. + Unset remains unclassified and not covered. Both backup strategies still + capture only the primary data claim. + enum: + - data + - rebuildable + type: string mountPath: description: MountPath is the absolute in-container path this volume is mounted at. diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modules.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modules.yaml index c1c4d0900..9de0b6e6f 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modules.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modules.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 5bb7d2029f782e9791e29305f35a91883aa3344fdeb81cb4c28969152578109e + gameplane.local/crd-bundle-sha256: 3839b91bcfc726f69c1df7b23e71af742432cb2dd9b955737f3a274ce3ebb809 controller-gen.kubebuilder.io/version: v0.22.0 name: modules.gameplane.local spec: diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modulesources.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modulesources.yaml index 5fdf5edff..bfc2cd543 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modulesources.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_modulesources.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 5bb7d2029f782e9791e29305f35a91883aa3344fdeb81cb4c28969152578109e + gameplane.local/crd-bundle-sha256: 3839b91bcfc726f69c1df7b23e71af742432cb2dd9b955737f3a274ce3ebb809 controller-gen.kubebuilder.io/version: v0.22.0 name: modulesources.gameplane.local spec: diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_networkcaptures.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_networkcaptures.yaml index 0dcfffc87..e3c4e16f4 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_networkcaptures.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_networkcaptures.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 5bb7d2029f782e9791e29305f35a91883aa3344fdeb81cb4c28969152578109e + gameplane.local/crd-bundle-sha256: 3839b91bcfc726f69c1df7b23e71af742432cb2dd9b955737f3a274ce3ebb809 controller-gen.kubebuilder.io/version: v0.22.0 name: networkcaptures.gameplane.local spec: diff --git a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_restores.yaml b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_restores.yaml index 92632bf4a..13f62e976 100644 --- a/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_restores.yaml +++ b/charts/gameplane/charts/gameplane-crds/crds/gameplane.local_restores.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 5bb7d2029f782e9791e29305f35a91883aa3344fdeb81cb4c28969152578109e + gameplane.local/crd-bundle-sha256: 3839b91bcfc726f69c1df7b23e71af742432cb2dd9b955737f3a274ce3ebb809 controller-gen.kubebuilder.io/version: v0.22.0 name: restores.gameplane.local spec: diff --git a/charts/gameplane/crd-manifests/gameplane.local_backups.yaml b/charts/gameplane/crd-manifests/gameplane.local_backups.yaml index 2df2ff0ca..d62bfcc61 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_backups.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_backups.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 5bb7d2029f782e9791e29305f35a91883aa3344fdeb81cb4c28969152578109e + gameplane.local/crd-bundle-sha256: 3839b91bcfc726f69c1df7b23e71af742432cb2dd9b955737f3a274ce3ebb809 controller-gen.kubebuilder.io/version: v0.22.0 name: backups.gameplane.local spec: @@ -129,6 +129,75 @@ spec: status: description: BackupStatus is the observed state of a Backup. properties: + captureManifest: + description: |- + CaptureManifest is recorded once, before the first capture worker starts. + It describes persistent-volume coverage independently of copy success. + Missing on pre-upgrade backups; never reconstruct it from current layout. + properties: + layoutSource: + description: LayoutSource identifies the frozen workload used + for mount paths. + type: string + quiesceOutcome: + description: |- + QuiesceOutcome is confirmed, already-stopped, unsupported, not-requested, + agent-unavailable, or unknown, independent of subsequent copy success. + type: string + recordedAt: + format: date-time + type: string + serverUID: + type: string + state: + description: |- + State is Recorded for new captures, LegacyUnknown for an existing worker + whose original layout was never recorded. + enum: + - Recorded + - LegacyUnknown + type: string + strategy: + type: string + volumes: + items: + description: |- + BackupVolumeCoverage describes one persistent claim. Extra and loader/mod + claims remain uncovered even when mounted beneath the primary data directory. + properties: + claimName: + type: string + claimUID: + description: ClaimUID is empty if the claim did not yet + exist at admission. + type: string + classification: + description: |- + Classification records the module/server declaration at admission; + it does not prove that the volume can actually be reconstructed. + enum: + - data + - rebuildable + - unclassified + type: string + included: + type: boolean + mountPaths: + items: + type: string + type: array + reason: + type: string + required: + - claimName + - classification + - included + - reason + type: object + type: array + required: + - state + type: object completionTime: format: date-time type: string diff --git a/charts/gameplane/crd-manifests/gameplane.local_backupschedules.yaml b/charts/gameplane/crd-manifests/gameplane.local_backupschedules.yaml index 231cdb10a..91db42032 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_backupschedules.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_backupschedules.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 5bb7d2029f782e9791e29305f35a91883aa3344fdeb81cb4c28969152578109e + gameplane.local/crd-bundle-sha256: 3839b91bcfc726f69c1df7b23e71af742432cb2dd9b955737f3a274ce3ebb809 controller-gen.kubebuilder.io/version: v0.22.0 name: backupschedules.gameplane.local spec: diff --git a/charts/gameplane/crd-manifests/gameplane.local_clusters.yaml b/charts/gameplane/crd-manifests/gameplane.local_clusters.yaml index 42e6b615d..7d5cca499 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_clusters.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_clusters.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 5bb7d2029f782e9791e29305f35a91883aa3344fdeb81cb4c28969152578109e + gameplane.local/crd-bundle-sha256: 3839b91bcfc726f69c1df7b23e71af742432cb2dd9b955737f3a274ce3ebb809 controller-gen.kubebuilder.io/version: v0.22.0 name: clusters.gameplane.local spec: diff --git a/charts/gameplane/crd-manifests/gameplane.local_gameservers.yaml b/charts/gameplane/crd-manifests/gameplane.local_gameservers.yaml index 31c1e9b34..aaa82dabb 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_gameservers.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_gameservers.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 5bb7d2029f782e9791e29305f35a91883aa3344fdeb81cb4c28969152578109e + gameplane.local/crd-bundle-sha256: 3839b91bcfc726f69c1df7b23e71af742432cb2dd9b955737f3a274ce3ebb809 controller-gen.kubebuilder.io/version: v0.22.0 name: gameservers.gameplane.local spec: @@ -2220,6 +2220,16 @@ spec: primary Storage.MountPath, since these exist precisely for directories that share no safe common parent with it. properties: + backupRole: + description: |- + BackupRole describes recovery requirements without adding this volume + to a backup. "data" needs a separate copy; "rebuildable" may be recreated. + Unset remains unclassified and not covered. Both backup strategies still + capture only the primary data claim. + enum: + - data + - rebuildable + type: string mountPath: description: MountPath is the absolute in-container path this volume is mounted at. @@ -2291,6 +2301,27 @@ spec: - message: templateRef is immutable; delete and recreate the GameServer to switch templates rule: self == oldSelf + templateRollout: + description: |- + TemplateRollout controls adoption of changes to the shared GameTemplate. + Omitted preserves Automatic adoption. Per-server overrides remain live. + properties: + policy: + default: Automatic + description: Policy defaults to Automatic. OnRequest requires + the exact desired digest. + enum: + - Automatic + - OnRequest + type: string + requestedRevision: + description: |- + RequestedRevision authorizes this SHA256 only, never future revisions. + Read status.templateRevision.desired and copy it here to admit an update. + Initial provisioning always adopts the installed template. + pattern: ^[a-f0-9]{64}$ + type: string + type: object tolerations: items: description: |- @@ -2626,6 +2657,25 @@ spec: observed as Ready. format: date-time type: string + templateRevision: + description: |- + TemplateRevision is a read model; operator-owned annotations and immutable + ConfigMaps retain admission state across status resets and leader changes. + properties: + desired: + description: Desired is the installed template's SHA256; empty + if it is unavailable. + type: string + effective: + description: Effective is the immutable content used for all derived + resources. + type: string + running: + description: |- + Running comes from an owned Pod with a running game container. Empty + means no running revision is known, including pre-upgrade legacy Pods. + type: string + type: object tunnelEndpoints: description: |- TunnelEndpoints are the per-port addresses reported by the tunnel pod. diff --git a/charts/gameplane/crd-manifests/gameplane.local_gametemplates.yaml b/charts/gameplane/crd-manifests/gameplane.local_gametemplates.yaml index 7705c2b02..c039b349b 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_gametemplates.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_gametemplates.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 5bb7d2029f782e9791e29305f35a91883aa3344fdeb81cb4c28969152578109e + gameplane.local/crd-bundle-sha256: 3839b91bcfc726f69c1df7b23e71af742432cb2dd9b955737f3a274ce3ebb809 controller-gen.kubebuilder.io/version: v0.22.0 name: gametemplates.gameplane.local spec: @@ -2166,6 +2166,16 @@ spec: primary Storage.MountPath, since these exist precisely for directories that share no safe common parent with it. properties: + backupRole: + description: |- + BackupRole describes recovery requirements without adding this volume + to a backup. "data" needs a separate copy; "rebuildable" may be recreated. + Unset remains unclassified and not covered. Both backup strategies still + capture only the primary data claim. + enum: + - data + - rebuildable + type: string mountPath: description: MountPath is the absolute in-container path this volume is mounted at. diff --git a/charts/gameplane/crd-manifests/gameplane.local_modules.yaml b/charts/gameplane/crd-manifests/gameplane.local_modules.yaml index c1c4d0900..9de0b6e6f 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_modules.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_modules.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 5bb7d2029f782e9791e29305f35a91883aa3344fdeb81cb4c28969152578109e + gameplane.local/crd-bundle-sha256: 3839b91bcfc726f69c1df7b23e71af742432cb2dd9b955737f3a274ce3ebb809 controller-gen.kubebuilder.io/version: v0.22.0 name: modules.gameplane.local spec: diff --git a/charts/gameplane/crd-manifests/gameplane.local_modulesources.yaml b/charts/gameplane/crd-manifests/gameplane.local_modulesources.yaml index 5fdf5edff..bfc2cd543 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_modulesources.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_modulesources.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 5bb7d2029f782e9791e29305f35a91883aa3344fdeb81cb4c28969152578109e + gameplane.local/crd-bundle-sha256: 3839b91bcfc726f69c1df7b23e71af742432cb2dd9b955737f3a274ce3ebb809 controller-gen.kubebuilder.io/version: v0.22.0 name: modulesources.gameplane.local spec: diff --git a/charts/gameplane/crd-manifests/gameplane.local_networkcaptures.yaml b/charts/gameplane/crd-manifests/gameplane.local_networkcaptures.yaml index 0dcfffc87..e3c4e16f4 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_networkcaptures.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_networkcaptures.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 5bb7d2029f782e9791e29305f35a91883aa3344fdeb81cb4c28969152578109e + gameplane.local/crd-bundle-sha256: 3839b91bcfc726f69c1df7b23e71af742432cb2dd9b955737f3a274ce3ebb809 controller-gen.kubebuilder.io/version: v0.22.0 name: networkcaptures.gameplane.local spec: diff --git a/charts/gameplane/crd-manifests/gameplane.local_restores.yaml b/charts/gameplane/crd-manifests/gameplane.local_restores.yaml index 92632bf4a..13f62e976 100644 --- a/charts/gameplane/crd-manifests/gameplane.local_restores.yaml +++ b/charts/gameplane/crd-manifests/gameplane.local_restores.yaml @@ -3,7 +3,7 @@ apiVersion: apiextensions.k8s.io/v1 kind: CustomResourceDefinition metadata: annotations: - gameplane.local/crd-bundle-sha256: 5bb7d2029f782e9791e29305f35a91883aa3344fdeb81cb4c28969152578109e + gameplane.local/crd-bundle-sha256: 3839b91bcfc726f69c1df7b23e71af742432cb2dd9b955737f3a274ce3ebb809 controller-gen.kubebuilder.io/version: v0.22.0 name: restores.gameplane.local spec: diff --git a/charts/gameplane/templates/_helpers.tpl b/charts/gameplane/templates/_helpers.tpl index 2230b4101..44bf31ee3 100644 --- a/charts/gameplane/templates/_helpers.tpl +++ b/charts/gameplane/templates/_helpers.tpl @@ -135,13 +135,6 @@ true true {{- end -}} {{- end -}} -{{- /* Resolve new diagnostics defaults for upgrades using older reuse-values. -mergeOverwrite preserves explicit zero/empty values so validation can reject them. */ -}} -{{- define "gameplane.nodeDiagnostics" -}} -{{- $configured := (.Values.capture | default dict).nodeDiagnostics | default dict -}} -{{- mergeOverwrite (dict "enabled" false "namespace" "gameplane-diagnostics" "maxCaptures" 10 "storageQuota" "10Gi" "storageClass" "") $configured | toJson -}} -{{- end -}} - {{- /* Shared PodSpec placement fields; absent keys preserve older reuse-values. */ -}} {{- define "gameplane.placement" -}} @@ -166,6 +159,13 @@ mergeOverwrite preserves explicit zero/empty values so validation can reject the {{- end -}} {{- end -}} +{{- /* Resolve new diagnostics defaults for upgrades using older reuse-values. +mergeOverwrite preserves explicit zero/empty values so validation can reject them. */ -}} +{{- define "gameplane.nodeDiagnostics" -}} +{{- $configured := (.Values.capture | default dict).nodeDiagnostics | default dict -}} +{{- mergeOverwrite (dict "enabled" false "namespace" "gameplane-diagnostics" "maxCaptures" 10 "storageQuota" "10Gi" "storageClass" "") $configured | toJson -}} +{{- end -}} + {{- /* Validate every namespace before it can reach RBAC or cache flags. */}} {{- define "gameplane.namespace" -}} {{- if or (not (kindIs "string" .)) (gt (len (toString .)) 63) (not (regexMatch "^[a-z0-9]([-a-z0-9]*[a-z0-9])?$" (toString .))) -}} diff --git a/charts/gameplane/templates/api.yaml b/charts/gameplane/templates/api.yaml index adf54a0d7..111983d83 100644 --- a/charts/gameplane/templates/api.yaml +++ b/charts/gameplane/templates/api.yaml @@ -96,6 +96,11 @@ rules: - apiGroups: ["snapshot.storage.k8s.io"] resources: [volumesnapshots] verbs: [get] + # Read owned, immutable template snapshots for pinned-server password + # redaction. These lookups never list or modify game namespace ConfigMaps. + - apiGroups: [""] + resources: [configmaps] + verbs: [get] # Verify GameServer -> StatefulSet -> Pod controller ownership before # serving pod logs or accepting terminal input. - apiGroups: ["apps"] diff --git a/charts/tests/check_standalone.py b/charts/tests/check_standalone.py index e30ea1211..c8214eb76 100644 --- a/charts/tests/check_standalone.py +++ b/charts/tests/check_standalone.py @@ -84,6 +84,11 @@ def main(): one(defaults, "ModuleSource", "default") api_rules = one(defaults, "ClusterRole", "gameplane-api-read")["rules"] assert any(rule.get("resources") == ["clusters"] and "update" in rule["verbs"] for rule in api_rules) + assert not any("configmaps" in rule.get("resources", []) for rule in api_rules) + games_role = one(defaults, "Role", "gameplane-api-games") + assert games_role["metadata"]["namespace"] == "gameplane-games" + snapshot_rules = [rule for rule in games_role["rules"] if "configmaps" in rule.get("resources", [])] + assert snapshot_rules == [{"apiGroups": [""], "resources": ["configmaps"], "verbs": ["get"]}] assert "--standalone" not in one(defaults, "Deployment", "gameplane-api")["spec"]["template"]["spec"]["containers"][0]["args"] values = { diff --git a/design.pen b/design.pen index 60dcac7fa..7e548c03d 100644 --- a/design.pen +++ b/design.pen @@ -119381,6 +119381,76 @@ } ] }, + { + "type": "frame", + "id": "T8CP1C", + "name": "Achieved consistency", + "width": "fill_container", + "layout": "vertical", + "gap": 4, + "children": [ + { + "type": "text", + "id": "No2fi", + "name": "Consistency label", + "fill": "$muted", + "content": "ACHIEVED CONSISTENCY", + "lineHeight": 1.333333, + "fontFamily": "$typography/font-sans", + "fontSize": 12, + "fontWeight": "normal", + "letterSpacing": 0.5 + }, + { + "type": "text", + "id": "f04lR", + "name": "Unknown consistency", + "fill": "$foreground/foreground", + "textGrowth": "fixed-width", + "width": "fill_container", + "content": "Consistency has not been reported.", + "lineHeight": 1.428571, + "fontFamily": "$typography/font-sans", + "fontSize": 14, + "fontWeight": "normal" + } + ] + }, + { + "type": "frame", + "id": "W6b7K", + "name": "Volume coverage — Legacy unknown", + "width": "fill_container", + "layout": "vertical", + "gap": 4, + "children": [ + { + "type": "text", + "id": "J3Xx6", + "name": "Coverage label", + "fill": "$muted", + "content": "VOLUME COVERAGE", + "lineHeight": 1.333333, + "fontFamily": "$typography/font-sans", + "fontSize": 12, + "fontWeight": "normal", + "letterSpacing": 0.5 + }, + { + "type": "text", + "id": "YFaQj", + "name": "Unknown capture coverage", + "fill": "$foreground/foreground", + "textGrowth": "fixed-width", + "width": "fill_container", + "content": "Capture-time volume metadata is unavailable. Inspect the original backup and module configuration before recovery.", + "lineHeight": 1.428571, + "fontFamily": "$typography/font-sans", + "fontSize": 14, + "fontWeight": "normal" + } + ] + }, { "type": "frame", "id": "llkFe", @@ -127393,6 +127463,279 @@ } } ] + }, + { + "type": "frame", + "id": "bHJy6", + "x": 2276, + "y": 4461, + "name": "Gameplane/Backup Volume Coverage — Recorded", + "reusable": true, + "width": 396, + "fill": "$surface/surface", + "layout": "vertical", + "gap": 4, + "children": [ + { + "type": "text", + "id": "r8Owu", + "name": "Coverage label", + "fill": "$muted", + "content": "VOLUME COVERAGE", + "lineHeight": 1.333333, + "fontFamily": "$typography/font-sans", + "fontSize": 12, + "fontWeight": "normal", + "letterSpacing": 0.5 + }, + { + "type": "frame", + "id": "mfHN5", + "name": "Recorded capture manifest", + "width": "fill_container", + "layout": "vertical", + "gap": 12, + "children": [ + { + "type": "text", + "id": "Z5aM17", + "name": "Capture recorded at", + "fill": "$muted", + "textGrowth": "fixed-width", + "width": "fill_container", + "content": "Recorded: 2026-07-13T00:12:04Z", + "lineHeight": 1.333333, + "fontFamily": "$typography/font-sans", + "fontSize": 12, + "fontWeight": "normal" + }, + { + "type": "text", + "id": "etbdz", + "name": "Coverage scope", + "fill": "$muted", + "textGrowth": "fixed-width", + "width": "fill_container", + "content": "Only the primary data claim is copied. Rebuildable is a declaration, not proof that recovery can reconstruct its contents.", + "lineHeight": 1.333333, + "fontFamily": "$typography/font-sans", + "fontSize": 12, + "fontWeight": "normal" + }, + { + "type": "frame", + "id": "zLDqu", + "name": "Captured volume inventory", + "width": "fill_container", + "layout": "vertical", + "gap": 12, + "children": [ + { + "type": "frame", + "id": "jgV6n", + "name": "Volume game-data", + "width": "fill_container", + "cornerRadius": 4, + "stroke": "$border/border", + "strokeWidth": 1, + "strokeAlignment": "inner", + "layout": "vertical", + "gap": 4, + "padding": 9, + "children": [ + { + "type": "text", + "id": "F0Qiz", + "name": "Claim name", + "fill": "$foreground/foreground", + "textGrowth": "fixed-width", + "width": "fill_container", + "content": "game-data", + "lineHeight": 1.333333, + "fontFamily": "$typography/font-mono", + "fontSize": 12, + "fontWeight": "normal" + }, + { + "type": "text", + "id": "b4aDFu", + "name": "Coverage result", + "fill": "$foreground/foreground", + "textGrowth": "fixed-width", + "width": "fill_container", + "content": "Included", + "lineHeight": 1.428571, + "fontFamily": "$typography/font-sans", + "fontSize": 14, + "fontWeight": "normal" + }, + { + "type": "text", + "id": "pq4ts", + "name": "Volume classification", + "fill": "$muted", + "textGrowth": "fixed-width", + "width": "fill_container", + "content": "Data", + "lineHeight": 1.333333, + "fontFamily": "$typography/font-sans", + "fontSize": 12, + "fontWeight": "normal" + }, + { + "type": "text", + "id": "fM4kj", + "name": "Mount path", + "fill": "$foreground/foreground", + "textGrowth": "fixed-width", + "width": "fill_container", + "content": "/world", + "lineHeight": 1.333333, + "fontFamily": "$typography/font-mono", + "fontSize": 12, + "fontWeight": "normal" + } + ] + }, + { + "type": "frame", + "id": "DTQiV", + "name": "Volume game-extra-install", + "width": "fill_container", + "cornerRadius": 4, + "stroke": "$border/border", + "strokeWidth": 1, + "strokeAlignment": "inner", + "layout": "vertical", + "gap": 4, + "padding": 9, + "children": [ + { + "type": "text", + "id": "MiYbU", + "name": "Claim name", + "fill": "$foreground/foreground", + "textGrowth": "fixed-width", + "width": "fill_container", + "content": "game-extra-install", + "lineHeight": 1.333333, + "fontFamily": "$typography/font-mono", + "fontSize": 12, + "fontWeight": "normal" + }, + { + "type": "text", + "id": "GhJX5", + "name": "Coverage result", + "fill": "$foreground/foreground", + "textGrowth": "fixed-width", + "width": "fill_container", + "content": "Not covered", + "lineHeight": 1.428571, + "fontFamily": "$typography/font-sans", + "fontSize": 14, + "fontWeight": "normal" + }, + { + "type": "text", + "id": "ektuq", + "name": "Volume classification", + "fill": "$muted", + "textGrowth": "fixed-width", + "width": "fill_container", + "content": "Rebuildable", + "lineHeight": 1.333333, + "fontFamily": "$typography/font-sans", + "fontSize": 12, + "fontWeight": "normal" + }, + { + "type": "text", + "id": "HGTOq", + "name": "Mount path", + "fill": "$foreground/foreground", + "textGrowth": "fixed-width", + "width": "fill_container", + "content": "/serverfiles", + "lineHeight": 1.333333, + "fontFamily": "$typography/font-mono", + "fontSize": 12, + "fontWeight": "normal" + } + ] + }, + { + "type": "frame", + "id": "VdVDP", + "name": "Volume game-mods", + "width": "fill_container", + "cornerRadius": 4, + "stroke": "$border/border", + "strokeWidth": 1, + "strokeAlignment": "inner", + "layout": "vertical", + "gap": 4, + "padding": 9, + "children": [ + { + "type": "text", + "id": "Xox0R", + "name": "Claim name", + "fill": "$foreground/foreground", + "textGrowth": "fixed-width", + "width": "fill_container", + "content": "game-mods", + "lineHeight": 1.333333, + "fontFamily": "$typography/font-mono", + "fontSize": 12, + "fontWeight": "normal" + }, + { + "type": "text", + "id": "Iarhd", + "name": "Coverage result", + "fill": "$foreground/foreground", + "textGrowth": "fixed-width", + "width": "fill_container", + "content": "Not covered", + "lineHeight": 1.428571, + "fontFamily": "$typography/font-sans", + "fontSize": 14, + "fontWeight": "normal" + }, + { + "type": "text", + "id": "xtjIY", + "name": "Volume classification", + "fill": "$muted", + "textGrowth": "fixed-width", + "width": "fill_container", + "content": "Unclassified", + "lineHeight": 1.333333, + "fontFamily": "$typography/font-sans", + "fontSize": 12, + "fontWeight": "normal" + }, + { + "type": "text", + "id": "o9BwkI", + "name": "Mount path", + "fill": "$foreground/foreground", + "textGrowth": "fixed-width", + "width": "fill_container", + "content": "/world/mods", + "lineHeight": 1.333333, + "fontFamily": "$typography/font-mono", + "fontSize": 12, + "fontWeight": "normal" + } + ] + } + ] + } + ] + } + ] } ] }, diff --git a/docs/install.md b/docs/install.md index 4ec438c77..6ef2122b3 100644 --- a/docs/install.md +++ b/docs/install.md @@ -684,6 +684,14 @@ Before registering a target cluster, ensure it has: - Gameplane operator and agent images accessible (same registry as the control-plane) - A valid kubeconfig scoped to the intended Gameplane operations and namespaces +Servers pinned to an admitted template revision also require `get` on core +`configmaps` in each allowed game namespace. The API verifies the snapshot's +ownership, immutability and content before using its password schema. If a +snapshot is missing, invalid or inaccessible, all non-empty config values are +redacted; the API never substitutes the latest catalog schema for a pin. The +chart grants this read permission to the local API. Update custom remote +kubeconfigs and extra namespace Roles with the same permission. + ### Pod logs and PTY console permissions The central API must reach the target Kubernetes API (including streaming/SPDY diff --git a/docs/module-authoring.md b/docs/module-authoring.md index 1141965bb..a0553c207 100644 --- a/docs/module-authoring.md +++ b/docs/module-authoring.md @@ -375,6 +375,13 @@ re-pinning `spec.version`. ### Verifying and pinning bundles +Pin production `Module.spec.version` to avoid following catalog updates across +the fleet. Servers default to automatic template adoption; opt into +`GameServer.spec.templateRollout.policy: OnRequest` before installing a new +version to admit each server separately. See [template adoption and rollback](template-rollouts.md) +for baseline upgrade behavior, exact revision requests, retained PVCs and the +difference between reverting a runtime and restoring application data. + Two independent, opt-in controls harden the supply chain for OCI modules: **Digest pin** (`Module.spec.digest`) refuses to install unless the resolved @@ -1636,7 +1643,35 @@ useful, so readiness falls back to TCP on the game port. `spec.storage.mountPath` is where the PVC is mounted inside the game container. Pick the path the game writes its world/config files to — -backups snapshot this whole directory. +backups copy the primary claim mounted here. Separate PVCs mounted beneath +that directory are not included. + +Extra persistent volumes can declare their recovery role: + +```yaml +spec: + storage: + extra: + - name: serverfiles + mountPath: /serverfiles + size: 10Gi + backupRole: rebuildable # or data +``` + +`backupRole: data` identifies recovery data; `rebuildable` declares that the +contents can be downloaded or reconstructed. Neither includes the extra claim +in today's single-volume backup. Omission remains valid and appears as +unclassified/not covered. Do not mark an installation volume rebuildable if it +also contains unique configuration or mods that cannot be reconstructed. +The GameServer's existing `spec.storage.extra` override can supply the same field. + +The Backup pins its volume list, claim UIDs, observed game-container mount paths +and these declarations before copying. Template changes do not rewrite that +record. Nested loader/mod claims and retained server-owned claims remain +explicitly uncovered. See [backup coverage](operator-operations.md#backup-consistency-and-volume-coverage). +The module editor schema is derived from the GameTemplate CRD by +`make module-schema`; publish its regenerated copy in the module repository +when opting modules into this field. ## Testing a new module locally diff --git a/docs/operator-operations.md b/docs/operator-operations.md index 1ea61c7fe..24ef33168 100644 --- a/docs/operator-operations.md +++ b/docs/operator-operations.md @@ -1,5 +1,42 @@ # Backup, restore and wipe ownership +## Backup consistency and volume coverage + +A successful copy is not necessarily a complete server recovery point. +`status.conditions[type=Consistency]` reports the achieved result separately +from `Completed`: Quiesced means the agent confirmed the module's quiesce +commands before the completed copy; it does not prove those commands flush all +application state. UnsupportedRawCopy, RawCopy and AgentUnavailableRawCopy +describe restic copies without confirmed quiesce. CSI without confirmed +quiesce reports CrashConsistentSnapshot. StoppedSource means the source was +already stopped and its writers drained before capture; the backup did not stop +it. Failed or unfinished copies, including completed Jobs still awaiting a +snapshot ID, cannot report achieved quiesce or stopped-source consistency. + +`status.captureManifest` is recorded once before the first copy worker is +created, after lifecycle ownership and quiesce preparation. It lists the primary +`-data` claim as included and every other observed persistent claim as +uncovered, including extra volumes, nested mod/loader mounts and retained +server-owned claims. It records claim UIDs, observed mount paths, source layout +and optional `data`/`rebuildable` declarations. Included describes copy scope; +check the Backup's phase and snapshot ID for copy success. A declaration that +an uncovered claim is rebuildable is not a recovery guarantee. + +The API returns this status and the backup details drawer displays consistency +and coverage. Capture metadata is immutable through ordinary reconciliation: +editing a template later never changes the historical list. This release does +not add multi-volume restic, automatic game stopping or a stricter consistency +request policy. + +Pre-upgrade captures have missing metadata or `state: LegacyUnknown`; their +Consistency condition is Unknown. The operator never reconstructs their history +from the current template. Before recovery, inspect the original restic snapshot +or CSI source claim, recover the original module/server configuration and arrange +separate recovery of extra/mod volumes. Use an isolated recovery server to +verify those prerequisites before replacing a live world. + +## Shared operation ownership + Gameplane serializes these operations per GameServer. All backups participate, including backups with `quiesce: false` and CSI VolumeSnapshots. A held operation also defers idle sleep/wake, restart, and pod-template rollout. Requests remain diff --git a/docs/template-rollouts.md b/docs/template-rollouts.md new file mode 100644 index 000000000..67e537590 --- /dev/null +++ b/docs/template-rollouts.md @@ -0,0 +1,108 @@ +# Adopting template updates + +Installing a Module updates its shared GameTemplate. Each GameServer now retains +an immutable copy of the template content it has admitted. The default +`Automatic` policy adopts installed changes as before, with lifecycle exclusion +and a graceful stop before replacement. To choose the time for a server, set: + +```yaml +spec: + templateRollout: + policy: OnRequest +``` + +Enable this before updating the Module. A new server always provisions from the +currently installed template. Existing servers without a recorded snapshot get +a baseline snapshot without a workload restart. The operator preserves their +existing Pod template, including API defaults and admission additions, until +workload inputs change or the workload has stopped and drained. Their `running` +revision stays unknown until a replacement Pod actually carries a revision. + +This baseline exception does not defer actual security changes: installing +agent-only credential projection or enabling the Restricted profile still +updates the workload. New template revisions must satisfy the enabled security +profile before admission and before the old workload is drained. + +The baseline records currently installed content; it cannot recover historical +template contents from an old Pod or prove that the old Pod used that content. +Pin `Module.spec.version` before upgrading and enable staging before a later +catalog change. A pre-upgrade backup, restore or wipe already in progress finishes +before baseline initialization. A new server gets its pin before first startup. + +Inspect the three digests: + +```sh +kubectl get gameserver my-server -n gameplane-games \ + -o jsonpath='{.status.templateRevision}' +``` + +- `desired` identifies the installed template contents. It is empty when the + installed template is unavailable. +- `effective` identifies the immutable template used for images, versions, + config files and init containers, storage, ports, probes, agent configuration + and other derived resources. During baseline initialization, the existing + unknown Pod template is retained until a real workload change or stopped drain. +- `running` is observed on an owned Pod with a running game container. It is + empty while stopped, while the game container is not running, or when the + pre-upgrade Pod has no revision annotation. It is not a health assertion. + +`TemplateUpdatePending=True` explains whether a revision is available, waiting +for another operation, or being adopted. `Unknown/TemplateUnavailable` preserves +an existing effective snapshot if the installed template disappears. A missing, +mutable, corrupt or foreign effective snapshot fails closed and requires repair; +it never silently switches the server to the latest catalog content. + +To admit an update, copy the exact 64-character `desired` digest into the request: + +```sh +revision=$(kubectl get gameserver my-server -n gameplane-games \ + -o jsonpath='{.status.templateRevision.desired}') +kubectl patch gameserver my-server -n gameplane-games --type=merge \ + -p "{\"spec\":{\"templateRollout\":{\"policy\":\"OnRequest\",\"requestedRevision\":\"$revision\"}}}" +``` + +An old request never approves a newer digest. Requests can be made one server at +a time or by an administrator's batch process. There is no built-in canary, +maintenance-window or fleet concurrency scheduler. Once admitted, an update +finishes using that exact snapshot even if another catalog update arrives or +the policy changes during shutdown. + +Admission uses the same durable owner as backups, restores and wipes. It waits +for their cleanup, runs the previous effective template's graceful stop contract, +waits for the old Pod to disappear, then switches the effective pin and creates +the replacement. Ownership is released when the new Pod is observed, or when a +suspended workload is confirmed stopped with the new template. A bad image may +still fail startup; check ordinary readiness and health conditions separately. +Suspend and idle intent remain in effect. User changes to per-server overrides, +including `spec.version`, `spec.image` and `spec.config`, remain immediate and are +not controlled by the shared-template adoption policy. + +Snapshots are immutable, GameServer-owned ConfigMaps, bounded to 900 KiB each. +The current and admitted revisions are retained during adoption; obsolete copies +are removed afterwards. Do not edit or delete them or the operator-owned revision +and operation annotations. A snapshot pins template contents, including image +references; use immutable image digests to prevent a mutable image tag changing +its registry content independently. + +## Pinning and rollback + +Set `Module.spec.version` to a known version to prevent tracking the catalog's +latest entry, including on installations that do not yet support staged adoption. +Use `Module.spec.digest` as well when the source supports OCI digest verification; +version tags alone may move. See [module installation](module-authoring.md#installing-a-module). + +Adoption retains existing primary, extra and mod PVC identities and their data; +it does not recreate them to apply new template storage defaults. Removed extra +or mod volumes remain retained. Deleting the GameServer is different: its owned +claims can be garbage-collected. Keep an independently verified backup before +changing a game's runtime or save format. + +Reinstalling an older Module version produces a desired template that can be +admitted with the same process. Reverting a runtime does not reverse save-format +changes. Restoring data may be necessary, and compatibility is game-specific. +Backups remain data backups, not automatic runtime rollback bundles. Record the +effective digest, immutable image reference and server configuration with your +backup procedure. Restic restores use the target's current effective layout; CSI +restores copy the source server's effective template at restore planning time, +not necessarily its template at backup time. Select a compatible Module version +and `OnRequest` policy before recovery if automatic adoption would be unsafe. diff --git a/hack/test_ci_scope.py b/hack/test_ci_scope.py index aa042c067..3436a0208 100644 --- a/hack/test_ci_scope.py +++ b/hack/test_ci_scope.py @@ -420,14 +420,19 @@ def test_audit_restricted_workloads_has_a_run_pattern(self): def test_audit_management_placement_has_a_run_pattern(self): """Affected components and the owning test file select the audit bucket.""" - for path in ("operator/cmd/main.go", - "api/cmd/main.go", - "agent/cmd/main.go", + for path in ("operator/cmd/main.go", "api/cmd/main.go", "agent/cmd/main.go", "test/e2e/management_placement_e2e_test.go"): with self.subTest(path=path): - values = self.select(path) self.assertIn("TestManagementPlacementAndDisruption", - values["e2e-run"].get("management-placement", "")) + self.select(path)["e2e-run"].get("management-placement", "")) + + def test_audit_staged_rollout_keeps_operator_coverage(self): + """Template adoption and its owning fixture select the rollout scenario.""" + for path in ("operator/internal/controller/gameserver_template_revision.go", + "test/e2e/template_rollout_e2e_test.go"): + with self.subTest(path=path): + self.assertIn("TestGameServer_TemplateOnRequestRetainsData", + self.select(path)["e2e-run"].get("operator", "")) def test_agent_only_change_selects_agent_tests_and_skips_auth_and_telemetry(self): values = self.select("agent/internal/console/console.go") diff --git a/i18n/locales/en/app.json b/i18n/locales/en/app.json index 5905b54fe..d192ef019 100644 --- a/i18n/locales/en/app.json +++ b/i18n/locales/en/app.json @@ -175,6 +175,42 @@ "defaultMessage": "The backup or restore completed successfully.", "description": "Completed condition on a Backup or Restore that finished." }, + "condition.consistency.agentunavailablerawcopy": { + "defaultMessage": "No agent quiesce capability was configured. The primary volume was copied without pausing game writes.", + "description": "Backup achieved consistency: AgentUnavailableRawCopy." + }, + "condition.consistency.capturefailed": { + "defaultMessage": "The capture failed; no usable consistency guarantee was achieved.", + "description": "Backup achieved consistency: CaptureFailed." + }, + "condition.consistency.capturepending": { + "defaultMessage": "The capture has not completed; its achieved consistency is not yet known.", + "description": "Backup achieved consistency: CapturePending." + }, + "condition.consistency.crashconsistentsnapshot": { + "defaultMessage": "The primary CSI snapshot is crash-consistent; application quiesce was not confirmed.", + "description": "Backup achieved consistency: CrashConsistentSnapshot." + }, + "condition.consistency.legacymetadataunknown": { + "defaultMessage": "Capture-time metadata is unavailable. Do not infer consistency or volume coverage from the current template.", + "description": "Backup achieved consistency: LegacyMetadataUnknown." + }, + "condition.consistency.quiesced": { + "defaultMessage": "The primary volume copy completed after the agent confirmed quiesce. This depends on the module's quiesce commands; other volumes remain outside this backup.", + "description": "Backup achieved consistency: Quiesced." + }, + "condition.consistency.rawcopy": { + "defaultMessage": "The primary volume was copied without confirmed application quiesce.", + "description": "Backup achieved consistency: RawCopy." + }, + "condition.consistency.stoppedsource": { + "defaultMessage": "The primary volume copy completed from a source that was already stopped and drained before capture. Other volumes remain outside this backup.", + "description": "Backup achieved consistency: StoppedSource." + }, + "condition.consistency.unsupportedrawcopy": { + "defaultMessage": "The game does not support quiesce. The primary volume was copied while it could still change.", + "description": "Backup achieved consistency: UnsupportedRawCopy." + }, "condition.datawipe.jobfailed": { "defaultMessage": "The data wipe job did not complete. The volume may not be fully cleared.", "description": "DataWipe condition when the wipe job fails." @@ -559,6 +595,26 @@ "defaultMessage": "Indexing the module source failed.", "description": "Synced condition after a failed index." }, + "condition.templateupdatepending.adoptingrevision": { + "defaultMessage": "The admitted template revision is being adopted.", + "description": "Template rollout owns the server lifecycle while draining and replacing its workload." + }, + "condition.templateupdatepending.operationinprogress": { + "defaultMessage": "Template adoption waits for the active lifecycle operation to finish.", + "description": "A backup, restore or wipe prevents admitting an available template revision." + }, + "condition.templateupdatepending.revisionavailable": { + "defaultMessage": "An installed template revision is waiting for admission.", + "description": "An installed template differs from the server's immutable effective revision." + }, + "condition.templateupdatepending.revisioncurrent": { + "defaultMessage": "The installed template revision is effective.", + "description": "No newer template revision is waiting for adoption. Running revision is observed separately." + }, + "condition.templateupdatepending.templateunavailable": { + "defaultMessage": "The installed template is unavailable; the effective snapshot is retained.", + "description": "Template availability is unknown but the server retains its immutable effective content." + }, "condition.tunneladdressinvalid.validationfailed": { "defaultMessage": "Some tunnel endpoints were rejected: {details}", "description": "Condition when endpoints reported by the playit tunnel pod fail validation. details is the English validation text." @@ -1151,10 +1207,34 @@ "defaultMessage": "Backup details", "description": "Backup details drawer: heading at the top of the side panel for one backup; the backup name is shown below it" }, + "web.components.backups.backup_detail_drawer.classification": { + "defaultMessage": "{role, select, data {Data} rebuildable {Rebuildable} other {Unclassified}}", + "description": "Backup volume: optional declaration from the module or server at capture time" + }, "web.components.backups.backup_detail_drawer.completed": { "defaultMessage": "Completed", "description": "Backup details drawer: field label above the time the backup completed, shown in relative time" }, + "web.components.backups.backup_detail_drawer.consistency": { + "defaultMessage": "Achieved consistency", + "description": "Backup details: actual consistency reported by the controller, independent of successful copying" + }, + "web.components.backups.backup_detail_drawer.consistency_unknown": { + "defaultMessage": "Consistency has not been reported.", + "description": "Backup details: no achieved-consistency condition is available" + }, + "web.components.backups.backup_detail_drawer.coverage": { + "defaultMessage": "Volume coverage", + "description": "Backup details: persistent claims recorded before the backup copy began" + }, + "web.components.backups.backup_detail_drawer.coverage_recorded": { + "defaultMessage": "Recorded: {time}", + "description": "Backup details: timestamp of the immutable capture manifest" + }, + "web.components.backups.backup_detail_drawer.coverage_scope": { + "defaultMessage": "Only the primary data claim is copied. Rebuildable is a declaration, not proof that recovery can reconstruct its contents.", + "description": "Backup details: explains the current single-volume backup scope and rebuildable declarations" + }, "web.components.backups.backup_detail_drawer.delete": { "defaultMessage": "Delete", "description": "Backup detail drawer: delete button label" @@ -1163,6 +1243,18 @@ "defaultMessage": "Deleting…", "description": "Backup detail drawer: delete button label while the delete request is pending" }, + "web.components.backups.backup_detail_drawer.included": { + "defaultMessage": "Included", + "description": "Backup volume: this claim is selected for copying; success is reported separately" + }, + "web.components.backups.backup_detail_drawer.legacy_coverage": { + "defaultMessage": "Capture-time volume metadata is unavailable. Inspect the original backup and module configuration before recovery.", + "description": "Backup details: older backups have no reliable volume manifest, so the current template is not used to invent coverage" + }, + "web.components.backups.backup_detail_drawer.not_covered": { + "defaultMessage": "Not covered", + "description": "Backup volume: contents of this claim are not included" + }, "web.components.backups.backup_detail_drawer.phase": { "defaultMessage": "Phase", "description": "Backup details drawer: field label above the backup's current phase, shown as a status chip" @@ -1175,6 +1267,10 @@ "defaultMessage": "Restore backup", "description": "Backup details drawer: screen-reader name of the restore button in the drawer header, read instead of its visible word Restore; not visible on screen" }, + "web.components.backups.backup_detail_drawer.retained_unmounted": { + "defaultMessage": "Retained claim; not mounted in the captured workload.", + "description": "Backup volume: an older server-owned claim was retained but was not part of the observed pod layout" + }, "web.components.backups.backup_detail_drawer.server": { "defaultMessage": "Server", "description": "Backup details drawer: field label above the name of the server the backup belongs to" diff --git a/operator/api/v1alpha1/backup_types.go b/operator/api/v1alpha1/backup_types.go index 0caeed81a..c2ec70c00 100644 --- a/operator/api/v1alpha1/backup_types.go +++ b/operator/api/v1alpha1/backup_types.go @@ -88,6 +88,12 @@ type LocalObjectRef struct { // BackupStatus is the observed state of a Backup. type BackupStatus struct { + // CaptureManifest is recorded once, before the first capture worker starts. + // It describes persistent-volume coverage independently of copy success. + // Missing on pre-upgrade backups; never reconstruct it from current layout. + // +optional + CaptureManifest *BackupCaptureManifest `json:"captureManifest,omitempty"` + // +optional Phase BackupPhase `json:"phase,omitempty"` @@ -128,6 +134,47 @@ type BackupStatus struct { Conditions []metav1.Condition `json:"conditions,omitempty" patchStrategy:"merge" patchMergeKey:"type"` } +// BackupCaptureManifest pins the layout and quiesce result at capture admission. +// Recorded coverage does not imply a successful or complete server backup. +type BackupCaptureManifest struct { + // State is Recorded for new captures, LegacyUnknown for an existing worker + // whose original layout was never recorded. + // +kubebuilder:validation:Enum=Recorded;LegacyUnknown + State string `json:"state"` + // +optional + RecordedAt *metav1.Time `json:"recordedAt,omitempty"` + // +optional + ServerUID string `json:"serverUID,omitempty"` + // +optional + Strategy string `json:"strategy,omitempty"` + // LayoutSource identifies the frozen workload used for mount paths. + // +optional + LayoutSource string `json:"layoutSource,omitempty"` + // QuiesceOutcome is confirmed, already-stopped, unsupported, not-requested, + // agent-unavailable, or unknown, independent of subsequent copy success. + // +optional + QuiesceOutcome string `json:"quiesceOutcome,omitempty"` + // +optional + Volumes []BackupVolumeCoverage `json:"volumes,omitempty"` +} + +// BackupVolumeCoverage describes one persistent claim. Extra and loader/mod +// claims remain uncovered even when mounted beneath the primary data directory. +type BackupVolumeCoverage struct { + ClaimName string `json:"claimName"` + // ClaimUID is empty if the claim did not yet exist at admission. + // +optional + ClaimUID string `json:"claimUID,omitempty"` + // +optional + MountPaths []string `json:"mountPaths,omitempty"` + Included bool `json:"included"` + // Classification records the module/server declaration at admission; + // it does not prove that the volume can actually be reconstructed. + // +kubebuilder:validation:Enum=data;rebuildable;unclassified + Classification string `json:"classification"` + CoverageReason string `json:"reason"` +} + // +kubebuilder:object:root=true // +kubebuilder:resource:shortName=bk // +kubebuilder:printcolumn:name="Server",type=string,JSONPath=`.spec.serverRef.name` diff --git a/operator/api/v1alpha1/gameserver_types.go b/operator/api/v1alpha1/gameserver_types.go index 2b4c553f1..e6377f3cb 100644 --- a/operator/api/v1alpha1/gameserver_types.go +++ b/operator/api/v1alpha1/gameserver_types.go @@ -49,6 +49,11 @@ type GameServerSpec struct { // +kubebuilder:validation:XValidation:rule="self == oldSelf",message="templateRef is immutable; delete and recreate the GameServer to switch templates" TemplateRef GameTemplateRef `json:"templateRef"` + // TemplateRollout controls adoption of changes to the shared GameTemplate. + // Omitted preserves Automatic adoption. Per-server overrides remain live. + // +optional + TemplateRollout *TemplateRolloutSpec `json:"templateRollout,omitempty"` + // Suspend, when true, scales the underlying StatefulSet to zero. // Data is preserved. Transitions the GameServer to Suspended. // +kubebuilder:default=false @@ -513,8 +518,41 @@ type InlineBackupPolicy struct { Suspend bool `json:"suspend,omitempty"` } +// TemplateRolloutSpec controls when installed template content is adopted. +type TemplateRolloutSpec struct { + // Policy defaults to Automatic. OnRequest requires the exact desired digest. + // +kubebuilder:validation:Enum=Automatic;OnRequest + // +kubebuilder:default=Automatic + // +optional + Policy string `json:"policy,omitempty"` + // RequestedRevision authorizes this SHA256 only, never future revisions. + // Read status.templateRevision.desired and copy it here to admit an update. + // Initial provisioning always adopts the installed template. + // +kubebuilder:validation:Pattern=`^[a-f0-9]{64}$` + // +optional + RequestedRevision string `json:"requestedRevision,omitempty"` +} + +// TemplateRevisionStatus distinguishes installed, admitted and observed content. +type TemplateRevisionStatus struct { + // Desired is the installed template's SHA256; empty if it is unavailable. + // +optional + Desired string `json:"desired,omitempty"` + // Effective is the immutable content used for all derived resources. + // +optional + Effective string `json:"effective,omitempty"` + // Running comes from an owned Pod with a running game container. Empty + // means no running revision is known, including pre-upgrade legacy Pods. + // +optional + Running string `json:"running,omitempty"` +} + // GameServerStatus is the observed state of a GameServer. type GameServerStatus struct { + // TemplateRevision is a read model; operator-owned annotations and immutable + // ConfigMaps retain admission state across status resets and leader changes. + // +optional + TemplateRevision *TemplateRevisionStatus `json:"templateRevision,omitempty"` // +optional Phase GameServerPhase `json:"phase,omitempty"` diff --git a/operator/api/v1alpha1/gametemplate_types.go b/operator/api/v1alpha1/gametemplate_types.go index bea234aea..8001a2b27 100644 --- a/operator/api/v1alpha1/gametemplate_types.go +++ b/operator/api/v1alpha1/gametemplate_types.go @@ -946,6 +946,14 @@ type GameStorageSpec struct { // primary Storage.MountPath, since these exist precisely for directories // that share no safe common parent with it. type ExtraVolumeSpec struct { + // BackupRole describes recovery requirements without adding this volume + // to a backup. "data" needs a separate copy; "rebuildable" may be recreated. + // Unset remains unclassified and not covered. Both backup strategies still + // capture only the primary data claim. + // +kubebuilder:validation:Enum=data;rebuildable + // +optional + BackupRole string `json:"backupRole,omitempty"` + // Name identifies the volume and forms part of its PVC name. Must be a // DNS-1123 label. // +kubebuilder:validation:Pattern=`^[a-z0-9]([-a-z0-9]*[a-z0-9])?$` diff --git a/operator/api/v1alpha1/zz_generated.deepcopy.go b/operator/api/v1alpha1/zz_generated.deepcopy.go index 05926fd58..85b8d51a5 100644 --- a/operator/api/v1alpha1/zz_generated.deepcopy.go +++ b/operator/api/v1alpha1/zz_generated.deepcopy.go @@ -178,6 +178,32 @@ func (in *Backup) DeepCopyObject() runtime.Object { return nil } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BackupCaptureManifest) DeepCopyInto(out *BackupCaptureManifest) { + *out = *in + if in.RecordedAt != nil { + in, out := &in.RecordedAt, &out.RecordedAt + *out = (*in).DeepCopy() + } + if in.Volumes != nil { + in, out := &in.Volumes, &out.Volumes + *out = make([]BackupVolumeCoverage, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BackupCaptureManifest. +func (in *BackupCaptureManifest) DeepCopy() *BackupCaptureManifest { + if in == nil { + return nil + } + out := new(BackupCaptureManifest) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *BackupList) DeepCopyInto(out *BackupList) { *out = *in @@ -388,6 +414,11 @@ func (in *BackupSpec) DeepCopy() *BackupSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *BackupStatus) DeepCopyInto(out *BackupStatus) { *out = *in + if in.CaptureManifest != nil { + in, out := &in.CaptureManifest, &out.CaptureManifest + *out = new(BackupCaptureManifest) + (*in).DeepCopyInto(*out) + } if in.StartTime != nil { in, out := &in.StartTime, &out.StartTime *out = (*in).DeepCopy() @@ -420,6 +451,26 @@ func (in *BackupStatus) DeepCopy() *BackupStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *BackupVolumeCoverage) DeepCopyInto(out *BackupVolumeCoverage) { + *out = *in + if in.MountPaths != nil { + in, out := &in.MountPaths, &out.MountPaths + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BackupVolumeCoverage. +func (in *BackupVolumeCoverage) DeepCopy() *BackupVolumeCoverage { + if in == nil { + return nil + } + out := new(BackupVolumeCoverage) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *BanListSpec) DeepCopyInto(out *BanListSpec) { *out = *in @@ -953,6 +1004,11 @@ func (in *GameServerNetworking) DeepCopy() *GameServerNetworking { func (in *GameServerSpec) DeepCopyInto(out *GameServerSpec) { *out = *in out.TemplateRef = in.TemplateRef + if in.TemplateRollout != nil { + in, out := &in.TemplateRollout, &out.TemplateRollout + *out = new(TemplateRolloutSpec) + **out = **in + } if in.StopGracePeriodSeconds != nil { in, out := &in.StopGracePeriodSeconds, &out.StopGracePeriodSeconds *out = new(int32) @@ -1042,6 +1098,11 @@ func (in *GameServerSpec) DeepCopy() *GameServerSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *GameServerStatus) DeepCopyInto(out *GameServerStatus) { *out = *in + if in.TemplateRevision != nil { + in, out := &in.TemplateRevision, &out.TemplateRevision + *out = new(TemplateRevisionStatus) + **out = **in + } if in.Conditions != nil { in, out := &in.Conditions, &out.Conditions *out = make([]v1.Condition, len(*in)) @@ -2573,6 +2634,36 @@ func (in *TailscaleTunnelSpec) DeepCopy() *TailscaleTunnelSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TemplateRevisionStatus) DeepCopyInto(out *TemplateRevisionStatus) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TemplateRevisionStatus. +func (in *TemplateRevisionStatus) DeepCopy() *TemplateRevisionStatus { + if in == nil { + return nil + } + out := new(TemplateRevisionStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TemplateRolloutSpec) DeepCopyInto(out *TemplateRolloutSpec) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TemplateRolloutSpec. +func (in *TemplateRolloutSpec) DeepCopy() *TemplateRolloutSpec { + if in == nil { + return nil + } + out := new(TemplateRolloutSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *VerifySpec) DeepCopyInto(out *VerifySpec) { *out = *in diff --git a/operator/config/crd/gameplane.local_backups.yaml b/operator/config/crd/gameplane.local_backups.yaml index 9b3af89a3..1a46399b8 100644 --- a/operator/config/crd/gameplane.local_backups.yaml +++ b/operator/config/crd/gameplane.local_backups.yaml @@ -128,6 +128,75 @@ spec: status: description: BackupStatus is the observed state of a Backup. properties: + captureManifest: + description: |- + CaptureManifest is recorded once, before the first capture worker starts. + It describes persistent-volume coverage independently of copy success. + Missing on pre-upgrade backups; never reconstruct it from current layout. + properties: + layoutSource: + description: LayoutSource identifies the frozen workload used + for mount paths. + type: string + quiesceOutcome: + description: |- + QuiesceOutcome is confirmed, already-stopped, unsupported, not-requested, + agent-unavailable, or unknown, independent of subsequent copy success. + type: string + recordedAt: + format: date-time + type: string + serverUID: + type: string + state: + description: |- + State is Recorded for new captures, LegacyUnknown for an existing worker + whose original layout was never recorded. + enum: + - Recorded + - LegacyUnknown + type: string + strategy: + type: string + volumes: + items: + description: |- + BackupVolumeCoverage describes one persistent claim. Extra and loader/mod + claims remain uncovered even when mounted beneath the primary data directory. + properties: + claimName: + type: string + claimUID: + description: ClaimUID is empty if the claim did not yet + exist at admission. + type: string + classification: + description: |- + Classification records the module/server declaration at admission; + it does not prove that the volume can actually be reconstructed. + enum: + - data + - rebuildable + - unclassified + type: string + included: + type: boolean + mountPaths: + items: + type: string + type: array + reason: + type: string + required: + - claimName + - classification + - included + - reason + type: object + type: array + required: + - state + type: object completionTime: format: date-time type: string diff --git a/operator/config/crd/gameplane.local_gameservers.yaml b/operator/config/crd/gameplane.local_gameservers.yaml index a4ee55a44..6598e07ff 100644 --- a/operator/config/crd/gameplane.local_gameservers.yaml +++ b/operator/config/crd/gameplane.local_gameservers.yaml @@ -2219,6 +2219,16 @@ spec: primary Storage.MountPath, since these exist precisely for directories that share no safe common parent with it. properties: + backupRole: + description: |- + BackupRole describes recovery requirements without adding this volume + to a backup. "data" needs a separate copy; "rebuildable" may be recreated. + Unset remains unclassified and not covered. Both backup strategies still + capture only the primary data claim. + enum: + - data + - rebuildable + type: string mountPath: description: MountPath is the absolute in-container path this volume is mounted at. @@ -2290,6 +2300,27 @@ spec: - message: templateRef is immutable; delete and recreate the GameServer to switch templates rule: self == oldSelf + templateRollout: + description: |- + TemplateRollout controls adoption of changes to the shared GameTemplate. + Omitted preserves Automatic adoption. Per-server overrides remain live. + properties: + policy: + default: Automatic + description: Policy defaults to Automatic. OnRequest requires + the exact desired digest. + enum: + - Automatic + - OnRequest + type: string + requestedRevision: + description: |- + RequestedRevision authorizes this SHA256 only, never future revisions. + Read status.templateRevision.desired and copy it here to admit an update. + Initial provisioning always adopts the installed template. + pattern: ^[a-f0-9]{64}$ + type: string + type: object tolerations: items: description: |- @@ -2625,6 +2656,25 @@ spec: observed as Ready. format: date-time type: string + templateRevision: + description: |- + TemplateRevision is a read model; operator-owned annotations and immutable + ConfigMaps retain admission state across status resets and leader changes. + properties: + desired: + description: Desired is the installed template's SHA256; empty + if it is unavailable. + type: string + effective: + description: Effective is the immutable content used for all derived + resources. + type: string + running: + description: |- + Running comes from an owned Pod with a running game container. Empty + means no running revision is known, including pre-upgrade legacy Pods. + type: string + type: object tunnelEndpoints: description: |- TunnelEndpoints are the per-port addresses reported by the tunnel pod. diff --git a/operator/config/crd/gameplane.local_gametemplates.yaml b/operator/config/crd/gameplane.local_gametemplates.yaml index c6428c858..729b872e1 100644 --- a/operator/config/crd/gameplane.local_gametemplates.yaml +++ b/operator/config/crd/gameplane.local_gametemplates.yaml @@ -2165,6 +2165,16 @@ spec: primary Storage.MountPath, since these exist precisely for directories that share no safe common parent with it. properties: + backupRole: + description: |- + BackupRole describes recovery requirements without adding this volume + to a backup. "data" needs a separate copy; "rebuildable" may be recreated. + Unset remains unclassified and not covered. Both backup strategies still + capture only the primary data claim. + enum: + - data + - rebuildable + type: string mountPath: description: MountPath is the absolute in-container path this volume is mounted at. diff --git a/operator/hack/condition-ids/main.go b/operator/hack/condition-ids/main.go index 06f0b2b1e..07b362029 100644 --- a/operator/hack/condition-ids/main.go +++ b/operator/hack/condition-ids/main.go @@ -46,6 +46,7 @@ type row struct { // guardCalls maps a helper that takes a condition reason as a string literal // to the index of that argument. var guardCalls = map[string]int{ + "setTemplateCondition": 2, "markUnhealthy": 2, "markFailed": 2, "markPending": 2, @@ -67,6 +68,11 @@ var guardSkipFiles = map[string]bool{ // rows is the English reference for every condition reason. var rows = []row{ + {types: []string{"TemplateUpdatePending"}, reason: "AdoptingRevision", english: "The admitted template revision is being adopted.", description: "Template rollout owns the server lifecycle while draining and replacing its workload."}, + {types: []string{"TemplateUpdatePending"}, reason: "TemplateUnavailable", english: "The installed template is unavailable; the effective snapshot is retained.", description: "Template availability is unknown but the server retains its immutable effective content."}, + {types: []string{"TemplateUpdatePending"}, reason: "RevisionCurrent", english: "The installed template revision is effective.", description: "No newer template revision is waiting for adoption. Running revision is observed separately."}, + {types: []string{"TemplateUpdatePending"}, reason: "OperationInProgress", english: "Template adoption waits for the active lifecycle operation to finish.", description: "A backup, restore or wipe prevents admitting an available template revision."}, + {types: []string{"TemplateUpdatePending"}, reason: "RevisionAvailable", english: "An installed template revision is waiting for admission.", description: "An installed template differs from the server's immutable effective revision."}, // GameServer: Ready {types: []string{"Ready"}, reason: "Running", english: "The server is running and the agent is reporting heartbeats.", description: "Ready condition while the game server is up."}, {types: []string{"Ready"}, reason: "Pending", english: "The server is waiting to start.", description: "Ready condition while the game server is being created."}, @@ -149,6 +155,17 @@ var rows = []row{ {types: []string{"SnapshotForgotten"}, reason: "Forgotten", english: "The snapshot was removed from the backup repository.", description: "SnapshotForgotten condition after a deleted Backup's snapshot was forgotten."}, {types: []string{"SnapshotForgotten"}, reason: "ForgetAbandoned", english: "The snapshot was not removed from the backup repository when the backup was deleted.", description: "SnapshotForgotten condition when the forget step was abandoned."}, + // Consistency is separate from copy completion and volume coverage. + {types: []string{"Consistency"}, reason: "CaptureFailed", english: "The capture failed; no usable consistency guarantee was achieved.", description: "Backup achieved consistency: CaptureFailed."}, + {types: []string{"Consistency"}, reason: "LegacyMetadataUnknown", english: "Capture-time metadata is unavailable. Do not infer consistency or volume coverage from the current template.", description: "Backup achieved consistency: LegacyMetadataUnknown."}, + {types: []string{"Consistency"}, reason: "CapturePending", english: "The capture has not completed; its achieved consistency is not yet known.", description: "Backup achieved consistency: CapturePending."}, + {types: []string{"Consistency"}, reason: "Quiesced", english: "The primary volume copy completed after the agent confirmed quiesce. This depends on the module's quiesce commands; other volumes remain outside this backup.", description: "Backup achieved consistency: Quiesced."}, + {types: []string{"Consistency"}, reason: "StoppedSource", english: "The primary volume copy completed from a source that was already stopped and drained before capture. Other volumes remain outside this backup.", description: "Backup achieved consistency: StoppedSource."}, + {types: []string{"Consistency"}, reason: "CrashConsistentSnapshot", english: "The primary CSI snapshot is crash-consistent; application quiesce was not confirmed.", description: "Backup achieved consistency: CrashConsistentSnapshot."}, + {types: []string{"Consistency"}, reason: "UnsupportedRawCopy", english: "The game does not support quiesce. The primary volume was copied while it could still change.", description: "Backup achieved consistency: UnsupportedRawCopy."}, + {types: []string{"Consistency"}, reason: "AgentUnavailableRawCopy", english: "No agent quiesce capability was configured. The primary volume was copied without pausing game writes.", description: "Backup achieved consistency: AgentUnavailableRawCopy."}, + {types: []string{"Consistency"}, reason: "RawCopy", english: "The primary volume was copied without confirmed application quiesce.", description: "Backup achieved consistency: RawCopy."}, + // BackupSchedule {types: []string{"ScheduleValid"}, reason: "CronParseError", english: "The schedule is not a valid cron expression.", description: "ScheduleValid condition for an unparseable schedule."}, {types: []string{"ScheduleValid"}, reason: "CronValid", english: "The schedule is a valid cron expression.", description: "ScheduleValid condition for a good schedule."}, diff --git a/operator/internal/controller/backup_controller.go b/operator/internal/controller/backup_controller.go index cefa931b7..1c7d97965 100644 --- a/operator/internal/controller/backup_controller.go +++ b/operator/internal/controller/backup_controller.go @@ -276,6 +276,11 @@ func (r *BackupReconciler) reconcile(ctx context.Context, req ctrl.Request) (ctr // Re-entering the full pipeline would re-validate the GameServer and repo // Secret (flipping an already-Succeeded Backup to Failed if either was since // deleted) and would re-create a cleaned-up Job, running a second backup. + if b.Status.Phase == gameplanev1alpha1.BackupPhaseSucceeded || b.Status.Phase == gameplanev1alpha1.BackupPhaseFailed { + if err := r.syncBackupConsistency(ctx, &b); err != nil { + return ctrl.Result{}, err + } + } if b.Status.Phase == gameplanev1alpha1.BackupPhaseSucceeded && b.Status.SnapshotID == "" { return r.retrySnapshotScrape(ctx, &b) } @@ -317,13 +322,6 @@ func (r *BackupReconciler) reconcile(ctx context.Context, req ctrl.Request) (ctr return ctrl.Result{}, err } - // A missing template is not fatal: it only refines the FSGroup, and - // buildBackupPodSecurityContext falls back to 65532 for a zero template. - var tmpl gameplanev1alpha1.GameTemplate - if err := r.Get(ctx, types.NamespacedName{Name: gs.Spec.TemplateRef.Name}, &tmpl); err != nil && !apierrors.IsNotFound(err) { - return ctrl.Result{}, err - } - // Volume-snapshot backups take a CSI snapshot of the data PVC rather // than running restic, so they branch off here — before the restic // repo Secret checks, which don't apply to them. @@ -362,6 +360,19 @@ func (r *BackupReconciler) reconcile(ctx context.Context, req ctrl.Request) (ctr if res, err := r.prepareBackupQuiesce(ctx, &b); err != nil || res.RequeueAfter > 0 { return res, err } + // Admission refreshes the target. Resolve the pin only after owning the + // lifecycle fence, so a concurrent rollout cannot change the copy layout. + bound, err := r.ownedBackupTarget(ctx, &b) + if err != nil { + return ctrl.Result{}, err + } + if bound == nil { + return ctrl.Result{RequeueAfter: 5 * time.Second}, nil + } + tmpl, err := effectiveGameTemplate(ctx, r.backupAPIReader(), bound) + if err != nil && (bound.Annotations[effectiveTemplateAnnotation] != "" || !apierrors.IsNotFound(err)) { + return ctrl.Result{}, err + } backoff, deadline := resolveJobLimits(r.JobBackoffLimit, r.JobActiveDeadlineSeconds) job := &batchv1.Job{ObjectMeta: metav1.ObjectMeta{Name: b.Name, Namespace: b.Namespace}} @@ -370,7 +381,7 @@ func (r *BackupReconciler) reconcile(ctx context.Context, req ctrl.Request) (ctr job.Spec.BackoffLimit = &backoff job.Spec.ActiveDeadlineSeconds = &deadline job.Spec.Template.Labels = map[string]string{backupRestoreJobLabel: backupRestoreJobValue, backupUIDLabel: string(b.UID)} - job.Spec.Template.Spec = r.buildBackupPodSpec(&b, &tmpl) + job.Spec.Template.Spec = r.buildBackupPodSpec(&b, tmpl) job.Spec.Template.Spec.RestartPolicy = corev1.RestartPolicyNever } return controllerutil.SetControllerReference(&b, job, r.Scheme) @@ -576,6 +587,7 @@ func (r *BackupReconciler) mirrorJobStatus( cond.Reason = string(phase) } newConds := upsertCondition(b.Status.Conditions, cond) + newConds = upsertCondition(newConds, backupConsistencyCondition(b)) if !sameConditions(b.Status.Conditions, newConds) { b.Status.Conditions = newConds dirty = true @@ -966,6 +978,7 @@ func (r *BackupReconciler) fail(ctx context.Context, b *gameplanev1alpha1.Backup Message: msg, ObservedGeneration: b.Generation, }) + b.Status.Conditions = upsertCondition(b.Status.Conditions, backupConsistencyCondition(b)) if err := r.Status().Update(ctx, b); err != nil { return ctrl.Result{}, err } diff --git a/operator/internal/controller/backup_guard.go b/operator/internal/controller/backup_guard.go index 56083c303..d29ff2b5b 100644 --- a/operator/internal/controller/backup_guard.go +++ b/operator/internal/controller/backup_guard.go @@ -291,6 +291,12 @@ func (r *BackupReconciler) prepareBackupQuiesce(ctx context.Context, b *gameplan if err != nil || !ready { return ctrl.Result{RequeueAfter: 5 * time.Second}, err } + if err := r.recordBackupManifest(ctx, b, gs); err != nil { + // Metadata must be durable before capture. Do not leave saving paused + // indefinitely when a lookup/status write prevents the copy starting. + _, failErr := r.fail(ctx, b, "cannot record capture metadata: "+err.Error()) + return ctrl.Result{RequeueAfter: time.Second}, failErr + } return ctrl.Result{}, nil } diff --git a/operator/internal/controller/backup_guard_test.go b/operator/internal/controller/backup_guard_test.go index 3667d187e..4782cc635 100644 --- a/operator/internal/controller/backup_guard_test.go +++ b/operator/internal/controller/backup_guard_test.go @@ -109,12 +109,23 @@ func TestBackupGuardSerializesAcrossBackupsAndRestart(t *testing.T) { if len(jobs.Items) != 1 || q.quiesced != 1 || q.unquiesced != 0 { t.Fatalf("overlapping protected copies: jobs=%d, agent=%+v", len(jobs.Items), q) } + first := guardGetBackup(t, r, "first") + if first.Status.CaptureManifest == nil || first.Status.CaptureManifest.QuiesceOutcome != "confirmed" { + t.Fatalf("worker started without confirmed capture-time metadata: %+v", first.Status) + } + if second := guardGetBackup(t, r, "second"); second.Status.CaptureManifest != nil { + t.Fatalf("queued copy recorded metadata before admission: %+v", second.Status.CaptureManifest) + } job := &jobs.Items[0] job.Status.Succeeded = 1 if err := r.Status().Update(context.Background(), job); err != nil { t.Fatal(err) } guardPass(t, r, "first") + first = guardGetBackup(t, r, "first") + if condition := meta.FindStatusCondition(first.Status.Conditions, "Consistency"); condition == nil || condition.Status != metav1.ConditionTrue || condition.Reason != "Quiesced" { + t.Fatalf("completed copy did not expose achieved quiesce: %+v", first.Status) + } guardPass(t, r, "second") if q.quiesced != 2 || q.unquiesced != 1 { t.Fatalf("second did not acquire after first cleanup: %+v", q) diff --git a/operator/internal/controller/backup_manifest.go b/operator/internal/controller/backup_manifest.go new file mode 100644 index 000000000..c2c2afb70 --- /dev/null +++ b/operator/internal/controller/backup_manifest.go @@ -0,0 +1,180 @@ +package controller + +import ( + "context" + "sort" + + snapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" + appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" +) + +// Called after lifecycle ownership and the workload fence, before creating the +// copy worker. Requeues and later template edits never rewrite this manifest. +func (r *BackupReconciler) recordBackupManifest(ctx context.Context, b *gameplanev1alpha1.Backup, gs *gameplanev1alpha1.GameServer) error { + if b.Status.CaptureManifest != nil { + return nil + } + reader := r.backupAPIReader() + var worker client.Object = &batchv1.Job{} + if b.Spec.Strategy == "volume-snapshot" { + worker = &snapshotv1.VolumeSnapshot{} + } + if err := reader.Get(ctx, client.ObjectKeyFromObject(b), worker); err == nil { + // Existing workers predate this metadata. Current layout cannot prove + // what they captured, even if their requested quiesce was successful. + b.Status.CaptureManifest = &gameplanev1alpha1.BackupCaptureManifest{State: "LegacyUnknown"} + return r.Status().Update(ctx, b) + } else if !apierrors.IsNotFound(err) { + return err + } + var ss appsv1.StatefulSet + err := reader.Get(ctx, client.ObjectKeyFromObject(gs), &ss) + if err != nil && !apierrors.IsNotFound(err) { + return err + } + var layout corev1.PodSpec + source := "Unavailable" + if err == nil && metav1.IsControlledBy(&ss, gs) { + layout, source = ss.Spec.Template.Spec, "StatefulSet" + var pod corev1.Pod + err := reader.Get(ctx, client.ObjectKey{Namespace: gs.Namespace, Name: gs.Name + "-0"}, &pod) + if err != nil && !apierrors.IsNotFound(err) { + return err + } + if err == nil && metav1.IsControlledBy(&pod, &ss) { + layout, source = pod.Spec, "Pod" + } + } + tmpl, err := effectiveGameTemplate(ctx, reader, gs) + if err != nil && (gs.Annotations[effectiveTemplateAnnotation] != "" || !apierrors.IsNotFound(err)) { + return err + } + primary := gs.Name + "-data" + volumes := map[string]*gameplanev1alpha1.BackupVolumeCoverage{ + primary: {ClaimName: primary, Included: true, Classification: "data", CoverageReason: "PrimaryDataClaim"}, + } + podNames := map[string]string{} + for _, volume := range layout.Volumes { + if volume.PersistentVolumeClaim == nil { + continue + } + name := volume.PersistentVolumeClaim.ClaimName + podNames[volume.Name] = name + if volumes[name] == nil { + volumes[name] = &gameplanev1alpha1.BackupVolumeCoverage{ClaimName: name, Classification: "unclassified", CoverageReason: "NotCovered"} + } + } + for _, container := range layout.Containers { + if container.Name != "game" { + continue + } + for _, mount := range container.VolumeMounts { + if volume := volumes[podNames[mount.Name]]; volume != nil { + volume.MountPaths = append(volume.MountPaths, mount.MountPath) + } + } + } + var claims corev1.PersistentVolumeClaimList + if err := reader.List(ctx, &claims, client.InNamespace(gs.Namespace)); err != nil { + return err + } + for _, claim := range claims.Items { + volume := volumes[claim.Name] + if volume == nil && metav1.IsControlledBy(&claim, gs) { + volume = &gameplanev1alpha1.BackupVolumeCoverage{ClaimName: claim.Name, Classification: "unclassified", CoverageReason: "RetainedUnmountedClaim"} + volumes[claim.Name] = volume + } + if volume != nil { + volume.ClaimUID = string(claim.UID) + } + } + for _, extra := range effectiveExtraVolumes(gs, tmpl) { + volume := volumes[extraPVCName(gs, extra.Name)] + if volume == nil { + continue + } + // A newer declaration for a different mount must not classify the + // frozen workload's volume as safely rebuildable. + for _, mount := range volume.MountPaths { + if mount == extra.MountPath && extra.BackupRole != "" { + volume.Classification = extra.BackupRole + } + } + } + now := metav1.Now() + manifest := &gameplanev1alpha1.BackupCaptureManifest{State: "Recorded", RecordedAt: &now, + ServerUID: string(gs.UID), Strategy: b.Spec.Strategy, LayoutSource: source, QuiesceOutcome: backupQuiesceOutcome(b)} + if manifest.Strategy == "" { + manifest.Strategy = "restic-snapshot" + } + for _, volume := range volumes { + sort.Strings(volume.MountPaths) + manifest.Volumes = append(manifest.Volumes, *volume) + } + sort.Slice(manifest.Volumes, func(i, j int) bool { return manifest.Volumes[i].ClaimName < manifest.Volumes[j].ClaimName }) + b.Status.CaptureManifest = manifest + return r.Status().Update(ctx, b) +} + +func backupQuiesceOutcome(b *gameplanev1alpha1.Backup) string { + if preparation, err := parseBackupPreparation(b); err == nil && preparation != nil && preparation.Outcome == "already-stopped" { + return "already-stopped" + } + switch b.Annotations[annoQuiesceAttempted] { + case "true": + return "confirmed" + case "unsupported": + return "unsupported" + case "pending": + return "unknown" + default: + if !b.Spec.Quiesce { + return "not-requested" + } + return "agent-unavailable" + } +} + +// Consistency=True means the copy completed after confirmed quiesce or from an +// already-stopped source whose writers had drained before capture admission. +// False is an explicit weaker guarantee, not a synonym for copy failure. +func backupConsistencyCondition(b *gameplanev1alpha1.Backup) metav1.Condition { + condition := metav1.Condition{Type: "Consistency", ObservedGeneration: b.Generation, Status: metav1.ConditionUnknown} + switch { + case b.Status.Phase == gameplanev1alpha1.BackupPhaseFailed: + condition.Status, condition.Reason, condition.Message = metav1.ConditionFalse, "CaptureFailed", "The capture failed; no usable consistency guarantee was achieved." + case b.Status.CaptureManifest == nil || b.Status.CaptureManifest.State != "Recorded": + condition.Reason, condition.Message = "LegacyMetadataUnknown", "Capture-time metadata is unavailable. Do not infer consistency or volume coverage from the current template." + case b.Status.Phase != gameplanev1alpha1.BackupPhaseSucceeded || b.Status.SnapshotID == "": + condition.Reason, condition.Message = "CapturePending", "The capture has not completed; its achieved consistency is not yet known." + case b.Status.CaptureManifest.QuiesceOutcome == "confirmed": + condition.Status, condition.Reason, condition.Message = metav1.ConditionTrue, "Quiesced", "The primary volume copy completed after the agent confirmed quiesce. This depends on the module's quiesce commands; other volumes remain outside this backup." + case b.Status.CaptureManifest.QuiesceOutcome == "already-stopped": + condition.Status, condition.Reason, condition.Message = metav1.ConditionTrue, "StoppedSource", "The primary volume copy completed from a source that was already stopped and drained before capture. Other volumes remain outside this backup." + case b.Status.CaptureManifest.Strategy == "volume-snapshot": + condition.Status, condition.Reason, condition.Message = metav1.ConditionFalse, "CrashConsistentSnapshot", "The primary CSI snapshot is crash-consistent; application quiesce was not confirmed ("+b.Status.CaptureManifest.QuiesceOutcome+")." + case b.Status.CaptureManifest.QuiesceOutcome == "unsupported": + condition.Status, condition.Reason, condition.Message = metav1.ConditionFalse, "UnsupportedRawCopy", "The game does not support quiesce. The primary volume was copied while it could still change." + case b.Status.CaptureManifest.QuiesceOutcome == "agent-unavailable": + condition.Status, condition.Reason, condition.Message = metav1.ConditionFalse, "AgentUnavailableRawCopy", "No agent quiesce capability was configured. The primary volume was copied without pausing game writes." + default: + condition.Status, condition.Reason, condition.Message = metav1.ConditionFalse, "RawCopy", "The primary volume was copied without confirmed application quiesce." + } + return condition +} + +func (r *BackupReconciler) syncBackupConsistency(ctx context.Context, b *gameplanev1alpha1.Backup) error { + conditions := upsertCondition(b.Status.Conditions, backupConsistencyCondition(b)) + if sameConditions(b.Status.Conditions, conditions) { + return nil + } + b.Status.Conditions = conditions + return r.Status().Update(ctx, b) +} diff --git a/operator/internal/controller/backup_manifest_test.go b/operator/internal/controller/backup_manifest_test.go new file mode 100644 index 000000000..89b6a00fd --- /dev/null +++ b/operator/internal/controller/backup_manifest_test.go @@ -0,0 +1,184 @@ +package controller + +import ( + "reflect" + "testing" + + snapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" + + appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" +) + +func TestBackupManifestPinsCoverageAndClassification(t *testing.T) { + scheme := runtime.NewScheme() + _ = corev1.AddToScheme(scheme) + _ = appsv1.AddToScheme(scheme) + _ = batchv1.AddToScheme(scheme) + _ = gameplanev1alpha1.AddToScheme(scheme) + gs := &gameplanev1alpha1.GameServer{ObjectMeta: metav1.ObjectMeta{Name: "game", Namespace: "games", UID: "game-uid"}, Spec: gameplanev1alpha1.GameServerSpec{TemplateRef: gameplanev1alpha1.GameTemplateRef{Name: "template"}}} + tmpl := &gameplanev1alpha1.GameTemplate{ObjectMeta: metav1.ObjectMeta{Name: "template"}, Spec: gameplanev1alpha1.GameTemplateSpec{Storage: gameplanev1alpha1.GameStorageSpec{ + Extra: []gameplanev1alpha1.ExtraVolumeSpec{{Name: "install", MountPath: "/serverfiles", BackupRole: "rebuildable"}}, + }}} + b := &gameplanev1alpha1.Backup{ObjectMeta: metav1.ObjectMeta{Name: "copy", Namespace: "games", UID: "backup-uid", Annotations: map[string]string{annoQuiesceAttempted: "unsupported"}}, Spec: gameplanev1alpha1.BackupSpec{ServerRef: gameplanev1alpha1.LocalObjectRef{Name: gs.Name}, Quiesce: true}} + owner := *metav1.NewControllerRef(gs, gameplanev1alpha1.GroupVersion.WithKind("GameServer")) + ss := &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{Name: gs.Name, Namespace: gs.Namespace, OwnerReferences: []metav1.OwnerReference{owner}}, + Spec: appsv1.StatefulSetSpec{Template: corev1.PodTemplateSpec{Spec: corev1.PodSpec{ + Containers: []corev1.Container{{Name: "game", VolumeMounts: []corev1.VolumeMount{{Name: "data", MountPath: "/world"}, {Name: "extra-install", MountPath: "/serverfiles"}, {Name: "mods", MountPath: "/world/mods"}}}}, + Volumes: []corev1.Volume{ + {Name: "data", VolumeSource: corev1.VolumeSource{PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: "game-data"}}}, + {Name: "extra-install", VolumeSource: corev1.VolumeSource{PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: "game-extra-install"}}}, + {Name: "mods", VolumeSource: corev1.VolumeSource{PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: "game-mods-active"}}}, + }, + }}}, + } + objects := []client.Object{gs, tmpl, b, ss} + for _, name := range []string{"game-data", "game-extra-install", "game-mods-active", "game-mods-old"} { + objects = append(objects, &corev1.PersistentVolumeClaim{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: gs.Namespace, UID: types.UID("uid-" + name), OwnerReferences: []metav1.OwnerReference{owner}}}) + } + c := fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(b).WithObjects(objects...).Build() + r := &BackupReconciler{Client: c, APIReader: c, Scheme: scheme} + if err := r.recordBackupManifest(t.Context(), b, gs); err != nil { + t.Fatal(err) + } + manifest := b.Status.CaptureManifest + if manifest == nil || manifest.State != "Recorded" || manifest.QuiesceOutcome != "unsupported" || len(manifest.Volumes) != 4 { + t.Fatalf("incomplete manifest: %+v", manifest) + } + byName := map[string]gameplanev1alpha1.BackupVolumeCoverage{} + for _, volume := range manifest.Volumes { + byName[volume.ClaimName] = volume + } + if !byName["game-data"].Included || byName["game-data"].ClaimUID != "uid-game-data" || !reflect.DeepEqual(byName["game-data"].MountPaths, []string{"/world"}) { + t.Fatalf("primary coverage: %+v", byName["game-data"]) + } + if byName["game-extra-install"].Included || byName["game-extra-install"].Classification != "rebuildable" || byName["game-mods-active"].Included || byName["game-mods-old"].Included { + t.Fatalf("extra or nested mod volume incorrectly covered: %+v", manifest.Volumes) + } + before := manifest.DeepCopy() + tmpl.Spec.Storage.Extra[0].BackupRole = "data" + if err := c.Update(t.Context(), tmpl); err != nil { + t.Fatal(err) + } + if err := r.recordBackupManifest(t.Context(), b, gs); err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(before, b.Status.CaptureManifest) { + t.Fatal("template change rewrote capture manifest") + } +} + +func TestBackupManifestUsesEffectiveTemplateClassification(t *testing.T) { + r, gs, installed := revisionFixture(t) + installed.Spec.Storage.Extra = []gameplanev1alpha1.ExtraVolumeSpec{{Name: "world", MountPath: "/world-extra", BackupRole: "data"}} + if err := r.Update(t.Context(), installed); err != nil { + t.Fatal(err) + } + pin, err := r.saveTemplateSnapshot(t.Context(), gs, installed) + if err != nil { + t.Fatal(err) + } + gs.Annotations[effectiveTemplateAnnotation] = pin + if err := r.Update(t.Context(), gs); err != nil { + t.Fatal(err) + } + installed.Spec.Storage.Extra[0].BackupRole = "rebuildable" + if err := r.Update(t.Context(), installed); err != nil { + t.Fatal(err) + } + b := &gameplanev1alpha1.Backup{ObjectMeta: metav1.ObjectMeta{Name: "capture", Namespace: gs.Namespace}} + ss := &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{Name: gs.Name, Namespace: gs.Namespace, OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(gs, gameplanev1alpha1.GroupVersion.WithKind("GameServer"))}}, Spec: appsv1.StatefulSetSpec{Template: corev1.PodTemplateSpec{Spec: corev1.PodSpec{ + Containers: []corev1.Container{{Name: "game", VolumeMounts: []corev1.VolumeMount{{Name: "world", MountPath: "/world-extra"}}}}, + Volumes: []corev1.Volume{{Name: "world", VolumeSource: corev1.VolumeSource{PersistentVolumeClaim: &corev1.PersistentVolumeClaimVolumeSource{ClaimName: "game-extra-world"}}}}, + }}}} + for _, obj := range []client.Object{b, ss} { + if err := r.Create(t.Context(), obj); err != nil { + t.Fatal(err) + } + } + backups := &BackupReconciler{Client: r.Client, APIReader: r.APIReader, Scheme: r.Scheme} + if err := backups.recordBackupManifest(t.Context(), b, gs); err != nil { + t.Fatal(err) + } + for _, volume := range b.Status.CaptureManifest.Volumes { + if volume.ClaimName == "game-extra-world" && volume.Classification == "data" { + return + } + } + t.Fatal("pending catalog revision changed effective volume classification") +} + +func TestBackupManifestDoesNotInventHistoryForExistingWorker(t *testing.T) { + for _, strategy := range []string{"restic-snapshot", "volume-snapshot"} { + t.Run(strategy, func(t *testing.T) { + scheme := runtime.NewScheme() + _ = corev1.AddToScheme(scheme) + _ = batchv1.AddToScheme(scheme) + _ = snapshotv1.AddToScheme(scheme) + _ = gameplanev1alpha1.AddToScheme(scheme) + b := &gameplanev1alpha1.Backup{ObjectMeta: metav1.ObjectMeta{Name: "legacy", Namespace: "games"}, Spec: gameplanev1alpha1.BackupSpec{Strategy: strategy}} + var worker client.Object = &batchv1.Job{ObjectMeta: b.ObjectMeta} + if strategy == "volume-snapshot" { + worker = &snapshotv1.VolumeSnapshot{ObjectMeta: b.ObjectMeta} + } + c := fake.NewClientBuilder().WithScheme(scheme).WithStatusSubresource(b).WithObjects(b, worker).Build() + r := &BackupReconciler{Client: c, APIReader: c, Scheme: scheme} + if err := r.recordBackupManifest(t.Context(), b, &gameplanev1alpha1.GameServer{}); err != nil { + t.Fatal(err) + } + if b.Status.CaptureManifest.State != "LegacyUnknown" || len(b.Status.CaptureManifest.Volumes) != 0 { + t.Fatalf("invented history: %+v", b.Status.CaptureManifest) + } + }) + } +} + +func TestBackupConsistencyDescribesAchievedResult(t *testing.T) { + for _, tc := range []struct { + strategy, outcome, reason string + status metav1.ConditionStatus + }{ + {"restic-snapshot", "confirmed", "Quiesced", metav1.ConditionTrue}, + {"restic-snapshot", "already-stopped", "StoppedSource", metav1.ConditionTrue}, + {"restic-snapshot", "unsupported", "UnsupportedRawCopy", metav1.ConditionFalse}, + {"restic-snapshot", "not-requested", "RawCopy", metav1.ConditionFalse}, + {"restic-snapshot", "agent-unavailable", "AgentUnavailableRawCopy", metav1.ConditionFalse}, + {"volume-snapshot", "unsupported", "CrashConsistentSnapshot", metav1.ConditionFalse}, + {"volume-snapshot", "confirmed", "Quiesced", metav1.ConditionTrue}, + {"volume-snapshot", "already-stopped", "StoppedSource", metav1.ConditionTrue}, + } { + t.Run(tc.strategy+"/"+tc.outcome, func(t *testing.T) { + b := &gameplanev1alpha1.Backup{Status: gameplanev1alpha1.BackupStatus{Phase: gameplanev1alpha1.BackupPhaseSucceeded, SnapshotID: "snapshot-id", + CaptureManifest: &gameplanev1alpha1.BackupCaptureManifest{State: "Recorded", Strategy: tc.strategy, QuiesceOutcome: tc.outcome}}} + condition := backupConsistencyCondition(b) + if condition.Reason != tc.reason || condition.Status != tc.status { + t.Fatalf("got %+v", condition) + } + b.Status.SnapshotID = "" + if condition := backupConsistencyCondition(b); condition.Reason != "CapturePending" || condition.Status != metav1.ConditionUnknown { + t.Fatalf("copy without restorable identity claims achieved consistency: %+v", condition) + } + b.Status.SnapshotID = "snapshot-id" + b.Status.Phase = gameplanev1alpha1.BackupPhaseRunning + if condition := backupConsistencyCondition(b); condition.Status != metav1.ConditionUnknown { + t.Fatalf("unfinished copy claims achieved consistency: %+v", condition) + } + b.Status.Phase = gameplanev1alpha1.BackupPhaseFailed + if condition := backupConsistencyCondition(b); condition.Reason != "CaptureFailed" || condition.Status != metav1.ConditionFalse { + t.Fatalf("failed copy claims usable result: %+v", condition) + } + }) + } + b := &gameplanev1alpha1.Backup{Status: gameplanev1alpha1.BackupStatus{Phase: gameplanev1alpha1.BackupPhaseSucceeded}} + if condition := backupConsistencyCondition(b); condition.Status != metav1.ConditionUnknown || condition.Reason != "LegacyMetadataUnknown" { + t.Fatalf("legacy backup guarantee invented: %+v", condition) + } +} diff --git a/operator/internal/controller/backup_stopped_test.go b/operator/internal/controller/backup_stopped_test.go index 3d0f30f39..eb07407b8 100644 --- a/operator/internal/controller/backup_stopped_test.go +++ b/operator/internal/controller/backup_stopped_test.go @@ -67,6 +67,9 @@ func TestBackupStoppedCopiesWithoutWakingAndReleasesForNextCopy(t *testing.T) { if prep.Outcome != "already-stopped" || prep.TargetUID != "server-uid" || prep.WorkloadUID != "workload-uid" || prep.PodUID != "" { t.Fatalf("wrong source proof: %+v", prep) } + if first.Status.CaptureManifest == nil || first.Status.CaptureManifest.QuiesceOutcome != "already-stopped" { + t.Fatalf("stopped source was not recorded at admission: %+v", first.Status.CaptureManifest) + } var job batchv1.Job if err := r.Get(t.Context(), client.ObjectKey{Namespace: "ns", Name: "first"}, &job); err != nil { t.Fatal(err) @@ -108,6 +111,10 @@ func TestBackupStoppedCopiesWithoutWakingAndReleasesForNextCopy(t *testing.T) { if got := guardGetBackup(t, r, "first"); got.Status.Phase != gameplanev1alpha1.BackupPhaseSucceeded || got.Status.SnapshotID == "" { t.Fatalf("not restorable: %+v", got.Status) } + first = guardGetBackup(t, r, "first") + if condition := backupConsistencyCondition(first); condition.Status != metav1.ConditionTrue || condition.Reason != "StoppedSource" { + t.Fatalf("stopped copy claims wrong consistency: %+v", condition) + } guardPass(t, r, "second") if err := r.Get(t.Context(), client.ObjectKey{Namespace: "ns", Name: "second"}, &second); err != nil { t.Fatal(err) diff --git a/operator/internal/controller/backup_volumesnapshot.go b/operator/internal/controller/backup_volumesnapshot.go index d9df66b0e..e90a0b6c9 100644 --- a/operator/internal/controller/backup_volumesnapshot.go +++ b/operator/internal/controller/backup_volumesnapshot.go @@ -119,6 +119,7 @@ func (r *BackupReconciler) markVolumeSnapshotRunning( Reason: string(gameplanev1alpha1.BackupPhaseRunning), ObservedGeneration: b.Generation, }) + b.Status.Conditions = upsertCondition(b.Status.Conditions, backupConsistencyCondition(b)) if err := r.Status().Update(ctx, b); err != nil { return ctrl.Result{}, err } @@ -159,6 +160,7 @@ func (r *BackupReconciler) completeVolumeSnapshot( Reason: "Succeeded", ObservedGeneration: b.Generation, }) + b.Status.Conditions = upsertCondition(b.Status.Conditions, backupConsistencyCondition(b)) if err := r.Status().Update(ctx, b); err != nil { return ctrl.Result{}, err } diff --git a/operator/internal/controller/gameserver_controller.go b/operator/internal/controller/gameserver_controller.go index 05d15006d..7f3115ea0 100644 --- a/operator/internal/controller/gameserver_controller.go +++ b/operator/internal/controller/gameserver_controller.go @@ -273,16 +273,19 @@ func (r *GameServerReconciler) Reconcile(ctx context.Context, req ctrl.Request) return ctrl.Result{}, r.setPhase(ctx, &gs, "restricted workload cannot use networking.expose=Hostport; use NodePort or LoadBalancer") } - // Resolve the template this GameServer points at. Templates are - // cluster-scoped so no namespace is needed. - var tmpl gameplanev1alpha1.GameTemplate - if err := r.Get(ctx, types.NamespacedName{Name: gs.Spec.TemplateRef.Name}, &tmpl); err != nil { - if apierrors.IsNotFound(err) { - return ctrl.Result{RequeueAfter: 15 * time.Second}, r.setPhase(ctx, &gs, - fmt.Sprintf("GameTemplate %q not found", gs.Spec.TemplateRef.Name)) + // Snapshot resolution precedes config, images, init containers, storage, + // services and every other template-derived resource. + effective, changed, err := r.reconcileTemplateRevision(ctx, &gs) + if err != nil { + if apierrors.IsConflict(err) { + return requeueOnConflict(ctrl.Result{}, err) } - return ctrl.Result{}, err + return ctrl.Result{RequeueAfter: 15 * time.Second}, r.setPhase(ctx, &gs, err.Error()) } + if changed { + return ctrl.Result{RequeueAfter: time.Second}, nil + } + tmpl := *effective // Resolve spec.config against the template's configSchema before // touching any children: invalid config must fail loudly (mirroring @@ -487,6 +490,9 @@ func (r *GameServerReconciler) Reconcile(ctx context.Context, req ctrl.Request) // or nothing fires the transition until an unrelated event happens to // wake the reconciler. hints := []time.Duration{stopRequeue, idleRequeue} + if _, held, _ := lifecycleOwner(&gs); held { + hints = append(hints, 5*time.Second) + } if plan.wantSentinel { // Bounded backstop for planSentinel's wait on the sentinel or the // game pod. Both are owned (Owns(&appsv1.Deployment{}), @@ -517,6 +523,7 @@ func (r *GameServerReconciler) SetupWithManager(mgr ctrl.Manager) error { Owns(&networkingv1.NetworkPolicy{}). Owns(&gameplanev1alpha1.BackupSchedule{}). Owns(&corev1.Secret{}). + Owns(&corev1.ConfigMap{}). Owns(&corev1.ServiceAccount{}). Owns(&rbacv1.Role{}). Owns(&rbacv1.RoleBinding{}). @@ -1241,6 +1248,12 @@ func (r *GameServerReconciler) desiredReplicas( } if op, held, err := lifecycleOwner(&live); err != nil { return 0, 0, err + } else if held && op.Kind == "Rollout" && !op.Baseline && len(op.Recovery) == 0 && op.Token != live.Annotations[effectiveTemplateAnnotation] { + replicas, poll, err := r.softStop(ctx, gs, tmpl) + if poll == 0 { + poll = restartDrainPoll + } + return replicas, poll, err } else if held && op.InitialStartup && len(op.Recovery) == 0 { if live.Spec.Suspend { return 0, 5 * time.Second, nil @@ -1248,7 +1261,7 @@ func (r *GameServerReconciler) desiredReplicas( return 1, 5 * time.Second, nil } else if held && lifecycleRecoveryNeedsStop(op) { return 0, 5 * time.Second, nil - } else if held && (op.Kind == "Backup" || len(op.Recovery) != 0) { + } else if held && (op.Kind == "Backup" || op.Baseline || len(op.Recovery) != 0) { var ss appsv1.StatefulSet err := r.wipeReader().Get(ctx, client.ObjectKeyFromObject(gs), &ss) if apierrors.IsNotFound(err) { @@ -1529,6 +1542,9 @@ func (r *GameServerReconciler) reconcileStatefulSet( if live.UID != gs.UID { return fmt.Errorf("GameServer changed identity before StatefulSet mutation") } + if live.Annotations[effectiveTemplateAnnotation] != gs.Annotations[effectiveTemplateAnnotation] { + return fmt.Errorf("effective template changed before StatefulSet mutation") + } if owner := metav1.GetControllerOf(ss); owner != nil && owner.UID != live.UID { return fmt.Errorf("StatefulSet belongs to a different GameServer identity") } @@ -1602,18 +1618,31 @@ func (r *GameServerReconciler) reconcileStatefulSet( } } } + preserveBaseline, err := r.preserveBaselinePodTemplate(ctx, &live, ss) + if err != nil { + return err + } ss.Spec.Replicas = &actualReplicas - if held && (!ss.CreationTimestamp.IsZero() || ss.ResourceVersion != "") { + if held && (op.Kind != "Rollout" || op.Token != live.Annotations[effectiveTemplateAnnotation] || len(op.Recovery) != 0) && (!ss.CreationTimestamp.IsZero() || ss.ResourceVersion != "") { // Restore and wipe may stop the existing workload, but rollout // waits until the operation and its worker cleanup have finished. return nil } + if preserveBaseline { + return nil + } ss.Spec.ServiceName = gs.Name ss.Spec.Selector = &metav1.LabelSelector{MatchLabels: labels} ss.Spec.Template.Labels = labels // Stamp (or clear) the config fingerprint without touching // annotations other actors may have set on the pod template. ann := ss.Spec.Template.Annotations + if revision := gs.Annotations[effectiveTemplateAnnotation]; revision != "" { + if ann == nil { + ann = map[string]string{} + } + ann[templateRevisionAnnotation] = revision + } if mc.hash != "" { if ann == nil { ann = map[string]string{} diff --git a/operator/internal/controller/gameserver_envtest_test.go b/operator/internal/controller/gameserver_envtest_test.go index 8c18ad7e3..bbdb2e707 100644 --- a/operator/internal/controller/gameserver_envtest_test.go +++ b/operator/internal/controller/gameserver_envtest_test.go @@ -2045,9 +2045,8 @@ func TestGameServer_ConfigFilesMaterialize(t *testing.T) { return true, "" }) - // Dropping configFiles from the template must delete the Secret and - // strip the init container + volume. Template edits don't trigger a - // GameServer reconcile (no cross-watch), so poke the GameServer. + // Dropping configFiles from the template must adopt the new revision, + // delete the Secret, and strip the init container + volume. // Retry on conflict — reconcilers patch status concurrently, racing a // bare Get+Update's resourceVersion. if err := retry.RetryOnConflict(retry.DefaultRetry, func() error { @@ -2070,14 +2069,19 @@ func TestGameServer_ConfigFilesMaterialize(t *testing.T) { t.Fatalf("update gameserver: %v", err) } eventually(t, func() (bool, string) { - // Re-poke each poll: the reconcile racing the template cache - // update must not strand the test. - poke := getGameServer(t, ns, "smp") - if poke.Annotations == nil { - poke.Annotations = map[string]string{} - } - poke.Annotations["test.gameplane.local/poke"] = time.Now().Format(time.RFC3339Nano) - _ = k8sClient.Update(context.Background(), poke) + // Envtest has no StatefulSet controller. Acknowledge only the + // actual stop request before the rollout may change its template. + var observed appsv1.StatefulSet + if err := k8sClient.Get(context.Background(), types.NamespacedName{Namespace: ns, Name: "smp"}, &observed); err != nil { + return false, "statefulset: " + err.Error() + } + if observed.Spec.Replicas != nil && *observed.Spec.Replicas == 0 && observed.Status.ObservedGeneration < observed.Generation { + observed.Status.ObservedGeneration = observed.Generation + observed.Status.Replicas, observed.Status.ReadyReplicas = 0, 0 + if err := k8sClient.Status().Update(context.Background(), &observed); err != nil { + return false, "observe stopped workload: " + err.Error() + } + } var sec corev1.Secret err := k8sClient.Get(context.Background(), diff --git a/operator/internal/controller/gameserver_lifecycle_reconcile_test.go b/operator/internal/controller/gameserver_lifecycle_reconcile_test.go index 58581c789..e4e5df6e2 100644 --- a/operator/internal/controller/gameserver_lifecycle_reconcile_test.go +++ b/operator/internal/controller/gameserver_lifecycle_reconcile_test.go @@ -52,7 +52,28 @@ func lifecycleGameServerFixture(t *testing.T, gs *gameplanev1alpha1.GameServer, objects := append([]client.Object{gs, tmpl, caSecret}, extras...) c := fake.NewClientBuilder().WithScheme(s).WithObjects(objects...). WithStatusSubresource(gs, &appsv1.StatefulSet{}, &batchv1.Job{}).Build() - return &GameServerReconciler{Client: c, APIReader: c, Scheme: s, AgentCASecretName: caSecret.Name, AgentCASecretNamespace: caSecret.Namespace} + r := &GameServerReconciler{Client: c, APIReader: c, Scheme: s, AgentCASecretName: caSecret.Name, AgentCASecretNamespace: caSecret.Namespace} + // These regressions start after template admission so their first public + // pass exercises lifecycle recovery or the intended annotation conflict. + revision, err := r.saveTemplateSnapshot(t.Context(), gs, tmpl) + if err != nil { + t.Fatal(err) + } + if gs.Annotations == nil { + gs.Annotations = map[string]string{} + } + gs.Annotations[effectiveTemplateAnnotation] = revision + if err := c.Update(t.Context(), gs); err != nil { + t.Fatal(err) + } + op, held, err := lifecycleOwner(gs) + if err != nil { + t.Fatal(err) + } + if err := r.reportTemplateRevision(t.Context(), gs, revision, op, held); err != nil { + t.Fatal(err) + } + return r } func requireNoWipeJob(t *testing.T, r *GameServerReconciler, gs *gameplanev1alpha1.GameServer) { diff --git a/operator/internal/controller/gameserver_template_revision.go b/operator/internal/controller/gameserver_template_revision.go new file mode 100644 index 000000000..53f8f8c44 --- /dev/null +++ b/operator/internal/controller/gameserver_template_revision.go @@ -0,0 +1,458 @@ +package controller + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "reflect" + + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" +) + +const ( + effectiveTemplateAnnotation = "gameplane.local/effective-template-revision" + templateRevisionAnnotation = "gameplane.local/template-revision" + templateSnapshotLabel = "gameplane.local/template-snapshot" + templateSnapshotKey = "template.json" + baselineTemplateAnnotation = "gameplane.local/template-baseline" + baselineWorkloadAnnotation = "gameplane.local/template-baseline-workload" + baselineInputsAnnotation = "gameplane.local/template-baseline-inputs" +) + +// Hash and persist content, not resourceVersion: status/metadata-only updates +// must not restart games. Name and the complete spec are all rendering inputs. +func templateSnapshot(tmpl *gameplanev1alpha1.GameTemplate) ([]byte, string, error) { + content := gameplanev1alpha1.GameTemplate{ObjectMeta: metav1.ObjectMeta{Name: tmpl.Name}, Spec: *tmpl.Spec.DeepCopy()} + data, err := json.Marshal(content) + if err != nil { + return nil, "", err + } + if len(data) > 900*1024 { + return nil, "", fmt.Errorf("template snapshot exceeds 900 KiB") + } + sum := sha256.Sum256(data) + return data, hex.EncodeToString(sum[:]), nil +} + +func validTemplateRevision(revision string) bool { + decoded, err := hex.DecodeString(revision) + return err == nil && len(decoded) == sha256.Size && hex.EncodeToString(decoded) == revision +} + +func templateSnapshotName(gs *gameplanev1alpha1.GameServer, revision string) string { + identity := sha256.Sum256([]byte(string(gs.UID) + "/" + gs.Namespace + "/" + gs.Name)) + return fmt.Sprintf("gpt-%x-%s", identity[:8], revision[:32]) +} + +func loadTemplateSnapshot(ctx context.Context, reader client.Reader, gs *gameplanev1alpha1.GameServer, revision string) (*gameplanev1alpha1.GameTemplate, error) { + if !validTemplateRevision(revision) { + return nil, fmt.Errorf("invalid effective template revision") + } + var cm corev1.ConfigMap + if err := reader.Get(ctx, client.ObjectKey{Namespace: gs.Namespace, Name: templateSnapshotName(gs, revision)}, &cm); err != nil { + return nil, fmt.Errorf("read pinned template snapshot: %w", err) + } + if !metav1.IsControlledBy(&cm, gs) || cm.Immutable == nil || !*cm.Immutable || !cm.DeletionTimestamp.IsZero() { + return nil, fmt.Errorf("template snapshot is foreign, mutable or deleting") + } + var tmpl gameplanev1alpha1.GameTemplate + if err := json.Unmarshal([]byte(cm.Data[templateSnapshotKey]), &tmpl); err != nil { + return nil, fmt.Errorf("decode template snapshot: %w", err) + } + _, digest, err := templateSnapshot(&tmpl) + if err != nil { + return nil, err + } + if digest != revision || tmpl.Name != gs.Spec.TemplateRef.Name { + return nil, fmt.Errorf("template snapshot content does not match pinned revision") + } + return &tmpl, nil +} + +// Existing pre-upgrade operations may not have a pin yet. Once a pin exists, +// absence/corruption must fail closed rather than silently reading latest. +func effectiveGameTemplate(ctx context.Context, reader client.Reader, gs *gameplanev1alpha1.GameServer) (*gameplanev1alpha1.GameTemplate, error) { + if revision := gs.Annotations[effectiveTemplateAnnotation]; revision != "" { + return loadTemplateSnapshot(ctx, reader, gs, revision) + } + var tmpl gameplanev1alpha1.GameTemplate + err := reader.Get(ctx, client.ObjectKey{Name: gs.Spec.TemplateRef.Name}, &tmpl) + return &tmpl, err +} + +func (r *GameServerReconciler) saveTemplateSnapshot(ctx context.Context, gs *gameplanev1alpha1.GameServer, tmpl *gameplanev1alpha1.GameTemplate) (string, error) { + data, revision, err := templateSnapshot(tmpl) + if err != nil { + return "", err + } + immutable := true + cm := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: templateSnapshotName(gs, revision), Namespace: gs.Namespace, Labels: map[string]string{templateSnapshotLabel: string(gs.UID)}}, Immutable: &immutable, Data: map[string]string{templateSnapshotKey: string(data)}} + if err := controllerutil.SetControllerReference(gs, cm, r.Scheme); err != nil { + return "", err + } + if err := r.Create(ctx, cm); err != nil && !apierrors.IsAlreadyExists(err) { + return "", err + } + _, err = loadTemplateSnapshot(ctx, r.wipeReader(), gs, revision) + return revision, err +} + +// reconcileTemplateRevision runs before every template-dependent calculation. +// A true second result asks the caller to re-read admission state next pass. +func (r *GameServerReconciler) reconcileTemplateRevision(ctx context.Context, gs *gameplanev1alpha1.GameServer) (*gameplanev1alpha1.GameTemplate, bool, error) { + var installed gameplanev1alpha1.GameTemplate + err := r.wipeReader().Get(ctx, client.ObjectKey{Name: gs.Spec.TemplateRef.Name}, &installed) + if err != nil && !apierrors.IsNotFound(err) { + return nil, false, err + } + desired := "" + if err == nil { + _, desired, err = templateSnapshot(&installed) + if err != nil { + return nil, false, err + } + } + pin := gs.Annotations[effectiveTemplateAnnotation] + var effective *gameplanev1alpha1.GameTemplate + if pin != "" { + effective, err = loadTemplateSnapshot(ctx, r.wipeReader(), gs, pin) + if err != nil { + return nil, false, err + } + } else if desired != "" { + effective = &installed + } + op, held, err := lifecycleOwner(gs) + if err != nil { + return nil, false, err + } + if pin == "" && held && op.Kind == "Rollout" { + effective, err = loadTemplateSnapshot(ctx, r.wipeReader(), gs, op.Token) + if err != nil { + return nil, false, err + } + } + if effective == nil { + return nil, false, fmt.Errorf("GameTemplate %q not found", gs.Spec.TemplateRef.Name) + } + if err := r.reportTemplateRevision(ctx, gs, desired, op, held); err != nil { + return nil, false, err + } + if pin == "" && gs.Annotations[restoreTemplateAnnotation] != "" { + // The Restore must copy its durable source snapshot before even a + // stopped StatefulSet is rendered, otherwise InitialStartup could + // preserve a workload derived from the newest catalog template. + return effective, true, nil + } + if held && op.Kind == "Rollout" { + ready, err := recoverLifecycleOperation(ctx, r.Client, r.wipeReader(), gs, op, r.OperationQuiescer) + if err != nil || !ready { + return effective, true, err + } + if op.Baseline { + return effective, true, r.initializeTemplateBaseline(ctx, gs, op) + } + if op.Token != pin { + // The admitted target survives a subsequent catalog update. + target, err := loadTemplateSnapshot(ctx, r.wipeReader(), gs, op.Token) + if err != nil { + return nil, false, err + } + if _, err := materializeConfig(gs, target); err != nil { + return nil, false, err + } + if _, err := resolveVersion(gs, target); err != nil { + return nil, false, err + } + if r.RestrictedWorkloads { + if err := validateRestrictedTemplate(target); err != nil { + return nil, false, err + } + } + drained, err := r.templateWorkloadDrained(ctx, gs) + if err != nil || !drained { + return effective, false, err + } + gs.Annotations[effectiveTemplateAnnotation] = op.Token + delete(gs.Annotations, stopRequestedAtAnnotation) + return effective, true, r.Update(ctx, gs) + } + observed, err := r.templateWorkloadAdopted(ctx, gs, pin) + if err != nil || !observed { + return effective, false, err + } + // Clean obsolete content before releasing, including when Automatic + // already has another catalog revision queued for the next pass. + if err := r.pruneTemplateSnapshots(ctx, gs); err != nil { + return nil, false, err + } + if err := clearLifecycleOperation(ctx, r.Client, r.wipeReader(), gs, op.lifecycleIdentity); err != nil { + return nil, false, err + } + return effective, true, r.Update(ctx, gs) + } + // Existing owners continue rendering the old pin; initial startup of a + // restore may proceed even before the first post-upgrade pin is recorded. + if held { + return effective, false, nil + } + if desired != "" && desired != pin { + policy := gs.Spec.TemplateRollout + admit := pin == "" || policy == nil || policy.Policy == "" || policy.Policy == "Automatic" || policy.RequestedRevision == desired + if admit { + if _, err := materializeConfig(gs, &installed); err != nil { + return nil, false, err + } + if _, err := resolveVersion(gs, &installed); err != nil { + return nil, false, err + } + if r.RestrictedWorkloads { + if err := validateRestrictedTemplate(&installed); err != nil { + return nil, false, err + } + } + // Do not replace an unrelated restart or already-issued stop. + pending, err := backupLifecycleTransitionPending(ctx, r.wipeReader(), gs) + if err != nil || pending { + return effective, false, err + } + revision, err := r.saveTemplateSnapshot(ctx, gs, &installed) + if err != nil { + return nil, false, err + } + want := lifecycleIdentity{Kind: "Rollout", Name: gs.Name, UID: gs.UID, TargetUID: gs.UID, Generation: gs.Generation, Token: revision, Baseline: pin == ""} + claimed, err := claimLifecycleOperation(ctx, r.Client, r.wipeReader(), gs, want, r.OperationQuiescer) + return effective, claimed, err + } + } + return effective, false, r.pruneTemplateSnapshots(ctx, gs) +} + +// Baseline captures render inputs rather than comparing reconstructed PodSpecs: +// API defaults/admission fields must not turn feature installation into a fleet +// rollout. These fields deliberately exclude power intent, service-only config, +// capture opt-in, scheduling backups and the rollout policy itself. +func (r *GameServerReconciler) baselineRenderInputs(gs *gameplanev1alpha1.GameServer, revision string) (string, error) { + spec := gs.Spec.DeepCopy() + spec.Suspend = false + spec.StopGracePeriodSeconds, spec.TemplateRollout, spec.BackupPolicy, spec.Idle, spec.Capture = nil, nil, nil, nil, nil + if spec.Networking.Expose == "Hostport" { + spec.Networking = gameplanev1alpha1.GameServerNetworking{Expose: "Hostport"} + } else { + spec.Networking = gameplanev1alpha1.GameServerNetworking{} + } + data, err := json.Marshal(struct { + Spec *gameplanev1alpha1.GameServerSpec + Revision, AgentImage, AgentLogLevel, ConfigInitImage string + AgentImagePullPolicy string + RestrictedWorkloads bool + AgentTokenExpirationSeconds int64 + AgentResources *corev1.ResourceRequirements + }{spec, revision, r.AgentImage, r.AgentLogLevel, r.ConfigInitImage, r.AgentImagePullPolicy, r.RestrictedWorkloads, r.AgentTokenExpirationSeconds, r.AgentResources}) + if err != nil { + return "", err + } + sum := sha256.Sum256(data) + return hex.EncodeToString(sum[:]), nil +} + +func (r *GameServerReconciler) initializeTemplateBaseline(ctx context.Context, gs *gameplanev1alpha1.GameServer, op lifecycleOperation) error { + if _, err := loadTemplateSnapshot(ctx, r.wipeReader(), gs, op.Token); err != nil { + return err + } + var ss appsv1.StatefulSet + err := r.wipeReader().Get(ctx, client.ObjectKeyFromObject(gs), &ss) + if err != nil && !apierrors.IsNotFound(err) { + return err + } + if err == nil { + if !metav1.IsControlledBy(&ss, gs) { + return fmt.Errorf("baseline workload belongs to another server") + } + // A real spec edit after admission must render normally. Recording + // its new inputs against the old workload would silently swallow it. + if gs.Generation == op.Generation { + inputs, err := r.baselineRenderInputs(gs, op.Token) + if err != nil { + return err + } + gs.Annotations[baselineTemplateAnnotation] = op.Token + gs.Annotations[baselineWorkloadAnnotation] = string(ss.UID) + gs.Annotations[baselineInputsAnnotation] = inputs + } + } + gs.Annotations[effectiveTemplateAnnotation] = op.Token + if err := clearLifecycleOperation(ctx, r.Client, r.wipeReader(), gs, op.lifecycleIdentity); err != nil { + return err + } + return r.Update(ctx, gs) +} + +func (r *GameServerReconciler) preserveBaselinePodTemplate(ctx context.Context, gs *gameplanev1alpha1.GameServer, ss *appsv1.StatefulSet) (bool, error) { + pin := gs.Annotations[effectiveTemplateAnnotation] + if pin == "" || gs.Annotations[baselineTemplateAnnotation] != pin || string(ss.UID) != gs.Annotations[baselineWorkloadAnnotation] || ss.Spec.Template.Annotations[templateRevisionAnnotation] != "" { + return false, nil + } + inputs, err := r.baselineRenderInputs(gs, pin) + if err != nil || inputs != gs.Annotations[baselineInputsAnnotation] { + return false, err + } + // Provenance initialization must not suppress a real security upgrade. + // This also covers installing credential isolation and revisions together. + if ss.Spec.Template.Spec.AutomountServiceAccountToken == nil || *ss.Spec.Template.Spec.AutomountServiceAccountToken || r.RestrictedWorkloads != (ss.Spec.Template.Annotations[workloadSecurityLabel] == "restricted") { + return false, nil + } + projected := false + for _, volume := range ss.Spec.Template.Spec.Volumes { + if volume.Name == agentTokenVolumeName && volume.Projected != nil { + projected = true + } + } + if !projected { + return false, nil + } + // Stamp while already drained, before resume; never stamp an existing + // unannotated Pod or use its name as evidence of a known running revision. + if ss.Spec.Replicas != nil && *ss.Spec.Replicas == 0 && ss.Status.Replicas == 0 && ss.Status.ObservedGeneration >= ss.Generation { + var pod corev1.Pod + err := r.wipeReader().Get(ctx, client.ObjectKey{Namespace: gs.Namespace, Name: gs.Name + "-0"}, &pod) + if apierrors.IsNotFound(err) { + return false, nil + } + if err != nil { + return false, err + } + } + return true, nil +} + +func (r *GameServerReconciler) reportTemplateRevision(ctx context.Context, gs *gameplanev1alpha1.GameServer, desired string, op lifecycleOperation, held bool) error { + running, err := observedTemplateRevision(ctx, r.wipeReader(), gs) + if err != nil { + return err + } + base := gs.DeepCopy() + gs.Status.TemplateRevision = &gameplanev1alpha1.TemplateRevisionStatus{Desired: desired, Effective: gs.Annotations[effectiveTemplateAnnotation], Running: running} + switch { + case held && op.Kind == "Rollout": + setTemplateCondition(gs, metav1.ConditionTrue, "AdoptingRevision", "The admitted template revision is being adopted.") + case desired == "": + setTemplateCondition(gs, metav1.ConditionUnknown, "TemplateUnavailable", "The installed template is unavailable; the effective snapshot is retained.") + case desired == gs.Annotations[effectiveTemplateAnnotation]: + setTemplateCondition(gs, metav1.ConditionFalse, "RevisionCurrent", "The installed template revision is effective.") + case held: + setTemplateCondition(gs, metav1.ConditionTrue, "OperationInProgress", "Template adoption waits for the active lifecycle operation to finish.") + default: + setTemplateCondition(gs, metav1.ConditionTrue, "RevisionAvailable", "An installed template revision is waiting for admission.") + } + if reflect.DeepEqual(base.Status, gs.Status) { + return nil + } + return r.Status().Patch(ctx, gs, client.MergeFromWithOptions(base, client.MergeFromWithOptimisticLock{})) +} + +func setTemplateCondition(gs *gameplanev1alpha1.GameServer, status metav1.ConditionStatus, reason, message string) { + meta.SetStatusCondition(&gs.Status.Conditions, metav1.Condition{Type: "TemplateUpdatePending", Status: status, Reason: reason, Message: message, ObservedGeneration: gs.Generation}) +} + +func ownedTemplatePod(ctx context.Context, reader client.Reader, gs *gameplanev1alpha1.GameServer) (*corev1.Pod, error) { + var ss appsv1.StatefulSet + if err := reader.Get(ctx, client.ObjectKeyFromObject(gs), &ss); err != nil { + return nil, client.IgnoreNotFound(err) + } + if !metav1.IsControlledBy(&ss, gs) { + return nil, nil + } + var pod corev1.Pod + if err := reader.Get(ctx, client.ObjectKey{Namespace: gs.Namespace, Name: gs.Name + "-0"}, &pod); err != nil { + return nil, client.IgnoreNotFound(err) + } + if !metav1.IsControlledBy(&pod, &ss) || !pod.DeletionTimestamp.IsZero() { + return nil, nil + } + return &pod, nil +} + +func observedTemplateRevision(ctx context.Context, reader client.Reader, gs *gameplanev1alpha1.GameServer) (string, error) { + pod, err := ownedTemplatePod(ctx, reader, gs) + if err != nil || pod == nil { + return "", err + } + revision := pod.Annotations[templateRevisionAnnotation] + if !validTemplateRevision(revision) { + return "", nil + } + for _, container := range pod.Status.ContainerStatuses { + if container.Name == "game" && container.State.Running != nil { + return revision, nil + } + } + return "", nil +} + +func (r *GameServerReconciler) templateWorkloadDrained(ctx context.Context, gs *gameplanev1alpha1.GameServer) (bool, error) { + var ss appsv1.StatefulSet + if err := r.wipeReader().Get(ctx, client.ObjectKeyFromObject(gs), &ss); err != nil { + return false, client.IgnoreNotFound(err) + } + if !metav1.IsControlledBy(&ss, gs) { + return false, fmt.Errorf("template rollout workload belongs to another server") + } + if ss.Annotations[lifecycleOperationAnnotation] != gs.Annotations[lifecycleOperationAnnotation] || ss.Spec.Replicas == nil || *ss.Spec.Replicas != 0 || ss.Status.Replicas != 0 || ss.Status.ObservedGeneration < ss.Generation { + return false, nil + } + var pods corev1.PodList + if err := r.wipeReader().List(ctx, &pods, client.InNamespace(gs.Namespace)); err != nil { + return false, err + } + for i := range pods.Items { + pod := &pods.Items[i] + if pod.Name == gs.Name+"-0" || metav1.IsControlledBy(pod, &ss) || (liveRestorePod(pod) && podMountsRestoreTarget(pod, gs.Name)) { + return false, nil + } + } + return true, nil +} + +func (r *GameServerReconciler) templateWorkloadAdopted(ctx context.Context, gs *gameplanev1alpha1.GameServer, revision string) (bool, error) { + var ss appsv1.StatefulSet + if err := r.wipeReader().Get(ctx, client.ObjectKeyFromObject(gs), &ss); err != nil { + return false, client.IgnoreNotFound(err) + } + if !metav1.IsControlledBy(&ss, gs) || ss.Spec.Template.Annotations[templateRevisionAnnotation] != revision || ss.Status.ObservedGeneration < ss.Generation { + return false, nil + } + if ss.Spec.Replicas != nil && *ss.Spec.Replicas == 0 { + return r.templateWorkloadDrained(ctx, gs) + } + pod, err := ownedTemplatePod(ctx, r.wipeReader(), gs) + return pod != nil && pod.Annotations[templateRevisionAnnotation] == revision, err +} + +func (r *GameServerReconciler) pruneTemplateSnapshots(ctx context.Context, gs *gameplanev1alpha1.GameServer) error { + var snapshots corev1.ConfigMapList + if err := r.wipeReader().List(ctx, &snapshots, client.InNamespace(gs.Namespace), client.MatchingLabels{templateSnapshotLabel: string(gs.UID)}); err != nil { + return err + } + pin := gs.Annotations[effectiveTemplateAnnotation] + for i := range snapshots.Items { + cm := &snapshots.Items[i] + if !metav1.IsControlledBy(cm, gs) || (validTemplateRevision(pin) && cm.Name == templateSnapshotName(gs, pin)) { + continue + } + if err := r.Delete(ctx, cm, client.Preconditions{UID: &cm.UID, ResourceVersion: &cm.ResourceVersion}); client.IgnoreNotFound(err) != nil { + return err + } + } + return nil +} diff --git a/operator/internal/controller/gameserver_template_revision_test.go b/operator/internal/controller/gameserver_template_revision_test.go new file mode 100644 index 000000000..46706339c --- /dev/null +++ b/operator/internal/controller/gameserver_template_revision_test.go @@ -0,0 +1,732 @@ +package controller + +import ( + "context" + "encoding/json" + "testing" + + appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" +) + +func revisionFixture(t *testing.T) (*GameServerReconciler, *gameplanev1alpha1.GameServer, *gameplanev1alpha1.GameTemplate) { + t.Helper() + s := testScheme(t) + if err := batchv1.AddToScheme(s); err != nil { + t.Fatal(err) + } + gs := &gameplanev1alpha1.GameServer{ObjectMeta: metav1.ObjectMeta{Name: "game", Namespace: "games", UID: "server-uid"}, Spec: gameplanev1alpha1.GameServerSpec{TemplateRef: gameplanev1alpha1.GameTemplateRef{Name: "template"}, TemplateRollout: &gameplanev1alpha1.TemplateRolloutSpec{Policy: "OnRequest"}}} + tmpl := &gameplanev1alpha1.GameTemplate{ObjectMeta: metav1.ObjectMeta{Name: "template"}, Spec: gameplanev1alpha1.GameTemplateSpec{Image: "example.invalid/game:old", Env: []corev1.EnvVar{{Name: "REVISION", Value: "old"}}}} + c := fake.NewClientBuilder().WithScheme(s).WithObjects(gs, tmpl).WithStatusSubresource(gs, &appsv1.StatefulSet{}, &gameplanev1alpha1.Backup{}).Build() + r := &GameServerReconciler{Client: c, APIReader: c, Scheme: s} + digest, err := r.saveTemplateSnapshot(t.Context(), gs, tmpl) + if err != nil { + t.Fatal(err) + } + gs.Annotations = map[string]string{effectiveTemplateAnnotation: digest} + if err := c.Update(t.Context(), gs); err != nil { + t.Fatal(err) + } + return r, gs, tmpl +} + +func TestTemplateOnRequestRetainsCompleteContents(t *testing.T) { + r, gs, installed := revisionFixture(t) + old := gs.Annotations[effectiveTemplateAnnotation] + installed.Spec.Image, installed.Spec.Env[0].Value = "example.invalid/game:new", "new" + if err := r.Update(t.Context(), installed); err != nil { + t.Fatal(err) + } + effective, _, err := r.reconcileTemplateRevision(t.Context(), gs) + if err != nil { + t.Fatal(err) + } + if effective.Spec.Image != "example.invalid/game:old" || effective.Spec.Env[0].Value != "old" { + t.Fatalf("render escaped snapshot: %#v", effective.Spec) + } + if gs.Status.TemplateRevision.Effective != old || gs.Status.TemplateRevision.Desired == old { + t.Fatalf("untruthful revisions: %#v", gs.Status.TemplateRevision) + } + if c := meta.FindStatusCondition(gs.Status.Conditions, "TemplateUpdatePending"); c == nil || c.Status != metav1.ConditionTrue { + t.Fatalf("missing pending condition: %v", c) + } + // Recreating the reconciler and clearing status must not discard the pin. + gs.Status = gameplanev1alpha1.GameServerStatus{} + if err := r.Status().Update(t.Context(), gs); err != nil { + t.Fatal(err) + } + r = &GameServerReconciler{Client: r.Client, APIReader: r.Client, Scheme: r.Scheme} + effective, _, err = r.reconcileTemplateRevision(t.Context(), gs) + if err != nil || effective.Spec.Image != "example.invalid/game:old" { + t.Fatalf("restart lost snapshot: %v, %v", effective, err) + } +} + +func TestTemplateSnapshotRejectsMissingCorruptAndForeign(t *testing.T) { + for _, mode := range []string{"missing", "corrupt", "foreign"} { + t.Run(mode, func(t *testing.T) { + r, gs, _ := revisionFixture(t) + var cm corev1.ConfigMap + key := client.ObjectKey{Namespace: gs.Namespace, Name: templateSnapshotName(gs, gs.Annotations[effectiveTemplateAnnotation])} + if err := r.Get(t.Context(), key, &cm); err != nil { + t.Fatal(err) + } + switch mode { + case "missing": + if err := r.Delete(t.Context(), &cm); err != nil { + t.Fatal(err) + } + case "corrupt": + cm.Data[templateSnapshotKey] = `{}` + if err := r.Update(t.Context(), &cm); err != nil { + t.Fatal(err) + } + case "foreign": + cm.OwnerReferences[0].UID = "other-server" + if err := r.Update(t.Context(), &cm); err != nil { + t.Fatal(err) + } + } + if _, err := effectiveGameTemplate(t.Context(), r.Client, gs); err == nil { + t.Fatal("invalid snapshot fell through to live template") + } + }) + } +} + +func TestTemplateRequestMustMatchInstalledDigest(t *testing.T) { + r, gs, installed := revisionFixture(t) + gs.Spec.TemplateRollout.RequestedRevision = gs.Annotations[effectiveTemplateAnnotation] + if err := r.Update(t.Context(), gs); err != nil { + t.Fatal(err) + } + installed.Spec.Image = "example.invalid/game:new" + if err := r.Update(t.Context(), installed); err != nil { + t.Fatal(err) + } + if _, _, err := r.reconcileTemplateRevision(t.Context(), gs); err != nil { + t.Fatal(err) + } + if _, held, _ := lifecycleOwner(gs); held { + t.Fatal("old request admitted a newer template") + } +} + +func TestTemplateAdoptionWaitsForLifecycleOwner(t *testing.T) { + for _, kind := range []string{"Backup", "Restore", "Wipe"} { + t.Run(kind, func(t *testing.T) { + r, gs, installed := revisionFixture(t) + gs.Spec.TemplateRollout.Policy = "Automatic" + op := lifecycleIdentity{Kind: kind, Name: "owner", UID: "owner-uid", TargetUID: gs.UID, Token: "request"} + gs.Annotations[lifecycleOperationAnnotation] = lifecycleOperationJSON(lifecycleOperation{lifecycleIdentity: op}) + if err := r.Update(t.Context(), gs); err != nil { + t.Fatal(err) + } + installed.Spec.Image = "example.invalid/game:new" + if err := r.Update(t.Context(), installed); err != nil { + t.Fatal(err) + } + effective, _, err := r.reconcileTemplateRevision(t.Context(), gs) + if err != nil || effective.Spec.Image != "example.invalid/game:old" { + t.Fatalf("protected operation adopted template: %v", err) + } + owner, _, _ := lifecycleOwner(gs) + if owner.Kind != kind { + t.Fatal("stole lifecycle owner") + } + }) + } +} + +func TestRunningTemplateRequiresOwnedRunningGame(t *testing.T) { + r, gs, _ := revisionFixture(t) + digest := gs.Annotations[effectiveTemplateAnnotation] + ss := &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{Name: gs.Name, Namespace: gs.Namespace, UID: "workload-uid", OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(gs, gameplanev1alpha1.GroupVersion.WithKind("GameServer"))}}} + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: gs.Name + "-0", Namespace: gs.Namespace, Annotations: map[string]string{templateRevisionAnnotation: digest}, OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(ss, appsv1.SchemeGroupVersion.WithKind("StatefulSet"))}}} + if err := r.Create(t.Context(), ss); err != nil { + t.Fatal(err) + } + if err := r.Create(t.Context(), pod); err != nil { + t.Fatal(err) + } + if got, err := observedTemplateRevision(t.Context(), r.Client, gs); err != nil || got != "" { + t.Fatalf("pending Pod reported running: %q %v", got, err) + } + pod.Status.ContainerStatuses = []corev1.ContainerStatus{{Name: "game", State: corev1.ContainerState{Running: &corev1.ContainerStateRunning{}}}} + if err := r.Status().Update(t.Context(), pod); err != nil { + t.Fatal(err) + } + if got, err := observedTemplateRevision(t.Context(), r.Client, gs); err != nil || got != digest { + t.Fatalf("running Pod not observed: %q %v", got, err) + } + pod.OwnerReferences[0].UID = types.UID("replaced-workload") + if err := r.Update(t.Context(), pod); err != nil { + t.Fatal(err) + } + if got, err := observedTemplateRevision(context.Background(), r.Client, gs); err != nil || got != "" { + t.Fatalf("foreign Pod reported running: %q %v", got, err) + } +} + +func TestTemplateAdoptionDrainsAndKeepsAdmittedRevisionAcrossRestart(t *testing.T) { + r, gs, installed := revisionFixture(t) + old := gs.Annotations[effectiveTemplateAnnotation] + zero, one := int32(0), int32(1) + ss := &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{Name: gs.Name, Namespace: gs.Namespace, UID: "workload", OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(gs, gameplanev1alpha1.GroupVersion.WithKind("GameServer"))}}, Spec: appsv1.StatefulSetSpec{Replicas: &one}, Status: appsv1.StatefulSetStatus{Replicas: 1}} + if err := r.Create(t.Context(), ss); err != nil { + t.Fatal(err) + } + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: gs.Name + "-0", Namespace: gs.Namespace, UID: "old-pod", OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(ss, appsv1.SchemeGroupVersion.WithKind("StatefulSet"))}}} + if err := r.Create(t.Context(), pod); err != nil { + t.Fatal(err) + } + installed.Spec.Image = "example.invalid/game:admitted" + if err := r.Update(t.Context(), installed); err != nil { + t.Fatal(err) + } + _, target, _ := templateSnapshot(installed) + gs.Spec.TemplateRollout.RequestedRevision = target + if err := r.Update(t.Context(), gs); err != nil { + t.Fatal(err) + } + if _, changed, err := r.reconcileTemplateRevision(t.Context(), gs); err != nil || !changed { + t.Fatalf("admission: %v %v", changed, err) + } + owner, held, _ := lifecycleOwner(gs) + if !held || owner.Kind != "Rollout" || owner.Token != target { + t.Fatalf("wrong owner: %#v", owner) + } + // A new leader and a newer catalog must retain the admitted target. + r = &GameServerReconciler{Client: r.Client, APIReader: r.Client, Scheme: r.Scheme} + installed.Spec.Image = "example.invalid/game:later" + if err := r.Update(t.Context(), installed); err != nil { + t.Fatal(err) + } + effective, changed, err := r.reconcileTemplateRevision(t.Context(), gs) + if err != nil || changed || effective.Spec.Image != "example.invalid/game:old" { + t.Fatalf("adopted before drain: %v %v", changed, err) + } + ss.Spec.Replicas = &zero + ss.Annotations = map[string]string{lifecycleOperationAnnotation: gs.Annotations[lifecycleOperationAnnotation]} + if err := r.Update(t.Context(), ss); err != nil { + t.Fatal(err) + } + ss.Status.Replicas = 0 + ss.Status.ObservedGeneration = ss.Generation + if err := r.Status().Update(t.Context(), ss); err != nil { + t.Fatal(err) + } + if _, changed, err := r.reconcileTemplateRevision(t.Context(), gs); err != nil || changed || gs.Annotations[effectiveTemplateAnnotation] != old { + t.Fatalf("live old Pod escaped drain: %v %v", changed, err) + } + if err := r.Delete(t.Context(), pod); err != nil { + t.Fatal(err) + } + if _, changed, err := r.reconcileTemplateRevision(t.Context(), gs); err != nil || !changed || gs.Annotations[effectiveTemplateAnnotation] != target { + t.Fatalf("pin not committed: %v %v", changed, err) + } + effective, err = effectiveGameTemplate(t.Context(), r.Client, gs) + if err != nil || effective.Spec.Image != "example.invalid/game:admitted" { + t.Fatalf("catalog replaced admitted content: %v", err) + } + // Suspended adoption releases after observing the new stopped template. + gs.Spec.Suspend = true + if err := r.Update(t.Context(), gs); err != nil { + t.Fatal(err) + } + ss.Spec.Template.Annotations = map[string]string{templateRevisionAnnotation: target} + if err := r.Update(t.Context(), ss); err != nil { + t.Fatal(err) + } + ss.Status.ObservedGeneration = ss.Generation + if err := r.Status().Update(t.Context(), ss); err != nil { + t.Fatal(err) + } + if _, changed, err := r.reconcileTemplateRevision(t.Context(), gs); err != nil || !changed { + t.Fatalf("release: %v %v", changed, err) + } + if _, held, _ := lifecycleOwner(gs); held || !gs.Spec.Suspend { + t.Fatal("owner retained or user suspend lost") + } + if _, _, err := r.reconcileTemplateRevision(t.Context(), gs); err != nil { + t.Fatal(err) + } + if gs.Status.TemplateRevision.Running != "" || gs.Status.TemplateRevision.Desired == target { + t.Fatalf("untruthful suspended/latest status: %#v", gs.Status.TemplateRevision) + } + var snapshots corev1.ConfigMapList + if err := r.List(t.Context(), &snapshots); err != nil { + t.Fatal(err) + } + if len(snapshots.Items) != 1 { + t.Fatalf("obsolete snapshots retained: %d", len(snapshots.Items)) + } +} + +func TestPinnedTemplateDrivesConfigInitStorageAndImage(t *testing.T) { + r, gs, old := revisionFixture(t) + old.Spec.Storage = gameplanev1alpha1.GameStorageSpec{MountPath: "/old-data", Size: resource.MustParse("1Gi"), Extra: []gameplanev1alpha1.ExtraVolumeSpec{{Name: "old", MountPath: "/old-extra", Size: resource.MustParse("1Gi")}}} + old.Spec.ConfigFiles = []gameplanev1alpha1.ConfigFile{{Path: "revision.txt", Template: "old contents"}} + revision, err := r.saveTemplateSnapshot(t.Context(), gs, old) + if err != nil { + t.Fatal(err) + } + gs.Annotations[effectiveTemplateAnnotation] = revision + if err := r.Update(t.Context(), gs); err != nil { + t.Fatal(err) + } + latest := old.DeepCopy() + latest.Spec.Image, latest.Spec.Storage.MountPath = "example.invalid/game:new", "/new-data" + latest.Spec.Storage.Extra[0].Name, latest.Spec.Storage.Extra[0].MountPath = "new", "/new-extra" + latest.Spec.ConfigFiles[0].Template = "new contents" + if err := r.Update(t.Context(), latest); err != nil { + t.Fatal(err) + } + effective, _, err := r.reconcileTemplateRevision(t.Context(), gs) + if err != nil { + t.Fatal(err) + } + mc, err := materializeConfig(gs, effective) + if err != nil { + t.Fatal(err) + } + if err := r.reconcilePVC(t.Context(), gs, effective); err != nil { + t.Fatal(err) + } + if err := r.reconcileExtraPVCs(t.Context(), gs, effective); err != nil { + t.Fatal(err) + } + if err := r.reconcileFilesSecret(t.Context(), gs, mc); err != nil { + t.Fatal(err) + } + if err := r.reconcileStatefulSet(t.Context(), gs, effective, nil, mc, 1); err != nil { + t.Fatal(err) + } + var ss appsv1.StatefulSet + if err := r.Get(t.Context(), client.ObjectKeyFromObject(gs), &ss); err != nil { + t.Fatal(err) + } + if ss.Spec.Template.Spec.Containers[0].Image != old.Spec.Image { + t.Fatal("image used latest") + } + for _, container := range append(ss.Spec.Template.Spec.Containers, ss.Spec.Template.Spec.InitContainers...) { + for _, mount := range container.VolumeMounts { + if mount.MountPath == "/new-data" || mount.MountPath == "/new-extra" { + t.Fatalf("mount escaped snapshot: %#v", mount) + } + } + } + if len(ss.Spec.Template.Spec.InitContainers) == 0 { + t.Fatal("pinned config-init absent") + } + var files corev1.Secret + if err := r.Get(t.Context(), client.ObjectKey{Namespace: gs.Namespace, Name: gs.Name + "-files"}, &files); err != nil { + t.Fatal(err) + } + found := false + for _, value := range files.Data { + if string(value) == "old contents" { + found = true + } + } + if !found { + t.Fatalf("files used latest: %#v", files.Data) + } + var pvcs corev1.PersistentVolumeClaimList + if err := r.List(t.Context(), &pvcs); err != nil { + t.Fatal(err) + } + if len(pvcs.Items) != 2 { + t.Fatalf("wrong effective PVC set: %d", len(pvcs.Items)) + } + for _, pvc := range pvcs.Items { + if pvc.Name == gs.Name+"-extra-new" { + t.Fatal("pending revision provisioned new extra storage") + } + } +} + +func TestSnapshotRestorePinsEffectiveSourceBeforeStartup(t *testing.T) { + r, source, installed := revisionFixture(t) + rs := &gameplanev1alpha1.Restore{ObjectMeta: metav1.ObjectMeta{Name: "restore", Namespace: source.Namespace, UID: "restore-uid"}} + if err := r.Create(t.Context(), rs); err != nil { + t.Fatal(err) + } + target := source.DeepCopy() + target.Name, target.UID, target.ResourceVersion = "restored", "target-uid", "" + target.Annotations = map[string]string{} + restorer := &RestoreReconciler{Client: r.Client, APIReader: r.Client, Scheme: r.Scheme} + if err := restorer.planRestoredTemplate(t.Context(), rs, source, target); err != nil { + t.Fatal(err) + } + if err := r.Create(t.Context(), target); err != nil { + t.Fatal(err) + } + installed.Spec.Image = "example.invalid/game:new" + if err := r.Update(t.Context(), installed); err != nil { + t.Fatal(err) + } + if _, wait, err := r.reconcileTemplateRevision(t.Context(), target); err != nil || !wait { + t.Fatalf("target bypassed snapshot copy: wait=%v err=%v", wait, err) + } + if err := restorer.copyRestoredTemplate(t.Context(), rs, target); err != nil { + t.Fatal(err) + } + effective, err := effectiveGameTemplate(t.Context(), r.Client, target) + if err != nil || effective.Spec.Image != "example.invalid/game:old" { + t.Fatalf("restore adopted latest content: %v", err) + } +} + +type revisionStopper struct{ calls int } + +func (s *revisionStopper) Stop(context.Context, string, string) error { s.calls++; return nil } +func (*revisionStopper) Enabled() bool { return true } + +func TestTemplateRolloutUsesOldGracefulStopAndBlocksBackup(t *testing.T) { + r, gs, old := revisionFixture(t) + old.Spec.RCON = &gameplanev1alpha1.RCONSpec{Protocol: "source"} + old.Spec.Capabilities = &gameplanev1alpha1.CapabilitiesSpec{Lifecycle: &gameplanev1alpha1.LifecycleSpec{Stop: []string{"save", "stop"}}} + pin, err := r.saveTemplateSnapshot(t.Context(), gs, old) + if err != nil { + t.Fatal(err) + } + gs.Annotations[effectiveTemplateAnnotation] = pin + gs.Spec.TemplateRollout.Policy = "Automatic" + if err := r.Update(t.Context(), gs); err != nil { + t.Fatal(err) + } + newTemplate := old.DeepCopy() + newTemplate.Spec.Image = "example.invalid/game:new" + newTemplate.Spec.Capabilities = nil // latest no longer declares graceful stop + if err := r.Update(t.Context(), newTemplate); err != nil { + t.Fatal(err) + } + if _, changed, err := r.reconcileTemplateRevision(t.Context(), gs); err != nil || !changed { + t.Fatalf("automatic admission: %v %v", changed, err) + } + if claimed, err := claimLifecycleOperation(t.Context(), r.Client, r.Client, gs, lifecycleIdentity{Kind: "Backup", Name: "copy", UID: "copy-uid", TargetUID: gs.UID}, nil); err != nil || claimed { + t.Fatalf("backup entered rollout: %v %v", claimed, err) + } + one := int32(1) + ss := &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{Name: gs.Name, Namespace: gs.Namespace, UID: "workload", OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(gs, gameplanev1alpha1.GroupVersion.WithKind("GameServer"))}}, Spec: appsv1.StatefulSetSpec{Replicas: &one}, Status: appsv1.StatefulSetStatus{ReadyReplicas: 1, Replicas: 1}} + if err := r.Create(t.Context(), ss); err != nil { + t.Fatal(err) + } + stopper := &revisionStopper{} + r.AgentClient = stopper + effective, _, err := r.reconcileTemplateRevision(t.Context(), gs) + if err != nil { + t.Fatal(err) + } + replicas, poll, err := r.desiredReplicas(t.Context(), gs, effective, idleAwake) + if err != nil || replicas != 1 || poll <= 0 || stopper.calls != 1 { + t.Fatalf("old graceful contract bypassed: replicas=%d poll=%v calls=%d err=%v", replicas, poll, stopper.calls, err) + } +} + +func TestPinnedTemplateSurvivesInstalledTemplateDeletion(t *testing.T) { + r, gs, installed := revisionFixture(t) + if err := r.Delete(t.Context(), installed); err != nil { + t.Fatal(err) + } + effective, _, err := r.reconcileTemplateRevision(t.Context(), gs) + if err != nil || effective.Spec.Image != "example.invalid/game:old" { + t.Fatalf("installed deletion lost pinned content: %v", err) + } + if gs.Status.TemplateRevision.Desired != "" { + t.Fatal("missing installed revision reported as available") + } + condition := meta.FindStatusCondition(gs.Status.Conditions, "TemplateUpdatePending") + if condition == nil || condition.Status != metav1.ConditionUnknown { + t.Fatalf("missing unavailable condition: %#v", condition) + } +} + +func TestStaleTemplateRenderCannotUndoCommittedPin(t *testing.T) { + r, gs, old := revisionFixture(t) + stale := gs.DeepCopy() + newTemplate := old.DeepCopy() + newTemplate.Spec.Image = "example.invalid/game:new" + pin, err := r.saveTemplateSnapshot(t.Context(), gs, newTemplate) + if err != nil { + t.Fatal(err) + } + gs.Annotations[effectiveTemplateAnnotation] = pin + if err := r.Update(t.Context(), gs); err != nil { + t.Fatal(err) + } + if err := r.reconcileStatefulSet(t.Context(), stale, old, nil, &materializedConfig{}, 1); err == nil { + t.Fatal("stale rendering created a workload after pin change") + } +} + +func TestTemplateBaselinePreservesLegacyWorkloadUntilAdmittedUpdate(t *testing.T) { + for _, policy := range []string{"Automatic", "OnRequest"} { + t.Run(policy, func(t *testing.T) { + r, gs, installed := revisionFixture(t) + delete(gs.Annotations, effectiveTemplateAnnotation) + gs.Spec.TemplateRollout.Policy = policy + if err := r.Update(t.Context(), gs); err != nil { + t.Fatal(err) + } + installed.Spec.RCON = &gameplanev1alpha1.RCONSpec{Protocol: "source"} + installed.Spec.Capabilities = &gameplanev1alpha1.CapabilitiesSpec{Lifecycle: &gameplanev1alpha1.LifecycleSpec{Stop: []string{"save", "stop"}}} + if err := r.Update(t.Context(), installed); err != nil { + t.Fatal(err) + } + one := int32(1) + ss := &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{Name: gs.Name, Namespace: gs.Namespace, UID: "legacy-workload", OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(gs, gameplanev1alpha1.GroupVersion.WithKind("GameServer"))}}, Spec: appsv1.StatefulSetSpec{Replicas: &one, Template: corev1.PodTemplateSpec{ObjectMeta: metav1.ObjectMeta{Annotations: map[string]string{"admission.example/default": "keep"}}, Spec: corev1.PodSpec{DNSPolicy: corev1.DNSClusterFirst, RestartPolicy: corev1.RestartPolicyAlways, SchedulerName: "default-scheduler", Containers: []corev1.Container{{Name: "game", Image: installed.Spec.Image, ImagePullPolicy: corev1.PullIfNotPresent, TerminationMessagePath: "/dev/termination-log", TerminationMessagePolicy: corev1.TerminationMessageReadFile}}}}}, Status: appsv1.StatefulSetStatus{Replicas: 1, ReadyReplicas: 1}} + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: gs.Name + "-0", Namespace: gs.Namespace, OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(ss, appsv1.SchemeGroupVersion.WithKind("StatefulSet"))}}, Status: corev1.PodStatus{ContainerStatuses: []corev1.ContainerStatus{{Name: "game", State: corev1.ContainerState{Running: &corev1.ContainerStateRunning{}}}}}} + for _, obj := range []client.Object{ss, pod} { + if err := r.Create(t.Context(), obj); err != nil { + t.Fatal(err) + } + } + // Credential isolation is already applied; provenance alone must + // preserve the API-defaulted template without restarting it. + applyWorkloadSecurity(&ss.Spec.Template.Spec, installed, false, 3600) + if err := r.Update(t.Context(), ss); err != nil { + t.Fatal(err) + } + before, err := json.Marshal(ss.Spec.Template) + if err != nil { + t.Fatal(err) + } + if _, changed, err := r.reconcileTemplateRevision(t.Context(), gs); err != nil || !changed { + t.Fatalf("baseline admission: changed=%v err=%v", changed, err) + } + owner, held, err := lifecycleOwner(gs) + if err != nil || !held || !owner.Baseline { + t.Fatalf("missing durable baseline intent: %#v %v", owner, err) + } + // A restarted reconciler must preserve replicas while baseline ownership is held. + stopper := &revisionStopper{} + r = &GameServerReconciler{Client: r.Client, APIReader: r.APIReader, Scheme: r.Scheme, AgentClient: stopper} + if replicas, _, err := r.desiredReplicas(t.Context(), gs, installed, idleAwake); err != nil || replicas != 1 || stopper.calls != 0 { + t.Fatalf("baseline stopped legacy workload: replicas=%d calls=%d err=%v", replicas, stopper.calls, err) + } + if _, changed, err := r.reconcileTemplateRevision(t.Context(), gs); err != nil || !changed { + t.Fatalf("baseline pin: changed=%v err=%v", changed, err) + } + pin := gs.Annotations[effectiveTemplateAnnotation] + if !validTemplateRevision(pin) { + t.Fatal("baseline did not save a pin") + } + if _, held, _ := lifecycleOwner(gs); held { + t.Fatal("baseline retained lifecycle owner") + } + effective, _, err := r.reconcileTemplateRevision(t.Context(), gs) + if err != nil { + t.Fatal(err) + } + if gs.Status.TemplateRevision.Running != "" { + t.Fatal("legacy Pod assigned an invented running revision") + } + mc, err := materializeConfig(gs, effective) + if err != nil { + t.Fatal(err) + } + if err := r.reconcileStatefulSet(t.Context(), gs, effective, nil, mc, 1); err != nil { + t.Fatal(err) + } + if err := r.Get(t.Context(), client.ObjectKeyFromObject(ss), ss); err != nil { + t.Fatal(err) + } + after, err := json.Marshal(ss.Spec.Template) + if err != nil { + t.Fatal(err) + } + if string(after) != string(before) || stopper.calls != 0 || *ss.Spec.Replicas != 1 { + t.Fatal("baseline changed legacy Pod template or power state") + } + + installed.Spec.Image = "example.invalid/game:new" + installed.Spec.Capabilities = nil + if err := r.Update(t.Context(), installed); err != nil { + t.Fatal(err) + } + _, desired, err := templateSnapshot(installed) + if err != nil { + t.Fatal(err) + } + if policy == "OnRequest" { + gs.Spec.TemplateRollout.RequestedRevision = desired + if err := r.Update(t.Context(), gs); err != nil { + t.Fatal(err) + } + } + if _, changed, err := r.reconcileTemplateRevision(t.Context(), gs); err != nil || !changed { + t.Fatalf("next revision admission: %v %v", changed, err) + } + effective, _, err = r.reconcileTemplateRevision(t.Context(), gs) + if err != nil { + t.Fatal(err) + } + if replicas, _, err := r.desiredReplicas(t.Context(), gs, effective, idleAwake); err != nil || replicas != 1 || stopper.calls != 1 { + t.Fatalf("next revision bypassed old graceful stop: replicas=%d calls=%d err=%v", replicas, stopper.calls, err) + } + if err := r.reconcileStatefulSet(t.Context(), gs, effective, nil, mc, 0); err != nil { + t.Fatal(err) + } + if _, changed, err := r.reconcileTemplateRevision(t.Context(), gs); err != nil || changed { + t.Fatalf("adopted before owned drain: changed=%v err=%v", changed, err) + } + if gs.Annotations[effectiveTemplateAnnotation] != pin { + t.Fatal("pin advanced while legacy Pod remains") + } + if err := r.Get(t.Context(), client.ObjectKeyFromObject(ss), ss); err != nil { + t.Fatal(err) + } + if ss.Annotations[lifecycleOperationAnnotation] != gs.Annotations[lifecycleOperationAnnotation] { + t.Fatal("drain not owned by admitted rollout") + } + ss.Status.Replicas, ss.Status.ReadyReplicas, ss.Status.ObservedGeneration = 0, 0, ss.Generation + if err := r.Status().Update(t.Context(), ss); err != nil { + t.Fatal(err) + } + if err := r.Delete(t.Context(), pod); err != nil { + t.Fatal(err) + } + if _, changed, err := r.reconcileTemplateRevision(t.Context(), gs); err != nil || !changed { + t.Fatalf("drained adoption: %v %v", changed, err) + } + if gs.Annotations[effectiveTemplateAnnotation] != desired { + t.Fatal("drained rollout did not adopt requested pin") + } + }) + } +} + +func TestTemplateBaselineAllowsRealWorkloadChanges(t *testing.T) { + for _, change := range []string{"inputs", "stopped", "new-workload", "credential-isolation", "restricted-profile"} { + t.Run(change, func(t *testing.T) { + r, gs, _ := revisionFixture(t) + pin := gs.Annotations[effectiveTemplateAnnotation] + inputs, err := r.baselineRenderInputs(gs, pin) + if err != nil { + t.Fatal(err) + } + gs.Annotations[baselineTemplateAnnotation], gs.Annotations[baselineWorkloadAnnotation], gs.Annotations[baselineInputsAnnotation] = pin, "legacy", inputs + one := int32(1) + ss := &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{UID: "legacy"}, Spec: appsv1.StatefulSetSpec{Replicas: &one}} + applyWorkloadSecurity(&ss.Spec.Template.Spec, &gameplanev1alpha1.GameTemplate{}, false, 3600) + if keep, err := r.preserveBaselinePodTemplate(t.Context(), gs, ss); err != nil || !keep { + t.Fatalf("baseline not preserved: %v %v", keep, err) + } + switch change { + case "inputs": + gs.Spec.Image = "example.invalid/game:override" + case "stopped": + zero := int32(0) + ss.Spec.Replicas = &zero + case "new-workload": + ss.UID = "replacement" + case "credential-isolation": + ss.Spec.Template.Spec.AutomountServiceAccountToken = nil + case "restricted-profile": + r.RestrictedWorkloads = true + } + if keep, err := r.preserveBaselinePodTemplate(t.Context(), gs, ss); err != nil || keep { + t.Fatalf("real workload change frozen: %v %v", keep, err) + } + }) + } +} + +func TestTemplateBaselineWithoutWorkloadPinsBeforeFirstRender(t *testing.T) { + r, gs, _ := revisionFixture(t) + delete(gs.Annotations, effectiveTemplateAnnotation) + if err := r.Update(t.Context(), gs); err != nil { + t.Fatal(err) + } + for range 2 { + if _, changed, err := r.reconcileTemplateRevision(t.Context(), gs); err != nil || !changed { + t.Fatalf("initial pin: %v %v", changed, err) + } + } + effective, _, err := r.reconcileTemplateRevision(t.Context(), gs) + if err != nil { + t.Fatal(err) + } + if err := r.reconcileStatefulSet(t.Context(), gs, effective, nil, &materializedConfig{}, 1); err != nil { + t.Fatal(err) + } + var ss appsv1.StatefulSet + if err := r.Get(t.Context(), client.ObjectKeyFromObject(gs), &ss); err != nil { + t.Fatal(err) + } + if ss.Spec.Template.Annotations[templateRevisionAnnotation] != gs.Annotations[effectiveTemplateAnnotation] || *ss.Spec.Replicas != 1 { + t.Fatal("first workload not stamped and started") + } +} + +func TestTemplateBaselineDoesNotSwallowConcurrentSpecEdit(t *testing.T) { + r, gs, installed := revisionFixture(t) + delete(gs.Annotations, effectiveTemplateAnnotation) + gs.Generation = 1 + if err := r.Update(t.Context(), gs); err != nil { + t.Fatal(err) + } + one := int32(1) + ss := &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{Name: gs.Name, Namespace: gs.Namespace, UID: "legacy", OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(gs, gameplanev1alpha1.GroupVersion.WithKind("GameServer"))}}, Spec: appsv1.StatefulSetSpec{Replicas: &one, Template: corev1.PodTemplateSpec{Spec: corev1.PodSpec{Containers: []corev1.Container{{Name: "game", Image: installed.Spec.Image}}}}}} + if err := r.Create(t.Context(), ss); err != nil { + t.Fatal(err) + } + if _, changed, err := r.reconcileTemplateRevision(t.Context(), gs); err != nil || !changed { + t.Fatalf("baseline admission: %v %v", changed, err) + } + gs.Generation++ // Fake clients do not increment spec generations. + gs.Spec.Image = "example.invalid/game:requested" + if err := r.Update(t.Context(), gs); err != nil { + t.Fatal(err) + } + if _, changed, err := r.reconcileTemplateRevision(t.Context(), gs); err != nil || !changed { + t.Fatalf("baseline initialization: %v %v", changed, err) + } + if gs.Annotations[baselineInputsAnnotation] != "" { + t.Fatal("concurrent edit was recorded as already applied") + } + effective, _, err := r.reconcileTemplateRevision(t.Context(), gs) + if err != nil { + t.Fatal(err) + } + if err := r.reconcileStatefulSet(t.Context(), gs, effective, nil, &materializedConfig{}, 1); err != nil { + t.Fatal(err) + } + if err := r.Get(t.Context(), client.ObjectKeyFromObject(ss), ss); err != nil { + t.Fatal(err) + } + if ss.Spec.Template.Spec.Containers[0].Image != gs.Spec.Image { + t.Fatal("baseline swallowed the requested image change") + } +} + +func TestTemplateRevisionRejectsRestrictedIncompatibleTargetBeforeAdmission(t *testing.T) { + r, gs, installed := revisionFixture(t) + r.RestrictedWorkloads = true + old := gs.Annotations[effectiveTemplateAnnotation] + installed.Spec.Image = "example.invalid/game:incompatible" + if err := r.Update(t.Context(), installed); err != nil { + t.Fatal(err) + } + gs.Spec.TemplateRollout.Policy = "Automatic" + if err := r.Update(t.Context(), gs); err != nil { + t.Fatal(err) + } + if _, _, err := r.reconcileTemplateRevision(t.Context(), gs); err == nil { + t.Fatal("admitted a template without Restricted-compatible identity") + } + if _, held, _ := lifecycleOwner(gs); held || gs.Annotations[effectiveTemplateAnnotation] != old { + t.Fatal("invalid security profile changed the pin or admitted a drain") + } +} diff --git a/operator/internal/controller/gameserver_wipe_inventory_test.go b/operator/internal/controller/gameserver_wipe_inventory_test.go index e4075147d..2ae64daf1 100644 --- a/operator/internal/controller/gameserver_wipe_inventory_test.go +++ b/operator/internal/controller/gameserver_wipe_inventory_test.go @@ -45,6 +45,22 @@ func TestGameServerReconcileInventoriesWipeWorkersOnce(t *testing.T) { c := fake.NewClientBuilder().WithScheme(s).WithObjects(gs, tmpl, caSecret).WithStatusSubresource(gs).Build() reader := &wipeInventoryReader{Reader: c} r := &GameServerReconciler{Client: c, APIReader: reader, Scheme: s, AgentCASecretName: caSecret.Name, AgentCASecretNamespace: caSecret.Namespace} + // This test measures an ordinary already-admitted reconcile. Template + // admission has its own live worker checks and transition tests. + revision, err := r.saveTemplateSnapshot(t.Context(), gs, tmpl) + if err != nil { + t.Fatal(err) + } + if gs.Annotations == nil { + gs.Annotations = map[string]string{} + } + gs.Annotations[effectiveTemplateAnnotation] = revision + if err := c.Update(t.Context(), gs); err != nil { + t.Fatal(err) + } + if err := r.reportTemplateRevision(t.Context(), gs, revision, lifecycleOperation{}, false); err != nil { + t.Fatal(err) + } // Exercise both initial child/status creation and the next full pass. Idle // and capture reconciliation participate, rather than just lifecycle helpers. for range 2 { diff --git a/operator/internal/controller/helpers_envtest_test.go b/operator/internal/controller/helpers_envtest_test.go index eaf4cb6c5..8069878a8 100644 --- a/operator/internal/controller/helpers_envtest_test.go +++ b/operator/internal/controller/helpers_envtest_test.go @@ -165,11 +165,10 @@ func startMgr(t *testing.T, ns string, setups ...setupReconciler) client.Client Scheme: scheme, Logger: logr.Discard(), Cache: cache.Options{ - // Include the test namespace AND cluster scope (""). Without - // the "" entry, controller-runtime's namespace-restricted - // cache will refuse to serve reads of cluster-scoped CRDs - // like GameTemplate. - DefaultNamespaces: map[string]cache.Config{ns: {}, "": {}}, + // Cluster-scoped kinds use the independent cluster cache. An + // empty namespace key means AllNamespaces and would make this + // manager reconcile fixtures left by every earlier test. + DefaultNamespaces: map[string]cache.Config{ns: {}}, }, Metrics: metricsserver.Options{BindAddress: "0"}, HealthProbeBindAddress: "0", diff --git a/operator/internal/controller/lifecycle_operation.go b/operator/internal/controller/lifecycle_operation.go index fc1d626fb..6445aa927 100644 --- a/operator/internal/controller/lifecycle_operation.go +++ b/operator/internal/controller/lifecycle_operation.go @@ -35,6 +35,9 @@ type lifecycleIdentity struct { // InitialStartup permits only the first workload of a CSI restore's new // target (including WaitForFirstConsumer PVCs), while other operations wait. InitialStartup bool `json:"initialStartup,omitempty"` + // Baseline establishes a first template pin without replacing an existing + // workload. Durable intent prevents a leader restart turning it into a drain. + Baseline bool `json:"baseline,omitempty"` } type lifecycleOperation struct { @@ -55,7 +58,7 @@ func parseLifecycleOperation(value string) (lifecycleOperation, error) { return op, fmt.Errorf("decode lifecycle owner: %w", err) } for _, id := range append([]lifecycleIdentity{op.lifecycleIdentity}, op.Recovery...) { - if id.Name == "" || id.UID == "" || id.TargetUID == "" || (id.Kind != "Backup" && id.Kind != "Restore" && id.Kind != "Wipe") || (id.Kind == "Wipe" && id.Token == "") { + if id.Name == "" || id.UID == "" || id.TargetUID == "" || (id.Kind != "Backup" && id.Kind != "Restore" && id.Kind != "Wipe" && id.Kind != "Rollout") || (id.Kind == "Wipe" && id.Token == "") || (id.Kind == "Rollout" && !validTemplateRevision(id.Token)) { return op, fmt.Errorf("invalid lifecycle owner identity") } if id.TargetUID != op.TargetUID { @@ -135,7 +138,7 @@ func lifecycleOwnerExists(ctx context.Context, reader client.Reader, namespace s obj = &gameplanev1alpha1.Backup{} case "Restore": obj = &gameplanev1alpha1.Restore{} - case "Wipe": + case "Wipe", "Rollout": // Only the GameServer reconciler may cancel/replace a wipe token after // its workers drain. A removed request is not proof of worker absence. return true, nil @@ -161,6 +164,9 @@ func claimLifecycleOperation(ctx context.Context, c client.Client, reader client if live.UID != want.TargetUID || !live.DeletionTimestamp.IsZero() { return false, fmt.Errorf("lifecycle target changed identity or is being deleted") } + if want.Kind == "Rollout" && live.Generation != want.Generation { + return false, errLifecycleGenerationChanged + } *gs = live op, occupied, err := lifecycleOwner(gs) if err != nil { @@ -179,7 +185,7 @@ func claimLifecycleOperation(ctx context.Context, c client.Client, reader client } op.lifecycleIdentity = want } else if occupied { - if gs.Annotations[lifecycleOperationAnnotation] != "" && op.Kind != "Wipe" && op.Generation != want.Generation { + if gs.Annotations[lifecycleOperationAnnotation] != "" && op.Kind != "Wipe" && op.Kind != "Rollout" && op.Generation != want.Generation { return false, errLifecycleGenerationChanged } if gs.Annotations[lifecycleOperationAnnotation] == "" { @@ -194,7 +200,7 @@ func claimLifecycleOperation(ctx context.Context, c client.Client, reader client } // A stop command already sent may still save data and exit. Waiting // here also lets pending restarts finish instead of freezing mid-stop. - if want.Kind == "Backup" { + if want.Kind == "Backup" || want.Kind == "Rollout" { pending, err := backupLifecycleTransitionPending(ctx, reader, gs) if err != nil || pending { return false, err diff --git a/operator/internal/controller/manager_scope_envtest_test.go b/operator/internal/controller/manager_scope_envtest_test.go new file mode 100644 index 000000000..4a4330f69 --- /dev/null +++ b/operator/internal/controller/manager_scope_envtest_test.go @@ -0,0 +1,39 @@ +//go:build envtest + +package controller + +import ( + "testing" + + "sigs.k8s.io/controller-runtime/pkg/client" + + gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" +) + +// The envtest API has no namespace garbage collector. Each new manager must +// ignore earlier fixtures while continuing to read cluster-scoped templates. +func TestManagerCacheKeepsNamespaceIsolationAndClusterScope(t *testing.T) { + ns, other := newNamespace(t), newNamespace(t) + tmpl := buildGameTemplate(uniqueName("cache-scope")) + if err := k8sClient.Create(t.Context(), tmpl); err != nil { + t.Fatal(err) + } + deleteCleanup(t, tmpl) + for _, namespace := range []string{ns, other} { + if err := k8sClient.Create(t.Context(), buildGameServer(namespace, "server", tmpl.Name)); err != nil { + t.Fatal(err) + } + } + cached := startMgr(t, ns) + var templates gameplanev1alpha1.GameTemplate + if err := cached.Get(t.Context(), client.ObjectKeyFromObject(tmpl), &templates); err != nil { + t.Fatalf("cluster-scoped template unavailable: %v", err) + } + var servers gameplanev1alpha1.GameServerList + if err := cached.List(t.Context(), &servers); err != nil { + t.Fatal(err) + } + if len(servers.Items) != 1 || servers.Items[0].Namespace != ns { + t.Fatalf("manager cache included resources outside its test namespace: %+v", servers.Items) + } +} diff --git a/operator/internal/controller/networkcapture_filter.go b/operator/internal/controller/networkcapture_filter.go index c3d0bfd49..8252a16ed 100644 --- a/operator/internal/controller/networkcapture_filter.go +++ b/operator/internal/controller/networkcapture_filter.go @@ -7,7 +7,6 @@ import ( corev1 "k8s.io/api/core/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" - "k8s.io/apimachinery/pkg/types" gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" ) @@ -50,14 +49,14 @@ func (r *NetworkCaptureReconciler) captureFilter( return nc.Spec.Filter, "", nil } name := gs.Spec.TemplateRef.Name - var tmpl gameplanev1alpha1.GameTemplate - if err := r.Get(ctx, types.NamespacedName{Name: name}, &tmpl); err != nil { + tmpl, err := effectiveGameTemplate(ctx, r.Client, gs) + if err != nil { if apierrors.IsNotFound(err) { return nil, fmt.Sprintf("gametemplate %s not found; cannot build the default capture filter", name), nil } return nil, "", fmt.Errorf("get gametemplate %s for the default capture filter: %w", name, err) } - f := defaultCaptureFilter(&tmpl) + f := defaultCaptureFilter(tmpl) if f == "" { return nil, fmt.Sprintf("gametemplate %s advertises no ports; supply a capture filter", name), nil } diff --git a/operator/internal/controller/restore_controller.go b/operator/internal/controller/restore_controller.go index 1deffc5ba..e979ee712 100644 --- a/operator/internal/controller/restore_controller.go +++ b/operator/internal/controller/restore_controller.go @@ -180,12 +180,13 @@ func (r *RestoreReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct } } - // A missing template is not fatal: it only refines the FSGroup, and - // buildBackupPodSecurityContext falls back to 65532 for a zero template. - var tmpl gameplanev1alpha1.GameTemplate - if err := r.Get(ctx, types.NamespacedName{Name: gs.Spec.TemplateRef.Name}, &tmpl); err != nil && !apierrors.IsNotFound(err) { + // A pinned snapshot is mandatory once established. Legacy targets retain + // the historical missing-template fallback for their initial operation. + effective, err := effectiveGameTemplate(ctx, r.apiReader(), &gs) + if err != nil && (gs.Annotations[effectiveTemplateAnnotation] != "" || !apierrors.IsNotFound(err)) { return ctrl.Result{}, err } + tmpl := *effective if rs.Status.Phase == gameplanev1alpha1.RestorePhaseSuspending { if gs.Status.Phase != gameplanev1alpha1.GameServerPhaseSuspended && diff --git a/operator/internal/controller/restore_template.go b/operator/internal/controller/restore_template.go new file mode 100644 index 000000000..039fa3dff --- /dev/null +++ b/operator/internal/controller/restore_template.go @@ -0,0 +1,84 @@ +package controller + +import ( + "context" + "crypto/sha256" + "encoding/json" + "fmt" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" +) + +const restoreTemplateAnnotation = "restore.gameplane.local/template-snapshot" + +func (r *RestoreReconciler) planRestoredTemplate(ctx context.Context, rs *gameplanev1alpha1.Restore, source, target *gameplanev1alpha1.GameServer) error { + // Pre-upgrade sources retain historical restore behavior. New snapshots + // cannot reconstruct the template used when an older backup was taken. + if source.Annotations[effectiveTemplateAnnotation] == "" { + return nil + } + identity := sha256.Sum256([]byte(string(rs.UID))) + name := fmt.Sprintf("restore-template-%x", identity[:16]) + var existing corev1.ConfigMap + err := r.apiReader().Get(ctx, client.ObjectKey{Namespace: rs.Namespace, Name: name}, &existing) + if err != nil && !apierrors.IsNotFound(err) { + return err + } + if err == nil { + if !metav1.IsControlledBy(&existing, rs) || existing.Immutable == nil || !*existing.Immutable { + return fmt.Errorf("restore template plan is foreign or mutable") + } + } else { + tmpl, err := effectiveGameTemplate(ctx, r.apiReader(), source) + if err != nil { + return err + } + data, revision, err := templateSnapshot(tmpl) + if err != nil { + return err + } + immutable := true + cm := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: name, Namespace: rs.Namespace, Annotations: map[string]string{templateRevisionAnnotation: revision}}, Immutable: &immutable, Data: map[string]string{templateSnapshotKey: string(data)}} + if err := controllerutil.SetControllerReference(rs, cm, r.Scheme); err != nil { + return err + } + if err := r.Create(ctx, cm); err != nil { + return err + } + } + target.Annotations[restoreTemplateAnnotation] = name + return nil +} + +func (r *RestoreReconciler) copyRestoredTemplate(ctx context.Context, rs *gameplanev1alpha1.Restore, target *gameplanev1alpha1.GameServer) error { + var cm corev1.ConfigMap + if err := r.apiReader().Get(ctx, client.ObjectKey{Namespace: target.Namespace, Name: target.Annotations[restoreTemplateAnnotation]}, &cm); err != nil { + return err + } + if !metav1.IsControlledBy(&cm, rs) || cm.Immutable == nil || !*cm.Immutable || !cm.DeletionTimestamp.IsZero() { + return fmt.Errorf("restore template plan is foreign, mutable or deleting") + } + var tmpl gameplanev1alpha1.GameTemplate + if err := json.Unmarshal([]byte(cm.Data[templateSnapshotKey]), &tmpl); err != nil { + return err + } + _, revision, err := templateSnapshot(&tmpl) + if err != nil { + return err + } + if tmpl.Name != target.Spec.TemplateRef.Name || revision != cm.Annotations[templateRevisionAnnotation] { + return fmt.Errorf("restore template plan content changed") + } + writer := &GameServerReconciler{Client: r.Client, APIReader: r.apiReader(), Scheme: r.Scheme} + if _, err := writer.saveTemplateSnapshot(ctx, target, &tmpl); err != nil { + return err + } + target.Annotations[effectiveTemplateAnnotation] = revision + return r.Update(ctx, target) +} diff --git a/operator/internal/controller/restore_volumesnapshot.go b/operator/internal/controller/restore_volumesnapshot.go index e017f90dc..c38ff5429 100644 --- a/operator/internal/controller/restore_volumesnapshot.go +++ b/operator/internal/controller/restore_volumesnapshot.go @@ -141,6 +141,12 @@ func (r *RestoreReconciler) reconcileVolumeSnapshotRestore( return r.failOrRequeue(ctx, rs, err) } rewriteRefs(&created.Spec, copies) + // Preserve the source's effective template before creating a target. A + // durable Restore-owned copy survives source adoption/deletion and retries; + // the target receives its own immutable copy before InitialStartup. + if err := r.planRestoredTemplate(ctx, rs, &orig, created); err != nil { + return r.failOrRequeue(ctx, rs, err) + } // Persist the copy plan on the Restore now, before creating anything. // ensureOwnedRefCopies below can fail transiently (e.g. a write @@ -224,6 +230,12 @@ func (r *RestoreReconciler) ensureRestoredRefsFromSpec( } func (r *RestoreReconciler) prepareSnapshotRestoreTarget(ctx context.Context, rs *gameplanev1alpha1.Restore, gs *gameplanev1alpha1.GameServer) (bool, error) { + if gs.Annotations[effectiveTemplateAnnotation] == "" && gs.Annotations[restoreTemplateAnnotation] != "" { + if err := r.copyRestoredTemplate(ctx, rs, gs); err != nil { + return false, err + } + return false, nil + } changed, err := r.bindRestoreTarget(ctx, rs, gs) if err != nil { return false, err diff --git a/operator/specs.md b/operator/specs.md index 813d72df0..d2c150b4c 100644 --- a/operator/specs.md +++ b/operator/specs.md @@ -84,6 +84,41 @@ start/stop and expiry remain reconciler responsibilities; the gateway cannot bypass them or operate an arbitrary pod. Existing game resources and storage are not recreated to enable remote parity. +## Template revision admission + +- `GameServer.spec.templateRollout.policy` defaults to Automatic; OnRequest + admits only `requestedRevision == status.templateRevision.desired`. Initial + provisioning/baseline always adopts the installed content. No fleet scheduler, + canary or maintenance-window controller is implied. +- First-pin initialization uses shared ownership and a checked metadata update + without stopping an existing workload. Its StatefulSet UID and render-input + fingerprint preserve the existing Pod template byte-for-byte, including API + defaults/admission fields, while those inputs are unchanged. A real workload + input change, replaced StatefulSet or observed stopped drain allows revision + stamping. An existing unannotated Pod never receives inferred provenance. + New servers pin before first rendering. Existing lifecycle owners defer both. +- Complete template Name/spec content is SHA256-addressed in immutable owned + ConfigMaps (900 KiB maximum). The effective annotation is authoritative; status + is reconstructible. A missing/foreign/mutable/corrupt pin fails closed. +- All GameServer config, version/image, init, storage, agent, networking and + workload calculations use the effective snapshot. Restic backup resolves the + pin after shared ownership, restore resolves its owned target pin, and default + capture filters use pinned ports. CSI restore planning persists a Restore-owned + source snapshot and copies it to the new server before startup permission. +- Rollout extends the shared operation owner: no admission during backup, + restore or wipe; old effective graceful stop and live observed Pod drain precede + a checked pin update. The owner survives restart and catalog changes, releasing + after the matching new Pod is observed or a suspended zero-replica template is + observed. Startup health remains independent. Per-server power intent survives. +- Desired/effective/running digests are distinct. Running requires the owned + StatefulSet/Pod chain and a running game container; legacy or stopped Pods do + not claim the desired revision. TemplateUpdatePending has RevisionAvailable, + OperationInProgress, AdoptingRevision, RevisionCurrent and TemplateUnavailable + reasons, included in the generated condition translation reference. +- Obsolete snapshots are pruned only after old workload drain and replacement + observation. PVCs are retained; data compatibility and runtime/data rollback + remain administrator responsibilities. See `docs/template-rollouts.md`. + ## Non-goals / boundaries Tailscale tags are normalized and syntax-checked before tunnel Deployment creation. @@ -309,6 +344,8 @@ Primary reconcilers register with the manager in `cmd/main.go` and handle CRD li - VolumeSnapshot API availability is checked before quiescing. Snapshot creation errors fail the Backup and run unquiesce cleanup immediately instead of retrying creation indefinitely with saving disabled. API/creation failures persist `backup.gameplane.local/snapshot-aborted` to allow cleanup without waiting for a failed/nonrestorable capture. Failed unquiesce remains visible and retried. A transport failure can leave a CSI snapshot object despite a Failed Backup; this does not make that Backup restorable. - Restic Job: mount data PVC, run restic backup to configured destination (S3, B2, rest-server, local fs). Job carries BackoffLimit and ActiveDeadlineSeconds from the reconciler's JobBackoffLimit / JobActiveDeadlineSeconds fields (operator flags --backup-job-backoff-limit / --backup-job-active-deadline-seconds, Helm operator.backupJobBackoffLimit / operator.backupJobActiveDeadlineSeconds; defaults 2 and 86400, applied when unset; spec 018 OD-027); only a Failed=True Job condition (set by jobPermanentlyFailed check) marks the Backup Failed. Pod SecurityContext takes FSGroup from the target GameTemplate's spec.security.fsGroup (fallback 65532 if template missing). - Snapshot tracking: parse restic JSON output from container logs to extract snapshot ID. + - Capture metadata: after ownership, workload fencing and quiesce admission, record `status.captureManifest` once before creating the first copy worker. It pins the GameServer UID, strategy, observed Pod/StatefulSet mount layout, PVC names/UIDs, included primary claim and excluded extra/mod/retained claims. `storage.extra[].backupRole` optionally declares `data` or `rebuildable`; absent declarations remain unclassified and excluded. Later template edits never rewrite the manifest. Existing workers and old terminal Backups have unknown historical coverage, never reconstructed from the current template. + - Achieved consistency: the `Consistency` condition is separate from copy phase and volume coverage. It stays Unknown until a usable snapshot identity is recorded; failed copies report `CaptureFailed`. Agent-confirmed quiesce, already-stopped/drained sources, unsupported/no-agent raw copy and unquiesced CSI snapshots have distinct reasons. Quiesce depends on the module commands and does not imply that excluded volumes were captured. The API exposes the stored status and the backup drawer shows consistency and included/excluded claims. `StoppedSource` records an observation, without adding a stop/copy/restart policy or multi-volume backup. - Snapshot cleanup on delete (`backup_forget.go`): restic-strategy Backups carry `BackupSnapshotFinalizer` (`gameplane.local/backup-snapshot-finalizer`, added on first reconcile, also for pre-existing non-Failed Backups; not added for volume-snapshot or Failed Backups). `finalizeDelete` runs the unquiesce stage (`BackupFinalizer`) first, then `finalizeSnapshot`: it deletes the Backup's own still-running Job (foreground, awaited), waits while a non-terminal Restore pins the Backup or its snapshot id, then runs Job `-forget` (`restic unlock || true; restic forget --prune --retry-lock 10m`, backoff 3, deadline 1800s, 50m/128Mi requests and 512Mi memory limit, no PVC; the snapshot id is passed through env `FORGET_SNAPSHOT_ID` and must match `^[0-9a-f]{8,64}$`). Success sets `SnapshotForgotten=True` and a Normal `SnapshotForgotten` event; a Failed Job, a missing/incomplete repo Secret, a terminating namespace, or `maxSnapshotForgetWait` (45m, from annotation `backup.gameplane.local/forget-started-at`) sets `SnapshotForgotten=False` with a Warning `SnapshotForgetAbandoned` event and releases the Backup, leaving the snapshot in the repository. A Backup with no recorded snapshot id is released with `SnapshotForgetSkipped` when it may have written one. Backup Jobs run `restic backup /data --json --retry-lock 10m ...`. RBAC markers on `backup_controller.go` grant `backups` update/patch, `backups/status`, `backups/finalizers` and `jobs` create/update/patch/delete. Tests: `backup_forget_test.go`, `backup_forget_envtest_test.go`, `retention_terminating_test.go`; e2e `TestBackup_DeleteForgetsSnapshot` and the extended `TestBackupSchedule_RetentionTrimsPast`. Spec: `specs/done_020-backup-snapshot-forget/`. - Status: report phase (Pending/Running/Succeeded/Failed), error details. - **Integrations:** Agent client (quiesce), Kubernetes Pod logs (restic output), VolumeSnapshot API (CSI backups). diff --git a/test/e2e/buckets.sh b/test/e2e/buckets.sh index d9cdde3ce..ba9f800ec 100755 --- a/test/e2e/buckets.sh +++ b/test/e2e/buckets.sh @@ -63,6 +63,7 @@ TestGameServer_TemplateDefaultOverridesInstallTime TestGameServer_ExplicitStorageClassOverridesDefault TestGameServer_NonexistentStorageClassSurfacesError TestGameServer_VersionSwitch +TestGameServer_TemplateOnRequestRetainsData TestGameServer_WipeReportsFailureOnPermissionDenied TestGameServer_CascadingDelete TestGameServer_TunnelCredentialRefusalSurfaced diff --git a/test/e2e/gameserver_e2e_test.go b/test/e2e/gameserver_e2e_test.go index 78c75baae..790773957 100644 --- a/test/e2e/gameserver_e2e_test.go +++ b/test/e2e/gameserver_e2e_test.go @@ -867,6 +867,29 @@ func TestGameServer_NetworkCaptureEphemeralContainer(t *testing.T) { // Created WITHOUT capture enabled — this test drives enable/disable // entirely through the :capture-enable/:capture-disable HTTP routes. applyBusyboxGameServer(t, ns, gsName, tmpl) + // Register after fixture creation so failure evidence is collected before + // its cleanup deletes the Pod and workflow-level diagnostics lose it. + t.Cleanup(func() { + if !t.Failed() { + return + } + diagnosticCtx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + obj, err := envInstance.Dyn.Resource(gameServerGVR).Namespace(ns).Get(diagnosticCtx, gsName, metav1.GetOptions{}) + if err != nil { + t.Logf("capture failure diagnostics: get GameServer: %v", err) + } else { + capture, _, _ := unstructured.NestedMap(obj.Object, "status", "capture") + t.Logf("capture failure diagnostics: GameServer UID=%s capture=%v", obj.GetUID(), capture) + } + pod, err := envInstance.K8s.CoreV1().Pods(ns).Get(diagnosticCtx, gsName+"-0", metav1.GetOptions{}) + if err != nil { + t.Logf("capture failure diagnostics: get Pod: %v", err) + } else { + t.Logf("capture failure diagnostics: Pod UID=%s phase=%s capture=%+v", pod.UID, pod.Status.Phase, + findContainerStatusByName(pod.Status.EphemeralContainerStatuses, "capture")) + } + }) waitPVCBound(t, ns, gsName+"-data", 90*time.Second) requireAgentReady(t, ns, gsName) diff --git a/test/e2e/specs.md b/test/e2e/specs.md index 46906a0b1..9cd5bd94d 100644 --- a/test/e2e/specs.md +++ b/test/e2e/specs.md @@ -33,6 +33,14 @@ It was one of three `go.work` modules without CI lint coverage until this featur ## Responsibilities +The operator bucket includes `TestGameServer_TemplateOnRequestRetainsData`: a +live busybox workload keeps its Pod and old rendered config after a shared +template update, then admits the exact desired digest and verifies a replacement +Pod, the same PVC UID, preserved save marker and updated config-init output. +Operator regression sources cover lifecycle exclusion, old-template graceful +stop, status reset/leader recovery, exact request binding, snapshot integrity, +complete pinned render inputs and source-template preservation for CSI restore. + - Keep the lint workflow, devcontainer and `lint-gate-verify.sh` on the exact `v2.14.0` linter pin, whose importer supports Go 1.27.3 export data. The verifier's valid fixtures use the same pin; regression fixtures reject `latest` and a stale fixed version in any lint step. - The optional-components lifecycle's `frp-identity-health` subtest runs two same-template TCP/UDP echo GameServers through one real frps. It asserts backend identity, registration rejection, replacement isolation, invalid credentials, relay outage/recovery and secret-free logs. It shares the lifecycle's controlled fixture and serial bucket; no extra top-level test or login budget is introduced. diff --git a/test/e2e/template_rollout_e2e_test.go b/test/e2e/template_rollout_e2e_test.go new file mode 100644 index 000000000..0f6e7c046 --- /dev/null +++ b/test/e2e/template_rollout_e2e_test.go @@ -0,0 +1,121 @@ +//go:build e2e + +package e2e + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "testing" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/util/retry" +) + +// A real workload remains unchanged while a new shared template is installed, +// then adopts only on an exact request while retaining its PVC and saved data. +func TestGameServer_TemplateOnRequestRetainsData(t *testing.T) { + t.Parallel() + ctx := context.Background() + const ns, name, template = "gameplane-games", "e2e-template-stage", "e2e-template-stage" + applyBusyboxTemplate(t, template) + setConfigFile := func(contents string) { + t.Helper() + // The template controller updates status concurrently with both + // spec changes. Retry the complete read-modify-write on conflict. + if err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + tmpl, err := envInstance.Dyn.Resource(gameTemplateGVR).Get(ctx, template, metav1.GetOptions{}) + if err != nil { + return err + } + if err := unstructured.SetNestedSlice(tmpl.Object, []any{map[string]any{"path": "revision.txt", "template": contents}}, "spec", "configFiles"); err != nil { + return err + } + _, err = envInstance.Dyn.Resource(gameTemplateGVR).Update(ctx, tmpl, metav1.UpdateOptions{}) + return err + }); err != nil { + t.Fatal(err) + } + } + setConfigFile("old") + gs := &unstructured.Unstructured{Object: map[string]any{"apiVersion": "gameplane.local/v1alpha1", "kind": "GameServer", "metadata": map[string]any{"name": name, "namespace": ns}, "spec": map[string]any{"templateRef": map[string]any{"name": template}, "templateRollout": map[string]any{"policy": "OnRequest"}}}} + if _, err := envInstance.Dyn.Resource(gameServerGVR).Namespace(ns).Create(ctx, gs, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + _ = envInstance.Dyn.Resource(gameServerGVR).Namespace(ns).Delete(context.Background(), name, metav1.DeleteOptions{}) + }) + get := func() *unstructured.Unstructured { + t.Helper() + obj, err := envInstance.Dyn.Resource(gameServerGVR).Namespace(ns).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + return obj + } + var old string + envInstance.Eventually(t, 3*time.Minute, func() (bool, string) { + obj := get() + running, _, _ := unstructured.NestedString(obj.Object, "status", "templateRevision", "running") + effective, _, _ := unstructured.NestedString(obj.Object, "status", "templateRevision", "effective") + old = effective + return effective != "" && running == effective && obj.GetAnnotations()["operations.gameplane.local/owner"] == "", "waiting for initial pinned workload" + }) + pod, err := envInstance.K8s.CoreV1().Pods(ns).Get(ctx, name+"-0", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + oldPod := pod.UID + pvc, err := envInstance.K8s.CoreV1().PersistentVolumeClaims(ns).Get(ctx, name+"-data", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + oldPVC := pvc.UID + if out, err := envInstance.Kubectl(ctx, "exec", "-n", ns, name+"-0", "-c", "game", "--", "sh", "-c", "printf retained > /data/save-marker; cat /data/revision.txt"); err != nil || strings.TrimSpace(out) != "old" { + t.Fatalf("seed persisted data: %s %v", out, err) + } + setConfigFile("new") + var desired string + envInstance.Eventually(t, time.Minute, func() (bool, string) { + obj := get() + desired, _, _ = unstructured.NestedString(obj.Object, "status", "templateRevision", "desired") + effective, _, _ := unstructured.NestedString(obj.Object, "status", "templateRevision", "effective") + return desired != "" && desired != old && effective == old, "waiting for pending revision" + }) + // Check after the controller reported processing the update, rather than + // relying on an arbitrary sleep before asserting an unchanged Pod. + pod, err = envInstance.K8s.CoreV1().Pods(ns).Get(ctx, name+"-0", metav1.GetOptions{}) + if err != nil || pod.UID != oldPod { + t.Fatalf("catalog update replaced staged Pod: %v", err) + } + if out, err := envInstance.Kubectl(ctx, "exec", "-n", ns, name+"-0", "-c", "game", "--", "cat", "/data/revision.txt"); err != nil || strings.TrimSpace(out) != "old" { + t.Fatalf("pending config changed: %s %v", out, err) + } + patch, err := json.Marshal(map[string]any{"spec": map[string]any{"templateRollout": map[string]any{"requestedRevision": desired}}}) + if err != nil { + t.Fatal(err) + } + if _, err := envInstance.Dyn.Resource(gameServerGVR).Namespace(ns).Patch(ctx, name, types.MergePatchType, patch, metav1.PatchOptions{}); err != nil { + t.Fatal(err) + } + envInstance.Eventually(t, 3*time.Minute, func() (bool, string) { + obj := get() + running, _, _ := unstructured.NestedString(obj.Object, "status", "templateRevision", "running") + current, err := envInstance.K8s.CoreV1().Pods(ns).Get(ctx, name+"-0", metav1.GetOptions{}) + if err != nil { + return false, err.Error() + } + return running == desired && current.UID != oldPod && obj.GetAnnotations()["operations.gameplane.local/owner"] == "", fmt.Sprintf("running revision=%s", running) + }) + pvc, err = envInstance.K8s.CoreV1().PersistentVolumeClaims(ns).Get(ctx, name+"-data", metav1.GetOptions{}) + if err != nil || pvc.UID != oldPVC { + t.Fatalf("adoption replaced PVC: %v", err) + } + if out, err := envInstance.Kubectl(ctx, "exec", "-n", ns, name+"-0", "-c", "game", "--", "sh", "-c", "cat /data/save-marker; printf ':'; cat /data/revision.txt"); err != nil || strings.TrimSpace(out) != "retained:new" { + t.Fatalf("persisted contents after adoption: %s %v", out, err) + } +} diff --git a/web/e2e/screenshots/capture.ts b/web/e2e/screenshots/capture.ts index cb1ef8271..f8e804091 100644 --- a/web/e2e/screenshots/capture.ts +++ b/web/e2e/screenshots/capture.ts @@ -187,18 +187,19 @@ export async function captureLocator(page: Page, id: string, locator: Locator): // (4 device px) per edge, so a genuine size regression still reaches the // scale gate. // - // This is specifically a *modal backdrop* artifact: only dialogs (or - // captures nested inside one) sit on top of a backdrop that can bleed in - // this way. A non-dialog element crop (a chip, a badge, an alert banner) - // has no backdrop, and its own top/bottom edge row is frequently a solid + // This is specifically a *modal backdrop* artifact: only whole-dialog + // captures can include the backdrop outside the panel. A nested content + // crop (such as backup volume coverage) contains the panel's own pixels, + // which must be preserved. A non-dialog element crop (a chip, a badge, an + // alert banner) has no backdrop, and its top/bottom edge is frequently a solid // fill or border that legitimately matches its corner pixel — stripping it // would eat real content (m1hP1j: the alert's own uniform red top/bottom // edge was mistaken for backdrop and trimmed, moving the crop out of // alignment with the design reference). Gate the strip on the capture // actually being a dialog. const isDialogCapture = handle - ? await handle.evaluate((el) => el.closest('[role="dialog"],[role="alertdialog"]') !== null) - : await locator.evaluate((el) => el.closest('[role="dialog"],[role="alertdialog"]') !== null); + ? await handle.evaluate((el) => el.matches('[role="dialog"],[role="alertdialog"]')) + : await locator.evaluate((el) => el.matches('[role="dialog"],[role="alertdialog"]')); if (isDialogCapture) { const NEAR_UNIFORM_TOLERANCE = 6; // per RGB channel, 0-255 const stripBudget = Math.max(0, 4 - heightAlreadyRemoved); // device px per edge diff --git a/web/e2e/screenshots/slice-3.spec.ts b/web/e2e/screenshots/slice-3.spec.ts index 687d07e40..af8e46555 100644 --- a/web/e2e/screenshots/slice-3.spec.ts +++ b/web/e2e/screenshots/slice-3.spec.ts @@ -245,9 +245,29 @@ test.describe("Slice 3: Create Server, Modules, Backups (Desktop — 1440x900) @ await expect(dialog).toBeVisible(); await expect(dialog.getByText("a1b2c3d4e5f6", { exact: true })).toBeVisible(); await expect(dialog.getByText("1.4 GiB", { exact: true })).toBeVisible(); + await expect(dialog.getByText("Consistency has not been reported.")).toBeVisible(); + await expect(dialog.getByText("Capture-time volume metadata is unavailable.", { exact: false })).toBeVisible(); await captureLocator(page, "zhLZN", dialog); }); + test("bHJy6: Backup recorded volume coverage", async ({ page }) => { + await setTheme(page, "light"); + await page.context().addCookies([{ + name: "e2e_backup_coverage", value: "recorded", domain: "localhost", path: "/", + }]); + await page.goto("/backups"); + const row = page.getByRole("row").filter({ hasText: "mc-survival-nightly-0713" }).filter({ hasText: "local / default" }); + await row.getByRole("rowheader").click(); + const dialog = page.getByRole("dialog", { name: /backup details/i }); + await expect(dialog.getByText("The game does not support quiesce.")).toBeVisible(); + const coverage = dialog.getByText("Volume coverage", { exact: true }).locator(".."); + await expect(coverage.getByText("Included", { exact: true })).toHaveCount(1); + await expect(coverage.getByText("Not covered", { exact: true })).toHaveCount(2); + await expect(coverage.getByText("Rebuildable", { exact: true })).toBeVisible(); + await expect(coverage.getByText("Unclassified", { exact: true })).toBeVisible(); + await captureLocator(page, "bHJy6", coverage); + }); + test("E9EEv0: Restore Backup dialog", async ({ page }) => { // Design PNG is light theme — see setTheme()'s note. await setTheme(page, "light"); diff --git a/web/scripts/compare-screenshots.mjs b/web/scripts/compare-screenshots.mjs index 9576ab1ca..d42510491 100755 --- a/web/scripts/compare-screenshots.mjs +++ b/web/scripts/compare-screenshots.mjs @@ -153,10 +153,10 @@ const SCALE_ALLOWLIST = { // alpha >= 250 bounding box. Re-measure it whenever the id is re-exported // or its shadow/size changes. const REFERENCE_CROP_ALLOWLIST = { - // Backup Detail Drawer: 440x760 panel at 2x; shadow offset x -12 / blur 32 / - // spread -8 leaves 44 CSS px bleed left, 20 right, 32 top and bottom (bbox - // (88,64)-(968,1584) => 880x1520). - zhLZN: { left: 88, top: 64, width: 880, height: 1520 }, + // Backup Detail Drawer: 440x760 panel at 2x. The refreshed Pencil export's + // shadow leaves 60 CSS px bleed left, 36 right, and 48 top and bottom; + // its alpha >= 250 panel bounds are (120,96)-(1000,1616) => 880x1520. + zhLZN: { left: 120, top: 96, width: 880, height: 1520 }, // Add module source dialog: 480x882 panel at 2x, bounding box re-verified // via PIL alpha>=250 scan of assets/design-export/screenshots/DMnEi.png (now // 1088x1898, bbox (64,40)-(1024,1810) => 960x1770). diff --git a/web/specs.md b/web/specs.md index 02893167c..bf1749378 100644 --- a/web/specs.md +++ b/web/specs.md @@ -10,6 +10,10 @@ The Gameplane dashboard is a React SPA providing a UI layer over the Gameplane A ## Responsibilities +GameServer domain types include optional `spec.templateRollout` policy/exact +revision request and `status.templateRevision` desired/effective/running digests. +These are CRD/API contracts; no new rollout controls or visual flow are defined. + - Render all authenticated Gameplane CRDs and operations as React pages and interactive forms - Handle user authentication (local + OIDC) and session management (cookies + CSRF tokens) - Fetch and cache data via TanStack Query; invalidate caches on Kubernetes watch events (SSE) diff --git a/web/src/components/backups/BackupDetailDrawer.test.tsx b/web/src/components/backups/BackupDetailDrawer.test.tsx index 8e5056db3..835470dd7 100644 --- a/web/src/components/backups/BackupDetailDrawer.test.tsx +++ b/web/src/components/backups/BackupDetailDrawer.test.tsx @@ -8,6 +8,34 @@ import { makeBackup } from "@/test/factories"; import { BackupDetailDrawer } from "./BackupDetailDrawer"; describe("BackupDetailDrawer", () => { + it("shows captured and excluded claims independently of copy success", async () => { + server.use(http.get("/backups/coverage", () => HttpResponse.json(makeBackup({ + metadata: { name: "coverage" }, + status: { + phase: "Succeeded", snapshotID: "copy", + conditions: [{ type: "Consistency", status: "False", reason: "UnsupportedRawCopy", message: "The game does not support quiesce." }], + captureManifest: { state: "Recorded", recordedAt: "2026-10-21T12:00:00Z", volumes: [ + { claimName: "game-data", claimUID: "world-uid", mountPaths: ["/world"], included: true, classification: "data", reason: "PrimaryDataClaim" }, + { claimName: "game-extra-install", mountPaths: ["/serverfiles"], included: false, classification: "rebuildable", reason: "NotCovered" }, + { claimName: "game-mods", mountPaths: ["/world/mods"], included: false, classification: "unclassified", reason: "NotCovered" }, + ] }, + }, + })))); + renderWithQuery( {}} onRestore={() => {}} />); + expect(await screen.findByText("The game does not support quiesce.")).toBeInTheDocument(); + expect(screen.getByText("game-data")).toBeInTheDocument(); + expect(screen.getByText("/world/mods")).toBeInTheDocument(); + expect(screen.getAllByText("Not covered")).toHaveLength(2); + expect(screen.getByText("Rebuildable")).toBeInTheDocument(); + }); + + it("shows legacy coverage as unknown without inventing a current layout", async () => { + server.use(http.get("/backups/legacy", () => HttpResponse.json(makeBackup({ metadata: { name: "legacy" }, status: { phase: "Succeeded" } })))); + renderWithQuery( {}} onRestore={() => {}} />); + expect(await screen.findByText(/Capture-time volume metadata is unavailable/)).toBeInTheDocument(); + expect(screen.queryByText("Included")).not.toBeInTheDocument(); + }); + it("does not render content when name is null", () => { renderWithQuery( {}} onRestore={() => {}} />); expect(screen.queryByText(/Backup details/i)).not.toBeInTheDocument(); diff --git a/web/src/components/backups/BackupDetailDrawer.tsx b/web/src/components/backups/BackupDetailDrawer.tsx index 7e9422231..eff9875e1 100644 --- a/web/src/components/backups/BackupDetailDrawer.tsx +++ b/web/src/components/backups/BackupDetailDrawer.tsx @@ -9,6 +9,7 @@ import type { Backup } from "@/types"; import { PhaseChip } from "@/components/ui/PhaseChip"; import { ErrorBanner } from "./ErrorBanner"; import { useOptionalIntl } from "@/lib/i18n/format"; +import { useConditionMessage } from "@/lib/i18n/conditionMessage"; interface Props { target?: ResourceTarget; @@ -44,6 +45,8 @@ export function BackupDetailDrawer({ name, ns, onClose, onRestore, target: expli const restorable = backup?.status?.phase === "Succeeded" && Boolean(backup.status.snapshotID); + const consistency = useConditionMessage(backup?.status?.conditions?.find((c) => c.type === "Consistency")); + const manifest = backup?.status?.captureManifest; return ( {backup.spec.serverRef.name} + + {consistency ?? intl.formatMessage({ id: "web.components.backups.backup_detail_drawer.consistency_unknown", defaultMessage: "Consistency has not been reported.", description: "Backup details: no achieved-consistency condition is available" })} + + + {manifest?.state === "Recorded" ? ( +
+ {manifest.recordedAt &&
{intl.formatMessage({ id: "web.components.backups.backup_detail_drawer.coverage_recorded", defaultMessage: "Recorded: {time}", description: "Backup details: timestamp of the immutable capture manifest" }, { time: manifest.recordedAt })}
} +

{intl.formatMessage({ id: "web.components.backups.backup_detail_drawer.coverage_scope", defaultMessage: "Only the primary data claim is copied. Rebuildable is a declaration, not proof that recovery can reconstruct its contents.", description: "Backup details: explains the current single-volume backup scope and rebuildable declarations" })}

+
    + {manifest.volumes?.map((volume) => ( +
  • +
    {volume.claimName}
    +
    {volume.included + ? intl.formatMessage({ id: "web.components.backups.backup_detail_drawer.included", defaultMessage: "Included", description: "Backup volume: this claim is selected for copying; success is reported separately" }) + : intl.formatMessage({ id: "web.components.backups.backup_detail_drawer.not_covered", defaultMessage: "Not covered", description: "Backup volume: contents of this claim are not included" })}
    +
    {intl.formatMessage({ id: "web.components.backups.backup_detail_drawer.classification", defaultMessage: "{role, select, data {Data} rebuildable {Rebuildable} other {Unclassified}}", description: "Backup volume: optional declaration from the module or server at capture time" }, { role: volume.classification })}
    + {volume.mountPaths?.map((path) =>
    {path}
    )} + {volume.reason === "RetainedUnmountedClaim" &&
    {intl.formatMessage({ id: "web.components.backups.backup_detail_drawer.retained_unmounted", defaultMessage: "Retained claim; not mounted in the captured workload.", description: "Backup volume: an older server-owned claim was retained but was not part of the observed pod layout" })}
    } +
    • + ))} +
+
+ ) : intl.formatMessage({ id: "web.components.backups.backup_detail_drawer.legacy_coverage", defaultMessage: "Capture-time volume metadata is unavailable. Inspect the original backup and module configuration before recovery.", description: "Backup details: older backups have no reliable volume manifest, so the current template is not used to invent coverage" })} + {backup.status?.snapshotID ?? intl.formatMessage({ id: "web.common.empty_value", defaultMessage: "—", description: "Shown in place of a value that is not set" })} diff --git a/web/src/test/handlers.ts b/web/src/test/handlers.ts index e3f0d05a2..1186a0f23 100644 --- a/web/src/test/handlers.ts +++ b/web/src/test/handlers.ts @@ -1254,7 +1254,7 @@ export function buildScreenshotHandlers() { ], }), ), - http.get("/backups/:name", ({ params }) => { + http.get("/backups/:name", ({ params, cookies }) => { const name = String(params.name); if (name === "mc-survival-nightly-0713") { return HttpResponse.json( @@ -1267,6 +1267,18 @@ export function buildScreenshotHandlers() { completionTime: "2026-07-13T00:14:41Z", size: "1.4 GiB", snapshotID: "a1b2c3d4e5f6", + conditions: cookies.e2e_backup_coverage === "recorded" + ? [{ type: "Consistency", status: "False", reason: "UnsupportedRawCopy", message: "The game does not support quiesce." }] + : undefined, + captureManifest: cookies.e2e_backup_coverage === "recorded" ? { + state: "Recorded", + recordedAt: "2026-07-13T00:12:04Z", + volumes: [ + { claimName: "game-data", mountPaths: ["/world"], included: true, classification: "data", reason: "PrimaryDataClaim" }, + { claimName: "game-extra-install", mountPaths: ["/serverfiles"], included: false, classification: "rebuildable", reason: "NotCovered" }, + { claimName: "game-mods", mountPaths: ["/world/mods"], included: false, classification: "unclassified", reason: "NotCovered" }, + ], + } : undefined, }, }), ); diff --git a/web/src/types.ts b/web/src/types.ts index 1832222bc..834a8d43c 100644 --- a/web/src/types.ts +++ b/web/src/types.ts @@ -551,6 +551,7 @@ export interface GameServer { metadata: ObjectMeta & { resourceVersion?: string }; spec: { templateRef: { name: string }; + templateRollout?: { policy?: "Automatic" | "OnRequest"; requestedRevision?: string }; suspend?: boolean; // Seconds the operator waits for the template's stop sequence to run // over RCON before scaling the pod down (soft-stop). Range 0–600, @@ -586,6 +587,7 @@ export interface GameServer { }; status?: { phase?: GameServerPhase; + templateRevision?: { desired?: string; effective?: string; running?: string }; endpoints?: GameServerEndpoint[]; agent?: { lastHeartbeat?: string; @@ -654,6 +656,23 @@ export interface ServerEvent { count: number; } +export interface BackupCaptureManifest { + state: "Recorded" | "LegacyUnknown"; + recordedAt?: string; + serverUID?: string; + strategy?: string; + layoutSource?: string; + quiesceOutcome?: string; + volumes?: Array<{ + claimName: string; + claimUID?: string; + mountPaths?: string[]; + included: boolean; + classification: "data" | "rebuildable" | "unclassified"; + reason: string; + }>; +} + export interface Backup { metadata: ObjectMeta; // strategy: restic-snapshot writes to a restic repo; volume-snapshot takes a @@ -665,6 +684,8 @@ export interface Backup { completionTime?: string; snapshotID?: string; size?: string; + captureManifest?: BackupCaptureManifest; + conditions?: Array<{ type: string; status: string; reason?: string; message?: string }>; }; }