fix(operator): serialize backup restore and wipe lifecycle ownership

.github/workflows/ci.yaml

@@ -443,7 +443,8 @@ jobs:
     needs: [changes]
     if: (needs.changes.outputs.images == 'true') || contains(fromJSON(needs.changes.outputs.ci-jobs || '[]'), 'build-images')
     runs-on: ubuntu-latest
-    timeout-minutes: 15
+    # Include all image builds, cache exports, artifact upload and cleanup.
+    timeout-minutes: 30
     permissions:
       contents: read
     steps:

api/internal/handlers/resources_annotations_test.go

@@ -39,6 +39,7 @@ func TestGenericServerWritesCannotInjectLifecycleAnnotations(t *testing.T) {
 					wipeRequestedAnnotation:                        "injected",
 					"gameplane.local/wipe-data-completed":          "forged",
 					"gameplane.local/restore-guard":                "forged",
+					"operations.gameplane.local/owner":             "forged",
 					"gameserver.gameplane.local/stop-requested-at": "forged",
 					"restore.gameplane.local/created-by":           "forged",
 					ownerIDAnnotation:                              "999",
@@ -51,7 +52,7 @@ func TestGenericServerWritesCannotInjectLifecycleAnnotations(t *testing.T) {
 					t.Fatalf("write failed: %d %s", response.Code, response.Body)
 				}
 				got := serverAnnotations(t, k, "alpha")
-				for _, key := range []string{wipeRequestedAnnotation, "gameplane.local/wipe-data-completed", "gameplane.local/restore-guard", "gameserver.gameplane.local/stop-requested-at", "restore.gameplane.local/created-by"} {
+				for _, key := range []string{wipeRequestedAnnotation, "gameplane.local/wipe-data-completed", "gameplane.local/restore-guard", "operations.gameplane.local/owner", "gameserver.gameplane.local/stop-requested-at", "restore.gameplane.local/created-by"} {
 					if _, found := got[key]; found {
 						t.Errorf("generic write injected %s", key)
 					}
@@ -98,6 +99,7 @@ func TestGenericServerUpdatePreservesControllerState(t *testing.T) {
 				wipeRequestedAnnotation:                        "pending",
 				"gameplane.local/wipe-data-completed":          "previous",
 				"gameplane.local/backup-guard":                 "active-backup",
+				"operations.gameplane.local/owner":             `{"kind":"Backup","name":"copy","uid":"copy-uid","targetUID":"server-uid","generation":1}`,
 				"gameserver.gameplane.local/stop-requested-at": "original-time",
 				"restore.gameplane.local/created-by":           "original-restore",
 			}

docs/operator-operations.md

@@ -0,0 +1,49 @@
+# Backup, restore and wipe ownership
+
+Gameplane serializes these operations per GameServer. All backups participate,
+including backups with `quiesce: false` and CSI VolumeSnapshots. A held operation
+also defers idle sleep/wake, restart, and pod-template rollout. Requests remain
+pending and reconciliation resumes after the operation's cleanup completes.
+
+The GameServer annotation `operations.gameplane.local/owner` identifies the
+operation kind, name, UID, admitted generation and target UID. A wipe is identified
+by the GameServer UID and its request token. The existing backup, restore and wipe
+owner annotations remain compatibility markers. A `recovery` array identifies
+older owners whose workers or quiesce obligations must be cleaned up first.
+
+Deleting an operation requests cleanup. Restic Jobs are canceled with foreground
+deletion; live and terminating worker pods must drain before the owner is released.
+Quiesced backups also owe a successful save-on call. The existing bounded Backup
+deletion retry may release its finalizer, but an unconfirmed save-on leaves the
+durable owner behind for recovery by the next operation. Time alone never releases
+ownership. A pending CSI capture retains the current capture-drain behavior; a
+separate CSI timeout change is tracked in issue #719.
+
+If an operation waits unexpectedly, inspect the referenced Backup/Restore, its
+conditions, Jobs and worker pods. Resolve terminating workers, repository failures
+or agent connectivity before retrying. Removing owner annotations or force-removing
+finalizers cannot establish that a worker has stopped. A failed or canceled restore
+leaves its target suspended for inspection.
+
+Upgrades recognize existing owner annotations and import backup recovery lineage.
+Already-running legacy quiesced copies continue being observed until their workers
+finish and cleanup succeeds. Legacy raw workers also block new admission. Conflicting
+legacy guards fail closed until their existing operations finish cleanup. Keep a
+single active operator version during upgrade; older binaries do not honor the
+shared owner. No new CRD fields or Kubernetes permissions are required.
+
+CSI restores create a new target and reserve it for the Restore. Initial startup
+is allowed under that reservation so storage using WaitForFirstConsumer can bind.
+Other operations wait until the restore completes. Existing data-PVC restores
+continue to require an observed stopped workload and drained writers.
+
+If initial startup fails or the Restore is deleted, cleanup first retires its
+startup permission and suspends the target. The operation remains held until the
+StatefulSet has observed zero replicas and every live or terminating game/PVC
+writer has drained. Recovery of a force-deleted Restore applies the same check
+before a successor can proceed, preserving the existing pod template while
+requesting zero replicas. A successful Restore leaves its initial workload running.
+
+Stopped-source preparation (#716), Restic preparation and quiesce deadlines (#717),
+CSI capture deadlines (#719), and staged template adoption (#731) are separate
+changes building on this ownership boundary.

operator/api/v1alpha1/backup_types.go

@@ -17,11 +17,10 @@ const (
 	BackupPhaseFailed    BackupPhase = "Failed"
 )
 
-// BackupFinalizer blocks a quiesced Backup's deletion until the operator has
-// sent the matching unquiesce. Without it, deleting a Backup while
-// quiesce-attempted=true drops the unquiesce entirely and the game is left
-// with auto-save off indefinitely (F-048). Added only for Backups with
-// spec.quiesce=true; see backup_forget.go's finalizeDelete.
+// BackupFinalizer blocks deletion until copy workers have drained and any
+// matching unquiesce has completed. Every Backup carries it, including raw
+// copies and CSI snapshots, so deletion cannot release shared lifecycle
+// ownership while a copy is still using the target. See finalizeDelete.
 const BackupFinalizer = "gameplane.local/backup-finalizer"
 
 // BackupSnapshotFinalizer blocks the deletion of a restic-strategy Backup until

operator/cmd/main.go

@@ -401,6 +401,7 @@ func main() {
 		AgentCASecretName:      agentCASecretName,
 		AgentCASecretNamespace: agentCASecretNamespace,
 		AgentClient:            agentClient,
+		OperationQuiescer:      agentClient,
 		PodAttacher: &controller.StopAttachClient{
 			Config:    mgr.GetConfig(),
 			Clientset: kubernetes.NewForConfigOrDie(mgr.GetConfig()),
@@ -448,6 +449,7 @@ func main() {
 		Client:                   mgr.GetClient(),
 		APIReader:                mgr.GetAPIReader(),
 		Scheme:                   mgr.GetScheme(),
+		OperationQuiescer:        agentClient,
 		ResticImage:              resticImage,
 		JobBackoffLimit:          &backupJobBackoff32,
 		JobActiveDeadlineSeconds: backupJobActiveDeadlineSeconds,

operator/internal/controller/backup_controller.go

@@ -247,14 +247,12 @@ func (r *BackupReconciler) reconcile(ctx context.Context, req ctrl.Request) (ctr
 	if !b.DeletionTimestamp.IsZero() {
 		return r.finalizeDelete(ctx, &b)
 	}
-	// Only Backups that might quiesce the world need to block on releasing
-	// it before deletion; everything else deletes immediately.
-	if b.Spec.Quiesce && !controllerutil.ContainsFinalizer(&b, gameplanev1alpha1.BackupFinalizer) {
+	// Every copy owns the source until its workers drain, even without quiesce.
+	if !controllerutil.ContainsFinalizer(&b, gameplanev1alpha1.BackupFinalizer) {
 		controllerutil.AddFinalizer(&b, gameplanev1alpha1.BackupFinalizer)
 		if err := r.Update(ctx, &b); err != nil {
 			return ctrl.Result{}, err
 		}
-		return ctrl.Result{}, nil
 	}
 
 	// Restic Backups own a snapshot in the repository, so hold their deletion
@@ -288,6 +286,19 @@ func (r *BackupReconciler) reconcile(ctx context.Context, req ctrl.Request) (ctr
 		b.Status.Phase == gameplanev1alpha1.BackupPhaseFailed {
 		return r.runUnquiesce(ctx, &b)
 	}
+	// Observe pre-upgrade workers even when multiple legacy quiesced copies
+	// already overlap. They must finish and unquiesce before new admission;
+	// trying to acquire exclusive ownership first would deadlock their cleanup.
+	if legacyBackupOwesUnquiesce(&b) && b.Spec.Strategy != "volume-snapshot" {
+		var existing batchv1.Job
+		err := r.backupAPIReader().Get(ctx, client.ObjectKeyFromObject(&b), &existing)
+		if err != nil && !apierrors.IsNotFound(err) {
+			return ctrl.Result{}, err
+		}
+		if err == nil && metav1.IsControlledBy(&existing, &b) {
+			return r.mirrorJobStatus(ctx, &b, &existing)
+		}
+	}
 
 	// Resolve the target before building the Job: a Backup against a
 	// GameServer that doesn't exist would produce a pod waiting forever
@@ -383,6 +394,9 @@ func (r *BackupReconciler) maybeQuiesce(ctx context.Context, b *gameplanev1alpha
 	}
 	// Persist cleanup intent BEFORE save-off: a crash or a partial agent failure
 	// must still owe save-on even when quiesced-at was never recorded.
+	if err := r.setLifecycleUnquiesceIntent(ctx, b, true); err != nil {
+		return err
+	}
 	patchBackupAnnotations(b, map[string]string{annoQuiesceAttempted: "pending"})
 	if err := r.Update(ctx, b); err != nil {
 		return err
@@ -402,6 +416,9 @@ func (r *BackupReconciler) maybeQuiesce(ctx context.Context, b *gameplanev1alpha
 		patchBackupAnnotations(b, map[string]string{
 			annoQuiesceAttempted: "unsupported",
 		})
+		if err := r.setLifecycleUnquiesceIntent(ctx, b, false); err != nil {
+			return err
+		}
 	default:
 		// Hard failure (network, RCON down). Surface as Failed: a
 		// backup taken against an unflushed Minecraft world can be

operator/internal/controller/backup_envtest_test.go

@@ -577,6 +577,7 @@ func TestBackup_UsesTemplateSecurityContext(t *testing.T) {
 	if err := k8sClient.Create(context.Background(), gs); err != nil {
 		t.Fatalf("create gameserver: %v", err)
 	}
+	seedBackupWorkload(t, gs)
 	if err := k8sClient.Create(context.Background(), buildResticRepoSecret(ns, "repo-secret")); err != nil {
 		t.Fatalf("create secret: %v", err)
 	}

operator/internal/controller/backup_forget_envtest_test.go

@@ -16,6 +16,7 @@ import (
 	"k8s.io/client-go/tools/events"
 	"k8s.io/client-go/util/retry"
 
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 	"sigs.k8s.io/controller-runtime/pkg/manager"
 
 	gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1"
@@ -149,17 +150,17 @@ func TestBackupForget_FinalizerOnlyOnResticBackups(t *testing.T) {
 		return false, "restic backup has no snapshot finalizer yet"
 	})
 
-	// Once the volume-snapshot Backup is Running the reconciler has been through
-	// the point where it would have added a finalizer.
+	// Every strategy gets an operation cleanup finalizer before admission. Do
+	// not wait for Running: the restic backup may hold the shared server owner.
 	eventually(t, func() (bool, string) {
 		b := getBackup(t, ns, "smp-vs")
-		if b.Status.Phase != gameplanev1alpha1.BackupPhaseRunning {
+		if !controllerutil.ContainsFinalizer(b, gameplanev1alpha1.BackupFinalizer) {
 			return false, describeBackupStatus(b)
 		}
 		return true, ""
 	})
-	if got := getBackup(t, ns, "smp-vs").Finalizers; len(got) != 0 {
-		t.Errorf("volume-snapshot backup finalizers = %v, want none", got)
+	if got := getBackup(t, ns, "smp-vs"); controllerutil.ContainsFinalizer(got, gameplanev1alpha1.BackupSnapshotFinalizer) {
+		t.Errorf("volume-snapshot backup finalizers = %v, want no restic snapshot finalizer", got.Finalizers)
 	}
 }
 

operator/internal/controller/backup_forget_test.go

@@ -16,6 +16,7 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/client/fake"
 	"sigs.k8s.io/controller-runtime/pkg/client/interceptor"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 
 	gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1"
 )
@@ -180,7 +181,7 @@ func TestReconcile_AddsSnapshotFinalizerForResticBackup(t *testing.T) {
 }
 
 // Volume-snapshot and Failed Backups never write a restic snapshot, so they
-// must not be held on deletion.
+// need operation cleanup but must not be held for restic snapshot deletion.
 func TestReconcile_NoSnapshotFinalizerWhenNothingToForget(t *testing.T) {
 	volume := forgetBackup()
 	volume.DeletionTimestamp = nil
@@ -205,8 +206,11 @@ func TestReconcile_NoSnapshotFinalizerWhenNothingToForget(t *testing.T) {
 			if !ok {
 				t.Fatal("backup vanished")
 			}
-			if len(got.Finalizers) != 0 {
-				t.Errorf("finalizers = %v, want none", got.Finalizers)
+			if !controllerutil.ContainsFinalizer(got, gameplanev1alpha1.BackupFinalizer) {
+				t.Errorf("finalizers = %v, want operation cleanup finalizer", got.Finalizers)
+			}
+			if controllerutil.ContainsFinalizer(got, gameplanev1alpha1.BackupSnapshotFinalizer) {
+				t.Errorf("finalizers = %v, want no restic snapshot finalizer", got.Finalizers)
 			}
 		})
 	}