fix: prepare backup workers before quiesce and bound CSI captures

.github/workflows/ci.yaml

@@ -1826,7 +1826,7 @@ jobs:
         # (operator does zero logins and is wait-dominated, so it runs
         # wide; the api buckets stay within the login limiter's budget).
         # The arm64 job below uses the same steps and per-bucket settings.
-        bucket: [operator, api-auth, api-roles, api-rbac, api-agent, api-mods, telemetry, optional-components, restricted-workloads, management-placement]
+        bucket: [operator, api-auth, api-roles, api-rbac, api-agent, api-mods, telemetry, optional-components, restricted-workloads, management-placement, backup-preparation]
         arch: [amd64]
         include:
           - bucket: operator
@@ -1837,6 +1837,10 @@ jobs:
             parallel: 1
             test_timeout: 15m
             job_timeout: 60
+          - bucket: backup-preparation
+            parallel: 1
+            test_timeout: 15m
+            job_timeout: 60
           - bucket: api-auth
             parallel: 4
             test_timeout: 15m
@@ -1898,7 +1902,7 @@ jobs:
       - name: bring up cluster + helm install
         if: env.RUN_REGEX != '' || env.TAIL_REGEX != ''
         env:
-          GAMEPLANE_E2E_PROFILE: ${{ matrix.bucket == 'management-placement' && 'management' || 'default' }}
+          GAMEPLANE_E2E_PROFILE: ${{ (matrix.bucket == 'management-placement' || matrix.bucket == 'backup-preparation') && 'management' || 'default' }}
         run: ./deploy/kind/e2e.sh up "$CLUSTER" e2e "$GAMEPLANE_E2E_PROFILE"
       - name: run go e2e (${{ matrix.bucket }})
         if: env.RUN_REGEX != ''
@@ -2716,7 +2720,7 @@ jobs:
               'go-e2e-unit': (n) => n === 'go (test/e2e)',
               'api-postgres': (n) => n === 'api (postgres)',
               'e2e-buckets': (n) => n === 'e2e bucket coverage',
-              'e2e-go': (n) => /^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components|restricted-workloads|management-placement) \//.test(n) && n.includes('/ amd64 '),
+              'e2e-go': (n) => /^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components|restricted-workloads|management-placement|backup-preparation) \//.test(n) && n.includes('/ amd64 '),
               'e2e-go-arm64': (n) => /^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components|restricted-workloads) \//.test(n) && n.includes('/ arm64 '),
               'e2e-multicluster': (n) => n.startsWith('e2e multicluster /') && n.includes('/ amd64 '),
               'e2e-multicluster-arm64': (n) => n.startsWith('e2e multicluster /') && n.includes('/ arm64 '),
@@ -2869,7 +2873,7 @@ jobs:
             const bucketSet = new Set();
             for (const j of apiJobs) {
               const n = j.name || '';
-              const m = n.match(/^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components|restricted-workloads|management-placement) \//);
+              const m = n.match(/^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components|restricted-workloads|management-placement|backup-preparation) \//);
               if (m) bucketSet.add(m[1]);
               if (/^e2e multicluster \//.test(n)) bucketSet.add('multicluster');
               if (/^e2e upgrade \//.test(n)) bucketSet.add('upgrade');

charts/gameplane/templates/operator.yaml

@@ -83,6 +83,11 @@ rules:
   - apiGroups: ["snapshot.storage.k8s.io"]
     resources: [volumesnapshots, volumesnapshotcontents]
     verbs: [get, list, watch]
+  # Abort reclamation is restricted in the controller to recorded snapshot
+  # references and exact content UIDs; VolumeSnapshotContent is cluster-scoped.
+  - apiGroups: [snapshot.storage.k8s.io]
+    resources: [volumesnapshotcontents]
+    verbs: [delete]
   # The GameServer controller registers Owns(&networkingv1.NetworkPolicy{})
   # unconditionally (regardless of networkPolicies.enabled), and the
   # manager's cache is cluster-scoped (no Cache/DefaultNamespaces option
@@ -326,6 +331,18 @@ spec:
             {{- if .Values.operator.resticImage }}
             - --restic-image={{ .Values.operator.resticImage }}
             {{- end }}
+            {{- with .Values.operator.backupPreparationTimeout }}
+            - --backup-preparation-timeout={{ . }}
+            {{- end }}
+            {{- with .Values.operator.backupQuiesceTimeout }}
+            - --backup-quiesce-timeout={{ . }}
+            {{- end }}
+            {{- with .Values.operator.backupCSICaptureTimeout }}
+            - --backup-csi-capture-timeout={{ . }}
+            {{- end }}
+            {{- if and (hasKey .Values.operator "backupHardSameNode") (ne .Values.operator.backupHardSameNode nil) }}
+            - --backup-hard-same-node={{ .Values.operator.backupHardSameNode }}
+            {{- end }}
             {{- /* Omit the flags when the keys are absent/null (e.g. --reuse-values
                    from a release predating them) so the operator's defaults
                    (2 / 86400) apply. Not `| default`: an explicit 0 backoff is

charts/gameplane/values.yaml

@@ -134,6 +134,13 @@ operator:
   # pod cannot hold a Backup/Restore in flight forever. Defaults: 2 and 86400 (24h).
   backupJobBackoffLimit: 2
   backupJobActiveDeadlineSeconds: 86400
+  # Preparation (scheduling, image pull, PVC mount, repository/lock probe)
+  # precedes save-off. Quiesce expiry cancels and drains the copy before save-on.
+  backupPreparationTimeout: 10m
+  backupQuiesceTimeout: 5m
+  backupCSICaptureTimeout: 10m
+  # Prefer the observed game's node by default; opt in to a hard requirement.
+  backupHardSameNode: false
   # Mount a directory of module bundles into the operator so local-type
   # ModuleSources can index it (homelab: drop module dirs on disk and
   # they appear in the catalog). Provide exactly one of existingClaim

docs/operator-operations.md

@@ -13,8 +13,8 @@ already stopped and its writers drained before capture; the backup did not stop
 it. Failed or unfinished copies, including completed Jobs still awaiting a
 snapshot ID, cannot report achieved quiesce or stopped-source consistency.
 
-`status.captureManifest` is recorded once before the first copy worker is
-created, after lifecycle ownership and quiesce preparation. It lists the primary
+`status.captureManifest` is recorded once before copy authorization or CSI
+capture, after lifecycle ownership and quiesce preparation. It lists the primary
 `<server>-data` claim as included and every other observed persistent claim as
 uncovered, including extra volumes, nested mod/loader mounts and retained
 server-owned claims. It records claim UIDs, observed mount paths, source layout
@@ -53,8 +53,9 @@ deletion; live and terminating worker pods must drain before the owner is releas
 Quiesced backups also owe a successful save-on call. The existing bounded Backup
 deletion retry may release its finalizer, but an unconfirmed save-on leaves the
 durable owner behind for recovery by the next operation. Time alone never releases
-ownership. A pending CSI capture retains the current capture-drain behavior; a
-separate CSI timeout change is tracked in issue #719.
+ownership. CSI cancellation or expiry first invalidates the capture, then allows
+save-on independently of driver reclamation. A late-ready aborted capture remains
+unrestorable.
 
 If an operation waits unexpectedly, inspect the referenced Backup/Restore, its
 conditions, Jobs and worker pods. Resolve terminating workers, repository failures
@@ -71,7 +72,9 @@ Already-running legacy quiesced copies continue being observed until their worke
 finish and cleanup succeeds. Legacy raw workers also block new admission. Conflicting
 legacy guards fail closed until their existing operations finish cleanup. Keep a
 single active operator version during upgrade; older binaries do not honor the
-shared owner. No new CRD fields or Kubernetes permissions are required.
+shared owner. CSI abort reclamation adds cluster-scoped delete permission for
+VolumeSnapshotContents; runtime checks restrict it to recorded, UID-verified
+contents with `Delete` policy.
 
 CSI restores create a new target and reserve it for the Restore. Initial startup
 is allowed under that reservation so storage using WaitForFirstConsumer can bind.
@@ -85,6 +88,58 @@ writer has drained. Recovery of a force-deleted Restore applies the same check
 before a successor can proceed, preserving the existing pod template while
 requesting zero replicas. A successful Restore leaves its initial workload running.
 
-Stopped-source preparation (#716), Restic preparation and quiesce deadlines (#717),
-CSI capture deadlines (#719), and staged template adoption (#731) are separate
-changes building on this ownership boundary.
+Stopped-source proof, bounded copy preparation and CSI invalidation build on this
+ownership boundary. Staged template adoption is tracked separately in #731.
+
+## Backup preparation and capture budgets
+
+Restic workers now schedule, pull their image, mount the source and validate the
+repository before the agent is asked to disable saving. A Ready worker waits for
+an expiring authorization bound to its pod UID, Backup UID and observed source.
+The preferred node is the game's current node; `operator.backupHardSameNode: true`
+makes that placement mandatory. Source tolerations and `operator.backupResources`
+apply to preparation and copy containers. Stopped sources leave node selection
+to the scheduler and volume affinity.
+
+When the source StatefulSet requests a running workload, initial preparation
+waits for its owned pod to be created and assigned a node before binding the
+worker to that source. This wait uses the preparation budget and does not disable
+saving. A pod or node replacement after binding still fails the copy. Raw backups
+of intentionally stopped workloads remain eligible without a source pod.
+
+The install settings are `operator.backupPreparationTimeout: 10m`,
+`operator.backupQuiesceTimeout: 5m`, and `operator.backupCSICaptureTimeout: 10m`.
+Equivalent operator flags use kebab-case names. The recorded deadlines remain
+fixed across restarts or later configuration edits. Restic's existing Job
+backoff/deadline settings remain separate; raw and already-stopped copies use
+the full Job deadline. A live quiesce attempt retains the short budget even if
+the agent reports the command unsupported. Increase the live quiesce budget
+deliberately for worlds that cannot finish within five minutes, or use a
+supported CSI capture strategy.
+
+Preparation failure leaves saving enabled. Quiesce expiry marks the copy failed,
+revokes authorization and cancels its Job. Saving is restored only after all
+live or terminating workers have stopped. The worker itself also stops Restic
+at the token deadline (TERM, then KILL after five seconds), including when the
+operator is unavailable. This bounds copying; save-on still requires the operator
+and agent to recover. `CaptureReady` describes preparation/authorization or a
+pending drain. Resolve stuck mounts, node failures or terminating workers before
+retrying; removing owner annotations is not proof that copying stopped.
+
+Custom Restic images must provide the restic CLI (including `snapshots --retry-lock`),
+POSIX `/bin/sh`, `date +%s`, `cat`, `touch`, `sleep`, and `timeout -s TERM -k 5`.
+The pinned `restic/restic:0.17.1` release uses Alpine/BusyBox and supports this
+contract. The gate runs under the existing non-root, read-only-root-filesystem
+security context. Node/operator clocks must be synchronized for absolute token
+expiry. No worker receives Kubernetes API credentials.
+
+CSI expiry and Backup deletion invalidate an incomplete capture before save-on.
+The `snapshot-aborted` marker is terminal even if a delayed driver later reports
+readiness. Normal restore and new raw GameServer PVC references reject such
+captures. `SnapshotCleaned` reports reclamation progress and records snapshot
+and content names/UIDs; `Retain` content and unverified replacements require
+administrator action. A slow driver can keep Backup deletion pending for storage
+cleanup after saving and game ownership have already been restored. Inspect the
+condition/event identities before deleting retained storage manually. Existing
+PVCs remain usable after source Backup retention; administrator-managed external
+snapshots keep their direct-CRD behavior.

hack/ci_scope.py

@@ -29,8 +29,8 @@
 FORCE_ALL_PREFIXES = (".github/actions/",)
 # Mirrors the e2e-go matrix buckets in .github/workflows/ci.yaml.
 E2E_BUCKETS = ["operator", "api-auth", "api-roles", "api-rbac", "api-agent", "api-mods",
-               "telemetry", "optional-components", "restricted-workloads", "management-placement"]
-E2E_AMD64_ONLY = {"management-placement"}
+               "telemetry", "optional-components", "restricted-workloads", "management-placement", "backup-preparation"]
+E2E_AMD64_ONLY = {"management-placement", "backup-preparation"}
 SUITES_FILE = "test/e2e/suite-components.json"
 # Paths no Go e2e test reads. Their own jobs keep their paths-filter gates.
 NO_E2E_FILES = {

hack/test_ci_scope.py

@@ -426,6 +426,14 @@ def test_audit_management_placement_has_a_run_pattern(self):
                 self.assertIn("TestManagementPlacementAndDisruption",
                               self.select(path)["e2e-run"].get("management-placement", ""))
 
+    def test_audit_backup_preparation_has_a_run_pattern(self):
+        """Worker preparation remains selected for each covered component."""
+        for path in ("operator/internal/controller/backup_worker.go", "api/cmd/main.go",
+                     "agent/cmd/main.go", "test/e2e/backup_preparation_e2e_test.go"):
+            with self.subTest(path=path):
+                self.assertIn("TestBackup_PreparationAndSameNodeCopy",
+                              self.select(path)["e2e-run"].get("backup-preparation", ""))
+
     def test_audit_staged_rollout_keeps_operator_coverage(self):
         """Template adoption and its owning fixture select the rollout scenario."""
         for path in ("operator/internal/controller/gameserver_template_revision.go",

i18n/locales/en/app.json

@@ -155,6 +155,22 @@
     "defaultMessage": "Requested {requestKind, select, addressAndPool {address ''{address}'' from pool ''{pool}''} address {address ''{address}''} other {pool ''{pool}''}}: waiting for the LoadBalancer Service that carries the request to exist.",
     "description": "AddressAssignment condition before the load-balancer Service exists."
   },
+  "condition.captureready.snapshotcapturepending": {
+    "defaultMessage": "Waiting for CSI capture within its persisted deadline.",
+    "description": "Bounded CSI capture is pending."
+  },
+  "condition.captureready.workerauthorized": {
+    "defaultMessage": "The prepared worker is authorized to capture data within its deadline.",
+    "description": "Backup copy authorization granted after preparation."
+  },
+  "condition.captureready.workerdrainpending": {
+    "defaultMessage": "The copy is invalid. Waiting for workers to stop before restoring saving.",
+    "description": "Backup cancellation awaiting worker drainage."
+  },
+  "condition.captureready.workerpreparing": {
+    "defaultMessage": "Waiting for worker scheduling, source mount and repository preparation.",
+    "description": "Backup preparation before saving is disabled."
+  },
   "condition.completed.failed": {
     "defaultMessage": "The backup or restore failed.",
     "description": "Completed condition on a Backup or Restore that failed. The message holds the English detail."
@@ -579,6 +595,22 @@
     "defaultMessage": "The capture sidecar was told to stop.",
     "description": "SidecarStopped condition after a stop request."
   },
+  "condition.snapshotcleaned.snapshotcleanupcomplete": {
+    "defaultMessage": "Aborted snapshot resources are absent.",
+    "description": "Snapshot reclamation completed."
+  },
+  "condition.snapshotcleaned.snapshotcleanuppending": {
+    "defaultMessage": "Aborted snapshot reclamation is pending. {detail}",
+    "description": "Aborted snapshot deletion has not completed; detail includes resource identities."
+  },
+  "condition.snapshotcleaned.snapshotcleanupretained": {
+    "defaultMessage": "Snapshot content requires administrator reclamation. {detail}",
+    "description": "Retained or replacement snapshot content is left for an administrator."
+  },
+  "condition.snapshotcleaned.snapshotcleanupunverified": {
+    "defaultMessage": "A snapshot resource was replaced. It was not deleted. {detail}",
+    "description": "Snapshot identity verification refused deletion."
+  },
   "condition.snapshotforgotten.forgetabandoned": {
     "defaultMessage": "The snapshot was not removed from the backup repository when the backup was deleted.",
     "description": "SnapshotForgotten condition when the forget step was abandoned."

operator/cmd/main.go

@@ -156,6 +156,10 @@ func main() {
 		captureDefaultMaxSizeBytes       int64
 		backupJobBackoffLimit            int64
 		backupJobActiveDeadlineSeconds   int64
+		backupPreparationTimeout         time.Duration
+		backupQuiesceTimeout             time.Duration
+		backupCSICaptureTimeout          time.Duration
+		backupHardSameNode               bool
 		agentResourcesJSON               string
 		sentinelResourcesJSON            string
 		tunnelResourcesJSON              string
@@ -187,6 +191,10 @@ func main() {
 		"BackoffLimit for the restic backup and restore Jobs (retries before the Job is marked Failed). Must be >= 0. Defaults to 2.")
 	flag.Int64Var(&backupJobActiveDeadlineSeconds, "backup-job-active-deadline-seconds", controller.DefaultBackupJobActiveDeadlineSeconds,
 		"ActiveDeadlineSeconds for the restic backup and restore Jobs. Must be > 0. Defaults to 86400 (24 hours).")
+	flag.DurationVar(&backupPreparationTimeout, "backup-preparation-timeout", controller.DefaultBackupPreparationTimeout, "Maximum worker preparation time before copy authorization; must be positive.")
+	flag.DurationVar(&backupQuiesceTimeout, "backup-quiesce-timeout", controller.DefaultBackupQuiesceTimeout, "Maximum live quiesced Restic copy duration; expiry cancels and drains workers before save-on.")
+	flag.DurationVar(&backupCSICaptureTimeout, "backup-csi-capture-timeout", controller.DefaultBackupCSICaptureTimeout, "Maximum CSI capture duration before invalidation and save-on; must be positive.")
+	flag.BoolVar(&backupHardSameNode, "backup-hard-same-node", false, "Require Restic workers on the observed game node instead of the default preferred placement.")
 	flag.StringVar(&sentinelImage, "sentinel-image", controller.DefaultSentinelImage,
 		"Image for the wake sentinel pod that holds advertised ports while a server is asleep. "+
 			"Point at a private registry mirror for air-gapped installs.")
@@ -326,6 +334,10 @@ func main() {
 		setupLog.Error(nil, "invalid --backup-job-active-deadline-seconds value (must be > 0)", "value", backupJobActiveDeadlineSeconds)
 		os.Exit(1)
 	}
+	if backupPreparationTimeout <= 0 || backupQuiesceTimeout <= 0 || backupCSICaptureTimeout <= 0 {
+		setupLog.Error(nil, "backup preparation, quiesce and CSI capture timeouts must be positive")
+		os.Exit(1)
+	}
 
 	// Validate capture retention flags while they are still the int64 type
 	// flag.Int64Var naturally produces. The minimum of 60 seconds aligns with
@@ -491,6 +503,10 @@ func main() {
 		JobBackoffLimit:          &backupJobBackoff32,
 		JobActiveDeadlineSeconds: backupJobActiveDeadlineSeconds,
 		EventRecorder:            mgr.GetEventRecorder("backup-controller"),
+		PreparationTimeout:       backupPreparationTimeout,
+		QuiesceTimeout:           backupQuiesceTimeout,
+		CSICaptureTimeout:        backupCSICaptureTimeout,
+		HardSameNode:             backupHardSameNode,
 	}).SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to set up controller", "controller", "Backup")
 		os.Exit(1)

operator/config/rbac/role.yaml

@@ -151,6 +151,14 @@ rules:
   - snapshot.storage.k8s.io
   resources:
   - volumesnapshotcontents
+  verbs:
+  - delete
+  - get
+  - list
+  - watch
+- apiGroups:
+  - snapshot.storage.k8s.io
+  resources:
   - volumesnapshots
   verbs:
   - get