diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 4faf68a35..ae83e8fde 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -1826,7 +1826,7 @@ jobs: # (operator does zero logins and is wait-dominated, so it runs # wide; the api buckets stay within the login limiter's budget). # The arm64 job below uses the same steps and per-bucket settings. - bucket: [operator, api-auth, api-roles, api-rbac, api-agent, api-mods, telemetry, optional-components, restricted-workloads, management-placement] + bucket: [operator, api-auth, api-roles, api-rbac, api-agent, api-mods, telemetry, optional-components, restricted-workloads, management-placement, backup-preparation] arch: [amd64] include: - bucket: operator @@ -1837,6 +1837,10 @@ jobs: parallel: 1 test_timeout: 15m job_timeout: 60 + - bucket: backup-preparation + parallel: 1 + test_timeout: 15m + job_timeout: 60 - bucket: api-auth parallel: 4 test_timeout: 15m @@ -1898,7 +1902,7 @@ jobs: - name: bring up cluster + helm install if: env.RUN_REGEX != '' || env.TAIL_REGEX != '' env: - GAMEPLANE_E2E_PROFILE: ${{ matrix.bucket == 'management-placement' && 'management' || 'default' }} + GAMEPLANE_E2E_PROFILE: ${{ (matrix.bucket == 'management-placement' || matrix.bucket == 'backup-preparation') && 'management' || 'default' }} run: ./deploy/kind/e2e.sh up "$CLUSTER" e2e "$GAMEPLANE_E2E_PROFILE" - name: run go e2e (${{ matrix.bucket }}) if: env.RUN_REGEX != '' @@ -2716,7 +2720,7 @@ jobs: 'go-e2e-unit': (n) => n === 'go (test/e2e)', 'api-postgres': (n) => n === 'api (postgres)', 'e2e-buckets': (n) => n === 'e2e bucket coverage', - 'e2e-go': (n) => /^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components|restricted-workloads|management-placement) \//.test(n) && n.includes('/ amd64 '), + 'e2e-go': (n) => /^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components|restricted-workloads|management-placement|backup-preparation) \//.test(n) && n.includes('/ amd64 '), 'e2e-go-arm64': (n) => /^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components|restricted-workloads) \//.test(n) && n.includes('/ arm64 '), 'e2e-multicluster': (n) => n.startsWith('e2e multicluster /') && n.includes('/ amd64 '), 'e2e-multicluster-arm64': (n) => n.startsWith('e2e multicluster /') && n.includes('/ arm64 '), @@ -2869,7 +2873,7 @@ jobs: const bucketSet = new Set(); for (const j of apiJobs) { const n = j.name || ''; - const m = n.match(/^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components|restricted-workloads|management-placement) \//); + const m = n.match(/^e2e (operator|api-auth|api-roles|api-rbac|api-agent|api-mods|telemetry|optional-components|restricted-workloads|management-placement|backup-preparation) \//); if (m) bucketSet.add(m[1]); if (/^e2e multicluster \//.test(n)) bucketSet.add('multicluster'); if (/^e2e upgrade \//.test(n)) bucketSet.add('upgrade'); diff --git a/charts/gameplane/templates/operator.yaml b/charts/gameplane/templates/operator.yaml index 15df3da9c..75ccb20fa 100644 --- a/charts/gameplane/templates/operator.yaml +++ b/charts/gameplane/templates/operator.yaml @@ -83,6 +83,11 @@ rules: - apiGroups: ["snapshot.storage.k8s.io"] resources: [volumesnapshots, volumesnapshotcontents] verbs: [get, list, watch] + # Abort reclamation is restricted in the controller to recorded snapshot + # references and exact content UIDs; VolumeSnapshotContent is cluster-scoped. + - apiGroups: [snapshot.storage.k8s.io] + resources: [volumesnapshotcontents] + verbs: [delete] # The GameServer controller registers Owns(&networkingv1.NetworkPolicy{}) # unconditionally (regardless of networkPolicies.enabled), and the # manager's cache is cluster-scoped (no Cache/DefaultNamespaces option @@ -326,6 +331,18 @@ spec: {{- if .Values.operator.resticImage }} - --restic-image={{ .Values.operator.resticImage }} {{- end }} + {{- with .Values.operator.backupPreparationTimeout }} + - --backup-preparation-timeout={{ . }} + {{- end }} + {{- with .Values.operator.backupQuiesceTimeout }} + - --backup-quiesce-timeout={{ . }} + {{- end }} + {{- with .Values.operator.backupCSICaptureTimeout }} + - --backup-csi-capture-timeout={{ . }} + {{- end }} + {{- if and (hasKey .Values.operator "backupHardSameNode") (ne .Values.operator.backupHardSameNode nil) }} + - --backup-hard-same-node={{ .Values.operator.backupHardSameNode }} + {{- end }} {{- /* Omit the flags when the keys are absent/null (e.g. --reuse-values from a release predating them) so the operator's defaults (2 / 86400) apply. Not `| default`: an explicit 0 backoff is diff --git a/charts/gameplane/values.yaml b/charts/gameplane/values.yaml index 54d297827..d660889ac 100644 --- a/charts/gameplane/values.yaml +++ b/charts/gameplane/values.yaml @@ -134,6 +134,13 @@ operator: # pod cannot hold a Backup/Restore in flight forever. Defaults: 2 and 86400 (24h). backupJobBackoffLimit: 2 backupJobActiveDeadlineSeconds: 86400 + # Preparation (scheduling, image pull, PVC mount, repository/lock probe) + # precedes save-off. Quiesce expiry cancels and drains the copy before save-on. + backupPreparationTimeout: 10m + backupQuiesceTimeout: 5m + backupCSICaptureTimeout: 10m + # Prefer the observed game's node by default; opt in to a hard requirement. + backupHardSameNode: false # Mount a directory of module bundles into the operator so local-type # ModuleSources can index it (homelab: drop module dirs on disk and # they appear in the catalog). Provide exactly one of existingClaim diff --git a/docs/operator-operations.md b/docs/operator-operations.md index 24ef33168..2865f7797 100644 --- a/docs/operator-operations.md +++ b/docs/operator-operations.md @@ -13,8 +13,8 @@ already stopped and its writers drained before capture; the backup did not stop it. Failed or unfinished copies, including completed Jobs still awaiting a snapshot ID, cannot report achieved quiesce or stopped-source consistency. -`status.captureManifest` is recorded once before the first copy worker is -created, after lifecycle ownership and quiesce preparation. It lists the primary +`status.captureManifest` is recorded once before copy authorization or CSI +capture, after lifecycle ownership and quiesce preparation. It lists the primary `-data` claim as included and every other observed persistent claim as uncovered, including extra volumes, nested mod/loader mounts and retained server-owned claims. It records claim UIDs, observed mount paths, source layout @@ -53,8 +53,9 @@ deletion; live and terminating worker pods must drain before the owner is releas Quiesced backups also owe a successful save-on call. The existing bounded Backup deletion retry may release its finalizer, but an unconfirmed save-on leaves the durable owner behind for recovery by the next operation. Time alone never releases -ownership. A pending CSI capture retains the current capture-drain behavior; a -separate CSI timeout change is tracked in issue #719. +ownership. CSI cancellation or expiry first invalidates the capture, then allows +save-on independently of driver reclamation. A late-ready aborted capture remains +unrestorable. If an operation waits unexpectedly, inspect the referenced Backup/Restore, its conditions, Jobs and worker pods. Resolve terminating workers, repository failures @@ -71,7 +72,9 @@ Already-running legacy quiesced copies continue being observed until their worke finish and cleanup succeeds. Legacy raw workers also block new admission. Conflicting legacy guards fail closed until their existing operations finish cleanup. Keep a single active operator version during upgrade; older binaries do not honor the -shared owner. No new CRD fields or Kubernetes permissions are required. +shared owner. CSI abort reclamation adds cluster-scoped delete permission for +VolumeSnapshotContents; runtime checks restrict it to recorded, UID-verified +contents with `Delete` policy. CSI restores create a new target and reserve it for the Restore. Initial startup is allowed under that reservation so storage using WaitForFirstConsumer can bind. @@ -85,6 +88,58 @@ writer has drained. Recovery of a force-deleted Restore applies the same check before a successor can proceed, preserving the existing pod template while requesting zero replicas. A successful Restore leaves its initial workload running. -Stopped-source preparation (#716), Restic preparation and quiesce deadlines (#717), -CSI capture deadlines (#719), and staged template adoption (#731) are separate -changes building on this ownership boundary. +Stopped-source proof, bounded copy preparation and CSI invalidation build on this +ownership boundary. Staged template adoption is tracked separately in #731. + +## Backup preparation and capture budgets + +Restic workers now schedule, pull their image, mount the source and validate the +repository before the agent is asked to disable saving. A Ready worker waits for +an expiring authorization bound to its pod UID, Backup UID and observed source. +The preferred node is the game's current node; `operator.backupHardSameNode: true` +makes that placement mandatory. Source tolerations and `operator.backupResources` +apply to preparation and copy containers. Stopped sources leave node selection +to the scheduler and volume affinity. + +When the source StatefulSet requests a running workload, initial preparation +waits for its owned pod to be created and assigned a node before binding the +worker to that source. This wait uses the preparation budget and does not disable +saving. A pod or node replacement after binding still fails the copy. Raw backups +of intentionally stopped workloads remain eligible without a source pod. + +The install settings are `operator.backupPreparationTimeout: 10m`, +`operator.backupQuiesceTimeout: 5m`, and `operator.backupCSICaptureTimeout: 10m`. +Equivalent operator flags use kebab-case names. The recorded deadlines remain +fixed across restarts or later configuration edits. Restic's existing Job +backoff/deadline settings remain separate; raw and already-stopped copies use +the full Job deadline. A live quiesce attempt retains the short budget even if +the agent reports the command unsupported. Increase the live quiesce budget +deliberately for worlds that cannot finish within five minutes, or use a +supported CSI capture strategy. + +Preparation failure leaves saving enabled. Quiesce expiry marks the copy failed, +revokes authorization and cancels its Job. Saving is restored only after all +live or terminating workers have stopped. The worker itself also stops Restic +at the token deadline (TERM, then KILL after five seconds), including when the +operator is unavailable. This bounds copying; save-on still requires the operator +and agent to recover. `CaptureReady` describes preparation/authorization or a +pending drain. Resolve stuck mounts, node failures or terminating workers before +retrying; removing owner annotations is not proof that copying stopped. + +Custom Restic images must provide the restic CLI (including `snapshots --retry-lock`), +POSIX `/bin/sh`, `date +%s`, `cat`, `touch`, `sleep`, and `timeout -s TERM -k 5`. +The pinned `restic/restic:0.17.1` release uses Alpine/BusyBox and supports this +contract. The gate runs under the existing non-root, read-only-root-filesystem +security context. Node/operator clocks must be synchronized for absolute token +expiry. No worker receives Kubernetes API credentials. + +CSI expiry and Backup deletion invalidate an incomplete capture before save-on. +The `snapshot-aborted` marker is terminal even if a delayed driver later reports +readiness. Normal restore and new raw GameServer PVC references reject such +captures. `SnapshotCleaned` reports reclamation progress and records snapshot +and content names/UIDs; `Retain` content and unverified replacements require +administrator action. A slow driver can keep Backup deletion pending for storage +cleanup after saving and game ownership have already been restored. Inspect the +condition/event identities before deleting retained storage manually. Existing +PVCs remain usable after source Backup retention; administrator-managed external +snapshots keep their direct-CRD behavior. diff --git a/hack/ci_scope.py b/hack/ci_scope.py index dfdb4709a..492efd102 100644 --- a/hack/ci_scope.py +++ b/hack/ci_scope.py @@ -29,8 +29,8 @@ FORCE_ALL_PREFIXES = (".github/actions/",) # Mirrors the e2e-go matrix buckets in .github/workflows/ci.yaml. E2E_BUCKETS = ["operator", "api-auth", "api-roles", "api-rbac", "api-agent", "api-mods", - "telemetry", "optional-components", "restricted-workloads", "management-placement"] -E2E_AMD64_ONLY = {"management-placement"} + "telemetry", "optional-components", "restricted-workloads", "management-placement", "backup-preparation"] +E2E_AMD64_ONLY = {"management-placement", "backup-preparation"} SUITES_FILE = "test/e2e/suite-components.json" # Paths no Go e2e test reads. Their own jobs keep their paths-filter gates. NO_E2E_FILES = { diff --git a/hack/test_ci_scope.py b/hack/test_ci_scope.py index 3436a0208..b5097a3ec 100644 --- a/hack/test_ci_scope.py +++ b/hack/test_ci_scope.py @@ -426,6 +426,14 @@ def test_audit_management_placement_has_a_run_pattern(self): self.assertIn("TestManagementPlacementAndDisruption", self.select(path)["e2e-run"].get("management-placement", "")) + def test_audit_backup_preparation_has_a_run_pattern(self): + """Worker preparation remains selected for each covered component.""" + for path in ("operator/internal/controller/backup_worker.go", "api/cmd/main.go", + "agent/cmd/main.go", "test/e2e/backup_preparation_e2e_test.go"): + with self.subTest(path=path): + self.assertIn("TestBackup_PreparationAndSameNodeCopy", + self.select(path)["e2e-run"].get("backup-preparation", "")) + def test_audit_staged_rollout_keeps_operator_coverage(self): """Template adoption and its owning fixture select the rollout scenario.""" for path in ("operator/internal/controller/gameserver_template_revision.go", diff --git a/i18n/locales/en/app.json b/i18n/locales/en/app.json index d192ef019..0c2525e10 100644 --- a/i18n/locales/en/app.json +++ b/i18n/locales/en/app.json @@ -155,6 +155,22 @@ "defaultMessage": "Requested {requestKind, select, addressAndPool {address ''{address}'' from pool ''{pool}''} address {address ''{address}''} other {pool ''{pool}''}}: waiting for the LoadBalancer Service that carries the request to exist.", "description": "AddressAssignment condition before the load-balancer Service exists." }, + "condition.captureready.snapshotcapturepending": { + "defaultMessage": "Waiting for CSI capture within its persisted deadline.", + "description": "Bounded CSI capture is pending." + }, + "condition.captureready.workerauthorized": { + "defaultMessage": "The prepared worker is authorized to capture data within its deadline.", + "description": "Backup copy authorization granted after preparation." + }, + "condition.captureready.workerdrainpending": { + "defaultMessage": "The copy is invalid. Waiting for workers to stop before restoring saving.", + "description": "Backup cancellation awaiting worker drainage." + }, + "condition.captureready.workerpreparing": { + "defaultMessage": "Waiting for worker scheduling, source mount and repository preparation.", + "description": "Backup preparation before saving is disabled." + }, "condition.completed.failed": { "defaultMessage": "The backup or restore failed.", "description": "Completed condition on a Backup or Restore that failed. The message holds the English detail." @@ -579,6 +595,22 @@ "defaultMessage": "The capture sidecar was told to stop.", "description": "SidecarStopped condition after a stop request." }, + "condition.snapshotcleaned.snapshotcleanupcomplete": { + "defaultMessage": "Aborted snapshot resources are absent.", + "description": "Snapshot reclamation completed." + }, + "condition.snapshotcleaned.snapshotcleanuppending": { + "defaultMessage": "Aborted snapshot reclamation is pending. {detail}", + "description": "Aborted snapshot deletion has not completed; detail includes resource identities." + }, + "condition.snapshotcleaned.snapshotcleanupretained": { + "defaultMessage": "Snapshot content requires administrator reclamation. {detail}", + "description": "Retained or replacement snapshot content is left for an administrator." + }, + "condition.snapshotcleaned.snapshotcleanupunverified": { + "defaultMessage": "A snapshot resource was replaced. It was not deleted. {detail}", + "description": "Snapshot identity verification refused deletion." + }, "condition.snapshotforgotten.forgetabandoned": { "defaultMessage": "The snapshot was not removed from the backup repository when the backup was deleted.", "description": "SnapshotForgotten condition when the forget step was abandoned." diff --git a/operator/cmd/main.go b/operator/cmd/main.go index 9fe5159ba..146b43040 100644 --- a/operator/cmd/main.go +++ b/operator/cmd/main.go @@ -156,6 +156,10 @@ func main() { captureDefaultMaxSizeBytes int64 backupJobBackoffLimit int64 backupJobActiveDeadlineSeconds int64 + backupPreparationTimeout time.Duration + backupQuiesceTimeout time.Duration + backupCSICaptureTimeout time.Duration + backupHardSameNode bool agentResourcesJSON string sentinelResourcesJSON string tunnelResourcesJSON string @@ -187,6 +191,10 @@ func main() { "BackoffLimit for the restic backup and restore Jobs (retries before the Job is marked Failed). Must be >= 0. Defaults to 2.") flag.Int64Var(&backupJobActiveDeadlineSeconds, "backup-job-active-deadline-seconds", controller.DefaultBackupJobActiveDeadlineSeconds, "ActiveDeadlineSeconds for the restic backup and restore Jobs. Must be > 0. Defaults to 86400 (24 hours).") + flag.DurationVar(&backupPreparationTimeout, "backup-preparation-timeout", controller.DefaultBackupPreparationTimeout, "Maximum worker preparation time before copy authorization; must be positive.") + flag.DurationVar(&backupQuiesceTimeout, "backup-quiesce-timeout", controller.DefaultBackupQuiesceTimeout, "Maximum live quiesced Restic copy duration; expiry cancels and drains workers before save-on.") + flag.DurationVar(&backupCSICaptureTimeout, "backup-csi-capture-timeout", controller.DefaultBackupCSICaptureTimeout, "Maximum CSI capture duration before invalidation and save-on; must be positive.") + flag.BoolVar(&backupHardSameNode, "backup-hard-same-node", false, "Require Restic workers on the observed game node instead of the default preferred placement.") flag.StringVar(&sentinelImage, "sentinel-image", controller.DefaultSentinelImage, "Image for the wake sentinel pod that holds advertised ports while a server is asleep. "+ "Point at a private registry mirror for air-gapped installs.") @@ -326,6 +334,10 @@ func main() { setupLog.Error(nil, "invalid --backup-job-active-deadline-seconds value (must be > 0)", "value", backupJobActiveDeadlineSeconds) os.Exit(1) } + if backupPreparationTimeout <= 0 || backupQuiesceTimeout <= 0 || backupCSICaptureTimeout <= 0 { + setupLog.Error(nil, "backup preparation, quiesce and CSI capture timeouts must be positive") + os.Exit(1) + } // Validate capture retention flags while they are still the int64 type // flag.Int64Var naturally produces. The minimum of 60 seconds aligns with @@ -491,6 +503,10 @@ func main() { JobBackoffLimit: &backupJobBackoff32, JobActiveDeadlineSeconds: backupJobActiveDeadlineSeconds, EventRecorder: mgr.GetEventRecorder("backup-controller"), + PreparationTimeout: backupPreparationTimeout, + QuiesceTimeout: backupQuiesceTimeout, + CSICaptureTimeout: backupCSICaptureTimeout, + HardSameNode: backupHardSameNode, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to set up controller", "controller", "Backup") os.Exit(1) diff --git a/operator/config/rbac/role.yaml b/operator/config/rbac/role.yaml index a676b73c6..ca8e31e35 100644 --- a/operator/config/rbac/role.yaml +++ b/operator/config/rbac/role.yaml @@ -151,6 +151,14 @@ rules: - snapshot.storage.k8s.io resources: - volumesnapshotcontents + verbs: + - delete + - get + - list + - watch +- apiGroups: + - snapshot.storage.k8s.io + resources: - volumesnapshots verbs: - get diff --git a/operator/hack/condition-ids/main.go b/operator/hack/condition-ids/main.go index 07b362029..b58c86606 100644 --- a/operator/hack/condition-ids/main.go +++ b/operator/hack/condition-ids/main.go @@ -52,6 +52,8 @@ var guardCalls = map[string]int{ "markPending": 2, "failWithReason": 2, "setSnapshotForgottenCondition": 3, + "setBackupProgress": 2, + "snapshotCleanupCondition": 3, } // guardReturnFuncs are the functions whose first string result is a reason. @@ -68,6 +70,14 @@ var guardSkipFiles = map[string]bool{ // rows is the English reference for every condition reason. var rows = []row{ + {types: []string{"CaptureReady"}, reason: "WorkerPreparing", english: "Waiting for worker scheduling, source mount and repository preparation.", description: "Backup preparation before saving is disabled."}, + {types: []string{"CaptureReady"}, reason: "WorkerAuthorized", english: "The prepared worker is authorized to capture data within its deadline.", description: "Backup copy authorization granted after preparation."}, + {types: []string{"CaptureReady"}, reason: "WorkerDrainPending", english: "The copy is invalid. Waiting for workers to stop before restoring saving.", description: "Backup cancellation awaiting worker drainage."}, + {types: []string{"CaptureReady"}, reason: "SnapshotCapturePending", english: "Waiting for CSI capture within its persisted deadline.", description: "Bounded CSI capture is pending."}, + {types: []string{"SnapshotCleaned"}, reason: "SnapshotCleanupPending", english: "Aborted snapshot reclamation is pending. {detail}", description: "Aborted snapshot deletion has not completed; detail includes resource identities."}, + {types: []string{"SnapshotCleaned"}, reason: "SnapshotCleanupUnverified", english: "A snapshot resource was replaced. It was not deleted. {detail}", description: "Snapshot identity verification refused deletion."}, + {types: []string{"SnapshotCleaned"}, reason: "SnapshotCleanupRetained", english: "Snapshot content requires administrator reclamation. {detail}", description: "Retained or replacement snapshot content is left for an administrator."}, + {types: []string{"SnapshotCleaned"}, reason: "SnapshotCleanupComplete", english: "Aborted snapshot resources are absent.", description: "Snapshot reclamation completed."}, {types: []string{"TemplateUpdatePending"}, reason: "AdoptingRevision", english: "The admitted template revision is being adopted.", description: "Template rollout owns the server lifecycle while draining and replacing its workload."}, {types: []string{"TemplateUpdatePending"}, reason: "TemplateUnavailable", english: "The installed template is unavailable; the effective snapshot is retained.", description: "Template availability is unknown but the server retains its immutable effective content."}, {types: []string{"TemplateUpdatePending"}, reason: "RevisionCurrent", english: "The installed template revision is effective.", description: "No newer template revision is waiting for adoption. Running revision is observed separately."}, diff --git a/operator/internal/controller/backup_budgets.go b/operator/internal/controller/backup_budgets.go new file mode 100644 index 000000000..ca0acf1d2 --- /dev/null +++ b/operator/internal/controller/backup_budgets.go @@ -0,0 +1,122 @@ +package controller + +import ( + "context" + "fmt" + "time" + + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" +) + +const ( + DefaultBackupPreparationTimeout = 10 * time.Minute + DefaultBackupQuiesceTimeout = 5 * time.Minute + DefaultBackupCSICaptureTimeout = 10 * time.Minute + backupPreparationDeadlineAnnotation = "backup.gameplane.local/preparation-deadline" + backupQuiesceDeadlineAnnotation = "backup.gameplane.local/quiesce-deadline" + backupCSIDeadlineAnnotation = "backup.gameplane.local/csi-deadline" + backupCopyAbortAnnotation = "backup.gameplane.local/copy-aborted" +) + +func backupTimeout(configured, fallback time.Duration) time.Duration { + if configured > 0 { + return configured + } + return fallback +} + +// Write once before the bounded action. Reconfiguration and restarts cannot +// extend an admitted attempt. Malformed stored state fails closed. +func (r *BackupReconciler) backupDeadline(ctx context.Context, b *gameplanev1alpha1.Backup, key string, duration time.Duration) (time.Time, error) { + if value := b.Annotations[key]; value != "" { + deadline, err := time.Parse(time.RFC3339Nano, value) + if err != nil { + return time.Time{}, fmt.Errorf("invalid persisted backup deadline %s: %w", key, err) + } + return deadline, nil + } + deadline := time.Now().UTC().Add(duration) + patchBackupAnnotations(b, map[string]string{key: deadline.Format(time.RFC3339Nano)}) + return deadline, r.Update(ctx, b) +} + +func (r *BackupReconciler) setBackupProgress(ctx context.Context, b *gameplanev1alpha1.Backup, reason, message string) error { + status := metav1.ConditionFalse + if reason == "WorkerAuthorized" { + status = metav1.ConditionTrue + } + conditions := upsertCondition(b.Status.Conditions, metav1.Condition{Type: "CaptureReady", Status: status, + Reason: reason, Message: message, ObservedGeneration: b.Generation}) + if sameConditions(b.Status.Conditions, conditions) { + return nil + } + b.Status.Conditions = conditions + return r.Status().Update(ctx, b) +} + +// Persist invalidation before revoking authorization or cancelling workers. +// A stale mounted token can still be observed until projection catches up, so +// neither token revocation nor a deadline substitutes for actual worker drain. +func (r *BackupReconciler) abortResticCopy(ctx context.Context, b *gameplanev1alpha1.Backup, message string) (ctrl.Result, error) { + if message == "" { + message = "backup copy invalidated" + } + if b.Annotations[backupCopyAbortAnnotation] == "" { + patchBackupAnnotations(b, map[string]string{backupCopyAbortAnnotation: message}) + if err := r.Update(ctx, b); err != nil { + return ctrl.Result{}, err + } + } + message = b.Annotations[backupCopyAbortAnnotation] + if b.Status.Phase != gameplanev1alpha1.BackupPhaseFailed || b.Status.SnapshotID != "" || b.Status.Message != message { + b.Status.SnapshotID = "" + if _, err := r.fail(ctx, b, message); err != nil { + return ctrl.Result{}, err + } + } + return r.cleanupResticCopy(ctx, b) +} + +func (r *BackupReconciler) cleanupResticCopy(ctx context.Context, b *gameplanev1alpha1.Backup) (ctrl.Result, error) { + if err := r.revokeBackupStart(ctx, b); err != nil { + return ctrl.Result{}, err + } + busy, err := r.backupCleanupWorkersLive(ctx, b, true) + if err != nil { + return ctrl.Result{}, err + } + if busy { + if err := r.setBackupProgress(ctx, b, "WorkerDrainPending", "copy invalidated; waiting for workers to stop before restoring saving"); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: 5 * time.Second}, nil + } + if meta.RemoveStatusCondition(&b.Status.Conditions, "CaptureReady") { + if err := r.Status().Update(ctx, b); err != nil { + return ctrl.Result{}, err + } + } + return r.runUnquiesce(ctx, b) +} + +func (r *BackupReconciler) revokeBackupStart(ctx context.Context, b *gameplanev1alpha1.Backup) error { + var gate corev1.ConfigMap + err := r.backupAPIReader().Get(ctx, client.ObjectKey{Namespace: b.Namespace, Name: backupStartConfigMapName(b)}, &gate) + if err != nil && !apierrors.IsNotFound(err) { + return err + } + if err == nil && metav1.IsControlledBy(&gate, b) && gate.Data[backupStartTokenKey] != "" { + gate.Data[backupStartTokenKey] = "" + if err := r.Update(ctx, &gate); err != nil { + return err + } + } + return nil +} diff --git a/operator/internal/controller/backup_budgets_test.go b/operator/internal/controller/backup_budgets_test.go new file mode 100644 index 000000000..b483e7538 --- /dev/null +++ b/operator/internal/controller/backup_budgets_test.go @@ -0,0 +1,684 @@ +package controller + +import ( + "context" + "fmt" + "os" + "os/exec" + "path/filepath" + "runtime" + "strings" + "testing" + "time" + + snapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + + gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" +) + +func budgetPass(t *testing.T, r *BackupReconciler, name string) { + t.Helper() + for range 4 { + if _, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "ns", Name: name}}); err != nil { + t.Fatal(err) + } + } +} + +// Model the Job controller and kubelet explicitly: no worker may be authorized +// merely because the Job exists or an informer has an old Ready observation. +func budgetReadyWorker(t *testing.T, r *BackupReconciler, name, uid string) *corev1.Pod { + t.Helper() + ctx := context.Background() + var job batchv1.Job + if err := r.Get(ctx, client.ObjectKey{Namespace: "ns", Name: name}, &job); err != nil { + t.Fatal(err) + } + if job.UID == "" { + job.UID = types.UID(name + "-job-uid") + if err := r.Update(ctx, &job); err != nil { + t.Fatal(err) + } + } + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: name + "-worker", Namespace: "ns", UID: types.UID(uid), Labels: job.Spec.Template.Labels, + OwnerReferences: []metav1.OwnerReference{{APIVersion: "batch/v1", Kind: "Job", Name: job.Name, UID: job.UID, Controller: ownerBoolPtr(true)}}}, + Spec: job.Spec.Template.Spec, Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionTrue}}}} + pod.Spec.NodeName = "node-a" + if err := r.Create(ctx, pod); err != nil { + t.Fatal(err) + } + return pod +} + +func TestBackupWorkerPreparesBeforeQuiesceAndPinsAuthorization(t *testing.T) { + r, q := guardFixtureWorkers(t, false, guardBackup("prepared")) + budgetPass(t, r, "prepared") + if q.quiesced != 0 { + t.Fatal("save-off preceded worker readiness") + } + b := guardGetBackup(t, r, "prepared") + if b.Status.CaptureManifest != nil { + t.Fatal("capture metadata recorded before worker preparation") + } + var gate corev1.ConfigMap + if err := r.Get(context.Background(), client.ObjectKey{Namespace: "ns", Name: backupStartConfigMapName(b)}, &gate); err != nil { + t.Fatal(err) + } + if gate.Data[backupStartTokenKey] != "" { + t.Fatal("unprepared worker received start authorization") + } + worker := budgetReadyWorker(t, r, b.Name, "prepared-worker-uid") + budgetPass(t, r, b.Name) + b = guardGetBackup(t, r, b.Name) + if q.quiesced != 1 || b.Status.CaptureManifest == nil || b.Status.CaptureManifest.State != "Recorded" || b.Status.CaptureManifest.QuiesceOutcome != "confirmed" { + t.Fatalf("prepared copy not quiesced/recorded: %+v, %+v", q, b.Status) + } + if volumes := b.Status.CaptureManifest.Volumes; len(volumes) != 1 || volumes[0].ClaimName != "server-data" || !volumes[0].Included || volumes[0].Classification != "data" { + t.Fatalf("prepared worker lost capture-time coverage: %+v", volumes) + } + if err := r.Get(context.Background(), client.ObjectKeyFromObject(&gate), &gate); err != nil { + t.Fatal(err) + } + if !strings.Contains(gate.Data[backupStartTokenKey], string(worker.UID)) || !strings.Contains(gate.Data[backupStartTokenKey], string(b.UID)) { + t.Fatalf("unbound authorization: %q", gate.Data[backupStartTokenKey]) + } + deadline := b.Annotations[backupQuiesceDeadlineAnnotation] + r = &BackupReconciler{Client: r.Client, Scheme: r.Scheme, AgentClient: q, QuiesceTimeout: time.Hour} + budgetPass(t, r, b.Name) + if got := guardGetBackup(t, r, b.Name).Annotations[backupQuiesceDeadlineAnnotation]; got != deadline { + t.Fatal("restart extended admitted quiesce deadline") + } +} + +func TestBackupPreparedWorkerUsesTemplateAdmittedBeforeOwnership(t *testing.T) { + for _, adoptBeforeClaim := range []bool{false, true} { + t.Run(fmt.Sprintf("adopt-before-claim=%t", adoptBeforeClaim), func(t *testing.T) { + r, q := guardFixtureWorkers(t, false, guardBackup("prepared")) + base := r.Client.(client.WithWatch) + r.APIReader = base + var gs gameplanev1alpha1.GameServer + if err := base.Get(t.Context(), client.ObjectKey{Namespace: "ns", Name: "server"}, &gs); err != nil { + t.Fatal(err) + } + oldGroup, nextGroup := int64(1000), int64(2000) + installed := &gameplanev1alpha1.GameTemplate{ObjectMeta: metav1.ObjectMeta{Name: "template", UID: "old-catalog-uid"}, + Spec: gameplanev1alpha1.GameTemplateSpec{Security: &gameplanev1alpha1.GameSecuritySpec{FSGroup: &oldGroup}}} + if err := base.Create(t.Context(), installed); err != nil { + t.Fatal(err) + } + revisions := &GameServerReconciler{Client: base, APIReader: base, Scheme: r.Scheme} + oldPin, err := revisions.saveTemplateSnapshot(t.Context(), &gs, installed) + if err != nil { + t.Fatal(err) + } + gs.Annotations = map[string]string{effectiveTemplateAnnotation: oldPin} + if err := base.Update(t.Context(), &gs); err != nil { + t.Fatal(err) + } + // Reinstalling the catalog can change both its UID and filesystem + // settings without admitting that content for this GameServer. + if err := base.Delete(t.Context(), installed); err != nil { + t.Fatal(err) + } + installed = &gameplanev1alpha1.GameTemplate{ObjectMeta: metav1.ObjectMeta{Name: "template", UID: "new-catalog-uid"}, + Spec: gameplanev1alpha1.GameTemplateSpec{Security: &gameplanev1alpha1.GameSecuritySpec{FSGroup: &nextGroup}}} + if err := base.Create(t.Context(), installed); err != nil { + t.Fatal(err) + } + nextPin, err := revisions.saveTemplateSnapshot(t.Context(), &gs, installed) + if err != nil { + t.Fatal(err) + } + adopted := false + r.Client = interceptor.NewClient(base, interceptor.Funcs{Update: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.UpdateOption) error { + if b, ok := obj.(*gameplanev1alpha1.Backup); ok && adoptBeforeClaim && !adopted && b.Annotations[backupTargetAnnotation] != "" { + // Finish a rollout after Reconcile reads the GameServer but + // before the Backup claims its shared lifecycle ownership. + var live gameplanev1alpha1.GameServer + if err := c.Get(ctx, client.ObjectKeyFromObject(&gs), &live); err != nil { + return err + } + live.Annotations[effectiveTemplateAnnotation] = nextPin + if err := c.Update(ctx, &live); err != nil { + return err + } + adopted = true + } + return c.Update(ctx, obj, opts...) + }}) + budgetPass(t, r, "prepared") + var job batchv1.Job + if err := base.Get(t.Context(), client.ObjectKey{Namespace: "ns", Name: "prepared"}, &job); err != nil { + t.Fatal(err) + } + want := oldGroup + if adoptBeforeClaim { + want = nextGroup + } + security := job.Spec.Template.Spec.SecurityContext + if adopted != adoptBeforeClaim || security == nil || security.FSGroup == nil || *security.FSGroup != want { + t.Fatalf("prepared worker did not use the admitted source: adopted=%t security=%+v want fsGroup=%d", adopted, security, want) + } + if q.quiesced != 0 { + t.Fatal("template resolution bypassed worker readiness") + } + budgetReadyWorker(t, r, "prepared", "ready-worker-uid") + budgetPass(t, r, "prepared") + b := guardGetBackup(t, r, "prepared") + if b.Annotations[backupAuthorizedWorkerAnnotation] == "" || b.Status.CaptureManifest == nil || b.Status.CaptureManifest.State != "Recorded" || q.quiesced != 1 { + t.Fatalf("pinned preparation did not reach authorized capture: %+v %+v", b, q) + } + }) + } +} + +func TestBackupPreparedWorkerRejectsMissingEffectiveTemplate(t *testing.T) { + r, q := guardFixtureWorkers(t, false, guardBackup("prepared")) + var gs gameplanev1alpha1.GameServer + if err := r.Get(t.Context(), client.ObjectKey{Namespace: "ns", Name: "server"}, &gs); err != nil { + t.Fatal(err) + } + gs.Annotations = map[string]string{effectiveTemplateAnnotation: strings.Repeat("a", 64)} + if err := r.Update(t.Context(), &gs); err != nil { + t.Fatal(err) + } + // A readable catalog must not substitute for the missing admitted content. + if err := r.Create(t.Context(), &gameplanev1alpha1.GameTemplate{ObjectMeta: metav1.ObjectMeta{Name: "template"}}); err != nil { + t.Fatal(err) + } + _, err := r.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKey{Namespace: "ns", Name: "prepared"}}) + if err == nil || !strings.Contains(err.Error(), "read pinned template snapshot") || q.quiesced != 0 { + t.Fatalf("missing pin did not fail before save-off: %v %+v", err, q) + } + var job batchv1.Job + if err := r.Get(t.Context(), client.ObjectKey{Namespace: "ns", Name: "prepared"}, &job); !apierrors.IsNotFound(err) { + t.Fatalf("missing pin created a worker: %v", err) + } +} + +func TestBackupPreparationDeadlineNeverDisablesSaving(t *testing.T) { + r, q := guardFixtureWorkers(t, false, guardBackup("expired")) + budgetPass(t, r, "expired") + b := guardGetBackup(t, r, "expired") + patchBackupAnnotations(b, map[string]string{backupPreparationDeadlineAnnotation: time.Now().Add(-time.Minute).UTC().Format(time.RFC3339Nano)}) + if err := r.Update(context.Background(), b); err != nil { + t.Fatal(err) + } + budgetPass(t, r, b.Name) + b = guardGetBackup(t, r, b.Name) + if b.Status.Phase != gameplanev1alpha1.BackupPhaseFailed || q.quiesced != 0 { + t.Fatalf("expired preparation disabled saving: %+v %+v", b.Status, q) + } +} + +func TestBackupRawAndStoppedCopiesKeepFullJobBudget(t *testing.T) { + for _, mode := range []string{"raw", "no-agent", "stopped"} { + t.Run(mode, func(t *testing.T) { + b := guardBackup("first") + b.Spec.Quiesce = mode != "raw" + r, q := guardFixture(t, b) + if mode == "no-agent" { + r.AgentClient = nil + } + if mode == "stopped" { + r, q = stoppedBackupFixture(t, true) + } + r.QuiesceTimeout, r.JobActiveDeadlineSeconds = time.Second, 7200 + budgetPass(t, r, "first") + b = guardGetBackup(t, r, "first") + deadline, err := time.Parse(time.RFC3339Nano, b.Annotations[backupQuiesceDeadlineAnnotation]) + if err != nil || time.Until(deadline) < time.Hour || b.Annotations[backupAuthorizedWorkerAnnotation] == "" || q.quiesced != 0 { + t.Fatalf("non-quiesced copy lost its full budget/authorization: %v %+v %+v", err, b.Annotations, q) + } + }) + } +} + +func TestBackupRawSourceWaitsForInitialScheduling(t *testing.T) { + for _, initiallyAbsent := range []bool{false, true} { + t.Run(fmt.Sprintf("absent=%t", initiallyAbsent), func(t *testing.T) { + b := guardBackup("starting") + b.Spec.Quiesce = false + r, q := guardFixtureWorkers(t, false, b) + var pod corev1.Pod + if err := r.Get(t.Context(), client.ObjectKey{Namespace: "ns", Name: "server-0"}, &pod); err != nil { + t.Fatal(err) + } + pod.Spec.NodeName = "" + if err := r.Update(t.Context(), &pod); err != nil { + t.Fatal(err) + } + if initiallyAbsent { + if err := r.Delete(t.Context(), &pod); err != nil { + t.Fatal(err) + } + } + budgetPass(t, r, b.Name) + b = guardGetBackup(t, r, b.Name) + var job batchv1.Job + if err := r.Get(t.Context(), client.ObjectKeyFromObject(b), &job); !apierrors.IsNotFound(err) || b.Annotations[backupWorkerSourceAnnotation] != "" || q.quiesced != 0 { + t.Fatalf("starting source was bound or quiesced: job=%v annotations=%v calls=%+v", err, b.Annotations, q) + } + pod.Spec.NodeName = "node-a" + if initiallyAbsent { + pod.ResourceVersion = "" + if err := r.Create(t.Context(), &pod); err != nil { + t.Fatal(err) + } + } else if err := r.Update(t.Context(), &pod); err != nil { + t.Fatal(err) + } + budgetPass(t, r, b.Name) + b = guardGetBackup(t, r, b.Name) + if err := r.Get(t.Context(), client.ObjectKeyFromObject(b), &job); err != nil || b.Status.Phase == gameplanev1alpha1.BackupPhaseFailed { + t.Fatalf("normal source scheduling failed backup: job=%v status=%+v", err, b.Status) + } + // Once captured in the worker's immutable environment, the source + // may not silently change to a different pod at the same name. + if err := r.Delete(t.Context(), &pod); err != nil { + t.Fatal(err) + } + pod.UID, pod.ResourceVersion = "replacement-pod", "" + if err := r.Create(t.Context(), &pod); err != nil { + t.Fatal(err) + } + budgetPass(t, r, b.Name) + b = guardGetBackup(t, r, b.Name) + if b.Status.Phase != gameplanev1alpha1.BackupPhaseFailed || b.Annotations[backupAuthorizedWorkerAnnotation] != "" { + t.Fatalf("replacement source escaped immutable binding: %+v", b) + } + }) + } +} + +func TestBackupPreparedWorkerFailurePreservesJobMessage(t *testing.T) { + r, _ := guardFixtureWorkers(t, false, guardBackup("failed-worker")) + budgetPass(t, r, "failed-worker") + var job batchv1.Job + if err := r.Get(t.Context(), client.ObjectKey{Namespace: "ns", Name: "failed-worker"}, &job); err != nil { + t.Fatal(err) + } + job.Status.Conditions = []batchv1.JobCondition{{Type: batchv1.JobFailed, Status: corev1.ConditionTrue}} + if err := r.Status().Update(t.Context(), &job); err != nil { + t.Fatal(err) + } + budgetPass(t, r, job.Name) + b := guardGetBackup(t, r, job.Name) + if b.Status.Phase != gameplanev1alpha1.BackupPhaseFailed || b.Status.Message != "backup job reported Failed" || b.Annotations[backupCopyAbortAnnotation] != b.Status.Message { + t.Fatalf("prepared worker lost failure compatibility/invalidation: %+v", b) + } +} + +func TestBackupQueuedCopyDoesNotSpendPreparationBudget(t *testing.T) { + r, _ := guardFixture(t, guardBackup("first"), guardBackup("second")) + budgetPass(t, r, "first") + budgetPass(t, r, "second") + second := guardGetBackup(t, r, "second") + if second.Annotations[backupPreparationDeadlineAnnotation] != "" || second.Annotations[backupAuthorizedWorkerAnnotation] != "" { + t.Fatal("queued Backup spent preparation budget before owning the source") + } +} + +func TestBackupQuiesceDeadlineDrainsBeforeSaveOn(t *testing.T) { + r, q := guardFixtureWorkers(t, false, guardBackup("expired")) + budgetPass(t, r, "expired") + worker := budgetReadyWorker(t, r, "expired", "worker-uid") + budgetPass(t, r, "expired") + b := guardGetBackup(t, r, "expired") + patchBackupAnnotations(b, map[string]string{backupQuiesceDeadlineAnnotation: time.Now().Add(-time.Minute).UTC().Format(time.RFC3339Nano)}) + if err := r.Update(context.Background(), b); err != nil { + t.Fatal(err) + } + budgetPass(t, r, b.Name) + b = guardGetBackup(t, r, b.Name) + if b.Status.Phase != gameplanev1alpha1.BackupPhaseFailed || q.unquiesced != 0 { + t.Fatalf("deadline bypassed worker drain: %+v %+v", b.Status, q) + } + if err := r.Delete(context.Background(), worker); err != nil { + t.Fatal(err) + } + budgetPass(t, r, b.Name) + if q.unquiesced != 1 { + t.Fatalf("save-on not retried after drain: %+v", q) + } +} + +func TestBackupDeletionRevokesAuthorizationBeforeTerminatingWorkerDrains(t *testing.T) { + r, q := guardFixtureWorkers(t, false, guardBackup("cancel")) + budgetPass(t, r, "cancel") + worker := budgetReadyWorker(t, r, "cancel", "worker-uid") + budgetPass(t, r, "cancel") + worker.Finalizers = []string{"test/hold"} + if err := r.Update(t.Context(), worker); err != nil { + t.Fatal(err) + } + if err := r.Delete(t.Context(), worker); err != nil { + t.Fatal(err) + } + b := guardGetBackup(t, r, "cancel") + if err := r.Delete(t.Context(), b); err != nil { + t.Fatal(err) + } + budgetPass(t, r, "cancel") + var gate corev1.ConfigMap + if err := r.Get(t.Context(), client.ObjectKey{Namespace: b.Namespace, Name: backupStartConfigMapName(b)}, &gate); err != nil { + t.Fatal(err) + } + if gate.Data[backupStartTokenKey] != "" || q.unquiesced != 0 { + t.Fatal("deletion retained authorization or bypassed terminating worker") + } + if err := r.Get(t.Context(), client.ObjectKeyFromObject(worker), worker); err != nil { + t.Fatal(err) + } + worker.Finalizers = nil + if err := r.Update(t.Context(), worker); err != nil { + t.Fatal(err) + } + budgetPass(t, r, "cancel") + if q.unquiesced != 1 { + t.Fatal("deletion did not restore saving after drain") + } +} + +func TestBackupRecoveryRevokesTokenBeforeCancellingWorkers(t *testing.T) { + r, _ := guardFixtureWorkers(t, false, guardBackup("orphan")) + budgetPass(t, r, "orphan") + budgetReadyWorker(t, r, "orphan", "orphan-worker") + budgetPass(t, r, "orphan") + b := guardGetBackup(t, r, "orphan") + deletionObserved := false + r.Client = interceptor.NewClient(r.Client.(client.WithWatch), interceptor.Funcs{Delete: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.DeleteOption) error { + if _, ok := obj.(*batchv1.Job); ok { + var gate corev1.ConfigMap + if err := c.Get(ctx, client.ObjectKey{Namespace: b.Namespace, Name: backupStartConfigMapName(b)}, &gate); err != nil { + t.Fatal(err) + } + if gate.Data[backupStartTokenKey] != "" { + t.Fatal("recovery cancelled Job before revoking authorization") + } + deletionObserved = true + } + return c.Delete(ctx, obj, opts...) + }}) + busy, err := r.backupWorkersLive(t.Context(), b.Namespace, backupIdentity{Name: b.Name, UID: b.UID}, true) + if err != nil || !busy || !deletionObserved { + t.Fatalf("recovery bypassed worker cancellation/drain: %t %t %v", busy, deletionObserved, err) + } +} + +func TestBackupCSIDeadlineInvalidatesLateReady(t *testing.T) { + for _, status := range []*snapshotv1.VolumeSnapshotStatus{nil, {ReadyToUse: ownerBoolPtr(false)}} { + b := guardBackup("snapshot") + b.Spec.Strategy = "volume-snapshot" + r, q := guardFixture(t, b) + budgetPass(t, r, b.Name) + b = guardGetBackup(t, r, b.Name) + patchBackupAnnotations(b, map[string]string{backupCSIDeadlineAnnotation: time.Now().Add(-time.Minute).UTC().Format(time.RFC3339Nano)}) + if err := r.Update(context.Background(), b); err != nil { + t.Fatal(err) + } + var vs snapshotv1.VolumeSnapshot + if err := r.Get(context.Background(), client.ObjectKeyFromObject(b), &vs); err != nil { + t.Fatal(err) + } + vs.Status = status + if err := r.Status().Update(context.Background(), &vs); err != nil { + t.Fatal(err) + } + budgetPass(t, r, b.Name) + b = guardGetBackup(t, r, b.Name) + if b.Annotations[backupSnapshotAbortAnnotation] != "true" || b.Status.Phase != gameplanev1alpha1.BackupPhaseFailed || b.Status.SnapshotID != "" || q.unquiesced != 1 { + t.Fatalf("CSI abort did not invalidate before cleanup: %+v %+v", b, q) + } + vs.Status = &snapshotv1.VolumeSnapshotStatus{ReadyToUse: ownerBoolPtr(true)} + if _, err := r.observeVolumeSnapshot(context.Background(), b, &vs); err != nil { + t.Fatal(err) + } + if got := guardGetBackup(t, r, b.Name); got.Status.Phase != gameplanev1alpha1.BackupPhaseFailed || got.Status.SnapshotID != "" { + t.Fatal("late readiness revived aborted capture") + } + } +} + +func TestBackupWorkerDoesNotTrustCachedReadinessOrReplacement(t *testing.T) { + for _, replacement := range []bool{false, true} { + t.Run(fmt.Sprintf("replacement-%t", replacement), func(t *testing.T) { + r, q := guardFixtureWorkers(t, false, guardBackup("copy")) + budgetPass(t, r, "copy") + worker := budgetReadyWorker(t, r, "copy", "first-worker") + if replacement { + q.afterQuiesce = func() { + if err := r.Delete(t.Context(), worker); err != nil { + t.Fatal(err) + } + budgetReadyWorker(t, r, "copy", "replacement-worker") + } + } else { + r.APIReader = interceptor.NewClient(r.Client.(client.WithWatch), interceptor.Funcs{List: func(ctx context.Context, cl client.WithWatch, list client.ObjectList, opts ...client.ListOption) error { + if err := cl.List(ctx, list, opts...); err != nil { + return err + } + if pods, ok := list.(*corev1.PodList); ok { + for i := range pods.Items { + if pods.Items[i].UID == worker.UID { + pods.Items[i].Status.Conditions = nil + } + } + } + return nil + }}) + } + if _, err := r.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKey{Namespace: "ns", Name: "copy"}}); err != nil { + t.Fatal(err) + } + b := guardGetBackup(t, r, "copy") + var gate corev1.ConfigMap + if err := r.Get(t.Context(), client.ObjectKey{Namespace: "ns", Name: backupStartConfigMapName(b)}, &gate); err != nil { + t.Fatal(err) + } + if gate.Data[backupStartTokenKey] != "" { + t.Fatal("stale readiness granted copy authorization") + } + if !replacement && q.quiesced != 0 { + t.Fatal("cached Ready disabled saving") + } + if replacement && q.quiesced != 1 { + t.Fatal("fixture did not replace worker during RPC") + } + }) + } +} + +func TestPreparedBackupPlacementAndResources(t *testing.T) { + for _, hard := range []bool{false, true} { + r := &BackupReconciler{HardSameNode: hard, BackupResources: &corev1.ResourceRequirements{}} + b := guardBackup("copy") + source := &backupWorkerSource{NodeName: "node-a", Tolerations: []corev1.Toleration{{Key: "games", Operator: corev1.TolerationOpExists, Effect: corev1.TaintEffectNoSchedule}}} + spec := r.buildPreparedBackupPodSpec(b, &gameplanev1alpha1.GameTemplate{}, source, time.Now().Add(time.Minute)) + if spec.AutomountServiceAccountToken == nil || *spec.AutomountServiceAccountToken || len(spec.Tolerations) != 1 || spec.Tolerations[0].Key != "games" { + t.Fatal("worker credentials or tolerations contract lost") + } + if spec.Affinity == nil || len(spec.Affinity.NodeAffinity.PreferredDuringSchedulingIgnoredDuringExecution) != 1 { + t.Fatal("same-node preference missing") + } + if (spec.Affinity.NodeAffinity.RequiredDuringSchedulingIgnoredDuringExecution != nil) != hard { + t.Fatal("hard placement opt-in not respected") + } + if len(spec.Containers[0].Resources.Requests) != 0 || spec.Containers[0].ReadinessProbe == nil { + t.Fatal("resource clearing or preparation probe missing") + } + source.NodeName = "" + if stopped := r.buildPreparedBackupPodSpec(b, &gameplanev1alpha1.GameTemplate{}, source, time.Now()); stopped.Affinity != nil { + t.Fatal("stopped source invented a game node") + } + } +} + +// Execute the real gate script in CI with only its mount paths redirected to +// temp directories. A fake restic records whether source capture was invoked. +func TestBackupWorkerScriptRejectsExpiredAndReplayedTokens(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("worker image uses a POSIX shell; executed by Linux CI") + } + for _, tokenKind := range []string{"valid", "expired", "other-worker", "other-backup", "other-source", "repo-failure"} { + t.Run(tokenKind, func(t *testing.T) { + dir := t.TempDir() + restic := "#!/bin/sh\nif [ \"$1\" = snapshots ]; then [ \"$REPO_FAIL\" != true ]; exit $?; fi\nprintf copied > \"$COPY_MARKER\"\n" + if err := os.WriteFile(filepath.Join(dir, "restic"), []byte(restic), 0o700); err != nil { + t.Fatal(err) + } + backup, pod, source, expiry := "backup", "worker", "source", time.Now().Add(time.Minute).Unix() + switch tokenKind { + case "expired": + expiry = time.Now().Add(-time.Minute).Unix() + case "other-worker": + pod = "old-worker" + case "other-backup": + backup = "old-backup" + case "other-source": + source = "old-source" + } + tokenFile, prepared, copied := filepath.Join(dir, "token"), filepath.Join(dir, "prepared"), filepath.Join(dir, "copied") + if err := os.WriteFile(tokenFile, fmt.Appendf(nil, "v1 %s %s %s %d\n", backup, pod, source, expiry), 0o600); err != nil { + t.Fatal(err) + } + script := strings.NewReplacer("/data", dir, "/start/start", tokenFile, "/tmp/prepared", prepared).Replace(resticWorkerScript) + ctx, cancel := context.WithTimeout(t.Context(), 4*time.Second) + defer cancel() + cmd := exec.CommandContext(ctx, "/bin/sh", "-c", script, "backup-worker", "backup", dir) + cmd.Env = append(os.Environ(), "PATH="+dir+":"+os.Getenv("PATH"), "BACKUP_UID=backup", "POD_UID=worker", "SOURCE_ID=source", fmt.Sprintf("PREPARATION_EXPIRES=%d", time.Now().Add(time.Second).Unix()), "COPY_MARKER="+copied, fmt.Sprintf("REPO_FAIL=%t", tokenKind == "repo-failure")) + output, err := cmd.CombinedOutput() + if ctx.Err() != nil { + t.Fatalf("gate did not terminate: %s", output) + } + _, copiedErr := os.Stat(copied) + if tokenKind == "valid" { + if err != nil || copiedErr != nil { + t.Fatalf("valid token did not start copy: %v %s", err, output) + } + } else if err == nil || copiedErr == nil { + t.Fatalf("invalid token/preparation started capture: %v %s", err, output) + } + if tokenKind == "repo-failure" { + if _, err := os.Stat(prepared); err == nil { + t.Fatal("repository failure became Ready") + } + } + }) + } +} + +func TestBackupCSIAbortReclaimsOnlyVerifiedDeleteContent(t *testing.T) { + for _, mode := range []string{"delete", "retain", "foreign-reference", "replacement-snapshot", "delete-error"} { + t.Run(mode, func(t *testing.T) { + b := guardBackup("snapshot") + b.Spec.Strategy = "volume-snapshot" + r, q := guardFixture(t, b) + budgetPass(t, r, b.Name) + b = guardGetBackup(t, r, b.Name) + var vs snapshotv1.VolumeSnapshot + if err := r.Get(t.Context(), client.ObjectKeyFromObject(b), &vs); err != nil { + t.Fatal(err) + } + contentName := "content" + content := &snapshotv1.VolumeSnapshotContent{ObjectMeta: metav1.ObjectMeta{Name: contentName, UID: "content-uid"}, Spec: snapshotv1.VolumeSnapshotContentSpec{DeletionPolicy: snapshotv1.VolumeSnapshotContentDelete, + VolumeSnapshotRef: corev1.ObjectReference{Name: vs.Name, Namespace: vs.Namespace, UID: vs.UID}}} + if mode == "retain" { + content.Spec.DeletionPolicy = snapshotv1.VolumeSnapshotContentRetain + } + if mode == "foreign-reference" { + content.Spec.VolumeSnapshotRef.UID = "foreign-snapshot" + } + if err := r.Create(t.Context(), content); err != nil { + t.Fatal(err) + } + vs.Status = &snapshotv1.VolumeSnapshotStatus{BoundVolumeSnapshotContentName: &contentName} + if err := r.Status().Update(t.Context(), &vs); err != nil { + t.Fatal(err) + } + if mode == "delete" { + r.Client = interceptor.NewClient(r.Client.(client.WithWatch), interceptor.Funcs{Delete: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.DeleteOption) error { + options := &client.DeleteOptions{} + for _, option := range opts { + option.ApplyToDelete(options) + } + if options.Preconditions == nil || options.Preconditions.UID == nil || *options.Preconditions.UID != obj.GetUID() { + t.Fatalf("cleanup deletion lacks exact UID precondition: %+v", options) + } + return c.Delete(ctx, obj, opts...) + }}) + } + if mode == "replacement-snapshot" { + if err := r.Delete(t.Context(), &vs); err != nil { + t.Fatal(err) + } + vs.UID, vs.ResourceVersion = "replacement-uid", "" + if err := r.Create(t.Context(), &vs); err != nil { + t.Fatal(err) + } + } + if mode == "delete-error" { + r.Client = interceptor.NewClient(r.Client.(client.WithWatch), interceptor.Funcs{Delete: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.DeleteOption) error { + if _, ok := obj.(*snapshotv1.VolumeSnapshot); ok { + return fmt.Errorf("driver deletion unavailable") + } + return c.Delete(ctx, obj, opts...) + }}) + } + patchBackupAnnotations(b, map[string]string{backupCSIDeadlineAnnotation: time.Now().Add(-time.Minute).UTC().Format(time.RFC3339Nano)}) + if err := r.Update(t.Context(), b); err != nil { + t.Fatal(err) + } + // First request deletion, then separately observe its completion. + // A single pass must not claim backend reclamation was observed. + if _, err := r.Reconcile(t.Context(), ctrl.Request{NamespacedName: client.ObjectKeyFromObject(b)}); err != nil { + t.Fatal(err) + } + b = guardGetBackup(t, r, b.Name) + if q.unquiesced != 1 || b.Status.Phase != gameplanev1alpha1.BackupPhaseFailed || b.Status.SnapshotID != "" { + t.Fatalf("reclamation blocked invalidation/save-on: %+v %+v", b.Status, q) + } + condition := meta.FindStatusCondition(b.Status.Conditions, "SnapshotCleaned") + if condition == nil { + t.Fatal("missing cleanup condition") + } + err := r.Get(t.Context(), client.ObjectKeyFromObject(content), content) + switch mode { + case "delete": + if !apierrors.IsNotFound(err) || condition.Status != metav1.ConditionFalse || condition.Reason != "SnapshotCleanupPending" { + t.Fatalf("deletion request was not recorded before observing absence: %v %+v", err, condition) + } + if err := r.Get(t.Context(), client.ObjectKeyFromObject(&vs), &vs); !apierrors.IsNotFound(err) { + t.Fatalf("snapshot remains after deletion: %v", err) + } + budgetPass(t, r, b.Name) + b = guardGetBackup(t, r, b.Name) + condition = meta.FindStatusCondition(b.Status.Conditions, "SnapshotCleaned") + if condition == nil || condition.Status != metav1.ConditionTrue || condition.Reason != "SnapshotCleanupComplete" { + t.Fatalf("observed resource absence did not complete reclamation: %+v", condition) + } + case "retain": + if err != nil || condition.Reason != "SnapshotCleanupRetained" || !strings.Contains(condition.Message, "content-uid") { + t.Fatalf("Retain identity was lost: %v %+v", err, condition) + } + case "foreign-reference", "replacement-snapshot": + if err != nil || condition.Reason != "SnapshotCleanupUnverified" { + t.Fatalf("foreign resource was touched: %v %+v", err, condition) + } + case "delete-error": + if condition.Reason != "SnapshotCleanupPending" || !strings.Contains(condition.Message, "ns/snapshot") || !strings.Contains(condition.Message, string(vs.UID)) { + t.Fatalf("delete failure not visible: %+v", condition) + } + } + }) + } +} diff --git a/operator/internal/controller/backup_controller.go b/operator/internal/controller/backup_controller.go index 1c7d97965..3b4d4e46c 100644 --- a/operator/internal/controller/backup_controller.go +++ b/operator/internal/controller/backup_controller.go @@ -127,6 +127,10 @@ type BackupReconciler struct { // DefaultBackupJobBackoffLimit / DefaultBackupJobActiveDeadlineSeconds. JobBackoffLimit *int32 JobActiveDeadlineSeconds int64 + PreparationTimeout time.Duration + QuiesceTimeout time.Duration + CSICaptureTimeout time.Duration + HardSameNode bool // SnapshotScrapeGracePeriod bounds how long after the restic Job // completes the reconciler keeps retrying to read the snapshot id from // the pod logs before failing the Backup. Zero uses @@ -235,7 +239,7 @@ func resticImageOrDefault(image string) string { // +kubebuilder:rbac:groups="",resources=events,verbs=create;patch // +kubebuilder:rbac:groups=events.k8s.io,resources=events,verbs=create;patch // +kubebuilder:rbac:groups=snapshot.storage.k8s.io,resources=volumesnapshots,verbs=get;list;watch -// +kubebuilder:rbac:groups=snapshot.storage.k8s.io,resources=volumesnapshotcontents,verbs=get;list;watch +// +kubebuilder:rbac:groups=snapshot.storage.k8s.io,resources=volumesnapshotcontents,verbs=get;list;watch;delete func (r *BackupReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { return requeueOnConflict(r.reconcile(ctx, req)) @@ -250,6 +254,12 @@ func (r *BackupReconciler) reconcile(ctx context.Context, req ctrl.Request) (ctr if !b.DeletionTimestamp.IsZero() { return r.finalizeDelete(ctx, &b) } + if b.Annotations[backupCopyAbortAnnotation] != "" { + return r.abortResticCopy(ctx, &b, b.Annotations[backupCopyAbortAnnotation]) + } + if b.Annotations[backupSnapshotAbortAnnotation] == "true" { + return r.abortVolumeSnapshot(ctx, &b, "volume snapshot capture was aborted") + } // Every copy owns the source until its workers drain, even without quiesce. if !controllerutil.ContainsFinalizer(&b, gameplanev1alpha1.BackupFinalizer) { controllerutil.AddFinalizer(&b, gameplanev1alpha1.BackupFinalizer) @@ -292,6 +302,12 @@ func (r *BackupReconciler) reconcile(ctx context.Context, req ctrl.Request) (ctr // is outstanding, so this stays cheap on the common path. if b.Status.Phase == gameplanev1alpha1.BackupPhaseSucceeded || b.Status.Phase == gameplanev1alpha1.BackupPhaseFailed { + if b.Status.Phase == gameplanev1alpha1.BackupPhaseFailed && b.Spec.Strategy == "volume-snapshot" { + return r.abortVolumeSnapshot(ctx, &b, b.Status.Message) + } + if b.Status.Phase == gameplanev1alpha1.BackupPhaseFailed && b.Annotations[backupPreparationDeadlineAnnotation] != "" { + return r.abortResticCopy(ctx, &b, b.Status.Message) + } return r.runUnquiesce(ctx, &b) } // Observe pre-upgrade workers even when multiple legacy quiesced copies @@ -357,39 +373,7 @@ func (r *BackupReconciler) reconcile(ctx context.Context, req ctrl.Request) (ctr b.Spec.RepoRef.Name, strings.Join(missing, ", "))) } - if res, err := r.prepareBackupQuiesce(ctx, &b); err != nil || res.RequeueAfter > 0 { - return res, err - } - // Admission refreshes the target. Resolve the pin only after owning the - // lifecycle fence, so a concurrent rollout cannot change the copy layout. - bound, err := r.ownedBackupTarget(ctx, &b) - if err != nil { - return ctrl.Result{}, err - } - if bound == nil { - return ctrl.Result{RequeueAfter: 5 * time.Second}, nil - } - tmpl, err := effectiveGameTemplate(ctx, r.backupAPIReader(), bound) - if err != nil && (bound.Annotations[effectiveTemplateAnnotation] != "" || !apierrors.IsNotFound(err)) { - return ctrl.Result{}, err - } - - backoff, deadline := resolveJobLimits(r.JobBackoffLimit, r.JobActiveDeadlineSeconds) - job := &batchv1.Job{ObjectMeta: metav1.ObjectMeta{Name: b.Name, Namespace: b.Namespace}} - if _, err := controllerutil.CreateOrUpdate(ctx, r.Client, job, func() error { - if job.CreationTimestamp.IsZero() { - job.Spec.BackoffLimit = &backoff - job.Spec.ActiveDeadlineSeconds = &deadline - job.Spec.Template.Labels = map[string]string{backupRestoreJobLabel: backupRestoreJobValue, backupUIDLabel: string(b.UID)} - job.Spec.Template.Spec = r.buildBackupPodSpec(&b, tmpl) - job.Spec.Template.Spec.RestartPolicy = corev1.RestartPolicyNever - } - return controllerutil.SetControllerReference(&b, job, r.Scheme) - }); err != nil { - return ctrl.Result{}, err - } - - return r.mirrorJobStatus(ctx, &b, job) + return r.reconcilePreparedRestic(ctx, &b) } // maybeQuiesce calls the agent's /quiesce endpoint once per Backup. @@ -587,6 +571,9 @@ func (r *BackupReconciler) mirrorJobStatus( cond.Reason = string(phase) } newConds := upsertCondition(b.Status.Conditions, cond) + if phase == gameplanev1alpha1.BackupPhaseSucceeded || phase == gameplanev1alpha1.BackupPhaseFailed { + meta.RemoveStatusCondition(&newConds, "CaptureReady") + } newConds = upsertCondition(newConds, backupConsistencyCondition(b)) if !sameConditions(b.Status.Conditions, newConds) { b.Status.Conditions = newConds @@ -785,6 +772,11 @@ func (r *BackupReconciler) finalizeUnquiesce(ctx context.Context, b *gameplanev1 return ctrl.Result{}, err } } + if b.Annotations[backupSnapshotAbortAnnotation] == "true" { + if res, err := r.cleanupAbortedSnapshot(ctx, b); err != nil || res.RequeueAfter > 0 { + return res, err + } + } controllerutil.RemoveFinalizer(b, gameplanev1alpha1.BackupFinalizer) if err := r.Update(ctx, b); err != nil { return ctrl.Result{}, err @@ -964,6 +956,7 @@ func (r *BackupReconciler) readResticSummary(ctx context.Context, b *gameplanev1 func (r *BackupReconciler) fail(ctx context.Context, b *gameplanev1alpha1.Backup, msg string) (ctrl.Result, error) { now := metav1.Now() b.Status.Phase = gameplanev1alpha1.BackupPhaseFailed + meta.RemoveStatusCondition(&b.Status.Conditions, "CaptureReady") b.Status.ObservedGeneration = b.Generation if b.Status.Message != msg { b.Status.Message = msg diff --git a/operator/internal/controller/backup_envtest_test.go b/operator/internal/controller/backup_envtest_test.go index f577563a6..e67328880 100644 --- a/operator/internal/controller/backup_envtest_test.go +++ b/operator/internal/controller/backup_envtest_test.go @@ -8,11 +8,9 @@ import ( "testing" "time" - appsv1 "k8s.io/api/apps/v1" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/types" gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" ) @@ -56,7 +54,7 @@ func TestBackup_CreatesJobWithExpectedSpec(t *testing.T) { if !strings.HasPrefix(c.Image, "restic/restic:") { t.Errorf("image = %q, want restic/restic:*", c.Image) } - if got, want := c.Args, []string{"backup", "/data", "--json", "--retry-lock", "10m", "--tag", "gameplane"}; !equalStrings(got, want) { + if got, want := c.Args, []string{resticWorkerScript, "backup-worker", "backup", "/data", "--json", "--retry-lock", "10m", "--tag", "gameplane"}; !equalStrings(got, want) { t.Errorf("args = %v, want %v", got, want) } @@ -365,6 +363,7 @@ func TestBackup_PassesTagsToRestic(t *testing.T) { return false, "job not yet created" } want := []string{"backup", "/data", "--json", "--retry-lock", "10m", "--tag", "gameplane", "--tag", "nightly", "--tag", "preupgrade"} + want = append([]string{resticWorkerScript, "backup-worker"}, want...) if !equalStrings(j.Spec.Template.Spec.Containers[0].Args, want) { return false, "args = " + strings.Join(j.Spec.Template.Spec.Containers[0].Args, " ") } @@ -372,29 +371,15 @@ func TestBackup_PassesTagsToRestic(t *testing.T) { }) } -// TestBackup_QuiesceAnnotationSetBeforeJob verifies that with a real -// agent client (faked) and spec.quiesce=true (the default), the -// controller calls Quiesce and stamps the quiesced-at annotation -// before the restic Job appears. -func TestBackup_QuiesceAnnotationSetBeforeJob(t *testing.T) { +// The Job must prepare before saving is disabled; authorization follows the +// durable quiesce annotation and cannot be granted to an unready worker. +func TestBackup_QuiesceAfterWorkerPreparation(t *testing.T) { ns := newNamespace(t) fa := &fakeAgent{} startMgr(t, ns, withBackupReconciler(backupReconcilerOpts{agent: fa})) + // The shared fixture supplies the observed workload and its scheduled, + // running source Pod; envtest has no StatefulSet controller or kubelet. seedGameServer(t, ns, "smp") - // envtest has no StatefulSet controller or kubelet: provide the observed - // running source whose immutable identity is bound before the agent RPC. - var workload appsv1.StatefulSet - if err := k8sClient.Get(context.Background(), types.NamespacedName{Namespace: ns, Name: "smp"}, &workload); err != nil { - t.Fatal(err) - } - pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "smp-0", Namespace: ns, OwnerReferences: []metav1.OwnerReference{{APIVersion: "apps/v1", Kind: "StatefulSet", Name: workload.Name, UID: workload.UID, Controller: ownerBoolPtr(true)}}}, Spec: workload.Spec.Template.Spec} - if err := k8sClient.Create(context.Background(), pod); err != nil { - t.Fatal(err) - } - pod.Status.Phase = corev1.PodRunning - if err := k8sClient.Status().Update(context.Background(), pod); err != nil { - t.Fatal(err) - } if err := k8sClient.Create(context.Background(), buildResticRepoSecret(ns, "repo")); err != nil { t.Fatalf("create secret: %v", err) @@ -404,6 +389,16 @@ func TestBackup_QuiesceAnnotationSetBeforeJob(t *testing.T) { if err := k8sClient.Create(context.Background(), b); err != nil { t.Fatalf("create backup: %v", err) } + var copyJob *batchv1.Job + eventually(t, func() (bool, string) { + var ok bool + copyJob, ok = getJob(t, ns, "smp-q") + return ok, "job not yet created" + }) + if fa.quiesceCount() != 0 { + t.Fatal("save-off happened before worker preparation") + } + prepareEnvtestBackupWorker(t, copyJob) eventually(t, func() (bool, string) { got := getBackup(t, ns, "smp-q") @@ -416,7 +411,7 @@ func TestBackup_QuiesceAnnotationSetBeforeJob(t *testing.T) { return true, "" }) - // Job exists and was created at-or-after the quiesce call. + // The prepared Job remains the authorized copy. eventually(t, func() (bool, string) { _, ok := getJob(t, ns, "smp-q") return ok, "job not yet created" diff --git a/operator/internal/controller/backup_forget.go b/operator/internal/controller/backup_forget.go index 2d97d9410..5d73b8170 100644 --- a/operator/internal/controller/backup_forget.go +++ b/operator/internal/controller/backup_forget.go @@ -112,6 +112,21 @@ func wantsSnapshotFinalizer(b *gameplanev1alpha1.Backup) bool { // forgets the restic snapshot. Unquiesce goes first because it is time // critical: a world left with auto-save off is worse than an orphaned snapshot. func (r *BackupReconciler) finalizeDelete(ctx context.Context, b *gameplanev1alpha1.Backup) (ctrl.Result, error) { + if b.Annotations[backupPreparationDeadlineAnnotation] != "" { + if err := r.revokeBackupStart(ctx, b); err != nil { + return ctrl.Result{}, err + } + } + if b.Spec.Strategy == "volume-snapshot" && b.Status.Phase != gameplanev1alpha1.BackupPhaseSucceeded { + if err := r.recordSnapshotAbort(ctx, b, "CSI capture cancelled by Backup deletion"); err != nil { + return ctrl.Result{}, err + } + } + if b.Annotations[backupPreparationDeadlineAnnotation] != "" && b.Status.Phase != gameplanev1alpha1.BackupPhaseSucceeded { + if res, err := r.abortResticCopy(ctx, b, "copy cancelled by Backup deletion"); err != nil || res.RequeueAfter > 0 { + return res, err + } + } if controllerutil.ContainsFinalizer(b, gameplanev1alpha1.BackupFinalizer) { res, err := r.finalizeUnquiesce(ctx, b) if err != nil || res.RequeueAfter > 0 { diff --git a/operator/internal/controller/backup_guard.go b/operator/internal/controller/backup_guard.go index d29ff2b5b..cbd5ec62f 100644 --- a/operator/internal/controller/backup_guard.go +++ b/operator/internal/controller/backup_guard.go @@ -247,7 +247,7 @@ func (r *BackupReconciler) legacyQuiescedBackups(ctx context.Context, b *gamepla return legacy, nil } -func (r *BackupReconciler) prepareBackupQuiesce(ctx context.Context, b *gameplanev1alpha1.Backup) (ctrl.Result, error) { +func (r *BackupReconciler) prepareBackupWorkload(ctx context.Context, b *gameplanev1alpha1.Backup) (ctrl.Result, error) { claimed, err := r.claimBackupTarget(ctx, b) if err != nil { if errors.Is(err, errLifecycleGenerationChanged) { @@ -278,12 +278,23 @@ func (r *BackupReconciler) prepareBackupQuiesce(ctx context.Context, b *gameplan if err != nil || !ready { return ctrl.Result{RequeueAfter: 5 * time.Second}, err } + return ctrl.Result{}, nil +} + +func (r *BackupReconciler) prepareBackupQuiesce(ctx context.Context, b *gameplanev1alpha1.Backup) (ctrl.Result, error) { + if res, err := r.prepareBackupWorkload(ctx, b); err != nil || res.RequeueAfter > 0 { + return res, err + } + gs, err := r.ownedBackupTarget(ctx, b) + if err != nil || gs == nil { + return ctrl.Result{RequeueAfter: 5 * time.Second}, err + } if err := r.maybeQuiesce(ctx, b); err != nil { return ctrl.Result{}, err } // The Service RPC may overlap a pod replacement. Never start a copy on a // different process from the one pinned before the request. - ready, err = r.prepareBackupSource(ctx, b, gs) + ready, err := r.prepareBackupSource(ctx, b, gs) if errors.Is(err, errBackupSourceChanged) { _, err = r.fail(ctx, b, err.Error()) return ctrl.Result{RequeueAfter: time.Nanosecond}, err @@ -351,6 +362,14 @@ func (r *BackupReconciler) releaseBackupTarget(ctx context.Context, b *gameplane // world while a deleting/failed Job still has a restic worker mounted. The // UID label keeps orphaned workers visible even after their Job disappears. func (r *BackupReconciler) backupWorkersLive(ctx context.Context, namespace string, id backupIdentity, cancel bool) (bool, error) { + if cancel { + // Recovery can run after the Backup disappeared. The immutable owner + // UID still identifies its gate, so revoke it before touching workers. + owner := &gameplanev1alpha1.Backup{ObjectMeta: metav1.ObjectMeta{Name: id.Name, Namespace: namespace, UID: id.UID}} + if err := r.revokeBackupStart(ctx, owner); err != nil { + return true, err + } + } var jobs batchv1.JobList if err := r.backupAPIReader().List(ctx, &jobs, client.InNamespace(namespace)); err != nil { return false, err @@ -393,13 +412,13 @@ func (r *BackupReconciler) backupWorkersLive(ctx context.Context, namespace stri if busy { return true, nil } - return r.backupSnapshotCopyLive(ctx, namespace, id) + return r.backupSnapshotCopyLive(ctx, namespace, id, cancel) } -// CSI captures have no Job or Pod. On normal deletion retain quiesce until -// capture completion/error instead of deleting the pending VolumeSnapshot; -// owner-reference GC will delete it after the Backup's finalizer clears. -func (r *BackupReconciler) backupSnapshotCopyLive(ctx context.Context, namespace string, id backupIdentity) (bool, error) { +// Normal captures remain live until ready/error. Cancellation first invalidates +// the Backup; orphan recovery marks its UID-bound snapshot invalid before +// save-on, since no surviving Backup can expose it as a restorable capture. +func (r *BackupReconciler) backupSnapshotCopyLive(ctx context.Context, namespace string, id backupIdentity, cancel bool) (bool, error) { scheme := r.Scheme if scheme == nil { scheme = r.Client.Scheme() @@ -427,6 +446,29 @@ func (r *BackupReconciler) backupSnapshotCopyLive(ctx context.Context, namespace if owner == nil || owner.Kind != "Backup" || owner.UID != id.UID { return false, nil } + if cancel { + if b.UID == id.UID && b.Status.Phase != gameplanev1alpha1.BackupPhaseSucceeded { + return false, r.recordSnapshotAbort(ctx, &b, "CSI capture cancelled during lifecycle recovery") + } + if b.UID != id.UID { + if vs.Annotations == nil { + vs.Annotations = map[string]string{} + } + if vs.Annotations[backupSnapshotAbortAnnotation] != "true" { + vs.Annotations[backupSnapshotAbortAnnotation] = "true" + if err := r.Update(ctx, &vs); err != nil { + return true, err + } + } + uid := vs.UID + if uid != "" && vs.DeletionTimestamp.IsZero() { + if err := r.Delete(ctx, &vs, &client.DeleteOptions{Preconditions: &metav1.Preconditions{UID: &uid}}); err != nil && !apierrors.IsNotFound(err) { + ctrl.LoggerFrom(ctx).Error(err, "orphaned aborted snapshot requires reclamation", "snapshot", client.ObjectKeyFromObject(&vs), "uid", uid) + } + } + return false, nil + } + } if vs.Status != nil && ((vs.Status.ReadyToUse != nil && *vs.Status.ReadyToUse) || (vs.Status.Error != nil && vs.Status.Error.Message != nil)) { return false, nil } diff --git a/operator/internal/controller/backup_guard_test.go b/operator/internal/controller/backup_guard_test.go index 4782cc635..90e6ee14a 100644 --- a/operator/internal/controller/backup_guard_test.go +++ b/operator/internal/controller/backup_guard_test.go @@ -58,6 +58,12 @@ func guardBackup(name string) *gameplanev1alpha1.Backup { } func guardFixture(t *testing.T, backups ...*gameplanev1alpha1.Backup) (*BackupReconciler, *guardQuiescer) { + return guardFixtureWorkers(t, true, backups...) +} + +// Most guard tests model a healthy Job controller/kubelet. Budget tests opt +// out and advance worker scheduling/readiness explicitly. +func guardFixtureWorkers(t *testing.T, automatic bool, backups ...*gameplanev1alpha1.Backup) (*BackupReconciler, *guardQuiescer) { t.Helper() s := scrapeScheme(t) if err := snapshotv1.AddToScheme(s); err != nil { @@ -69,11 +75,36 @@ func guardFixture(t *testing.T, backups ...*gameplanev1alpha1.Backup) (*BackupRe objects := []client.Object{&gameplanev1alpha1.GameServer{ObjectMeta: metav1.ObjectMeta{Name: "server", Namespace: "ns", UID: "server-uid"}, Spec: gameplanev1alpha1.GameServerSpec{TemplateRef: gameplanev1alpha1.GameTemplateRef{Name: "template"}}}, &corev1.Secret{ObjectMeta: metav1.ObjectMeta{Name: "repo", Namespace: "ns"}, Data: map[string][]byte{"repo": []byte("repo"), "password": []byte("password")}}} one := int32(1) objects = append(objects, &appsv1.StatefulSet{ObjectMeta: metav1.ObjectMeta{Name: "server", Namespace: "ns", UID: "workload-uid", OwnerReferences: []metav1.OwnerReference{{APIVersion: gameplanev1alpha1.GroupVersion.String(), Kind: "GameServer", Name: "server", UID: "server-uid", Controller: ownerBoolPtr(true)}}}, Spec: appsv1.StatefulSetSpec{Replicas: &one}}) - objects = append(objects, &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "server-0", Namespace: "ns", UID: "game-pod-uid", OwnerReferences: []metav1.OwnerReference{{APIVersion: "apps/v1", Kind: "StatefulSet", Name: "server", UID: "workload-uid", Controller: ownerBoolPtr(true)}}}, Status: corev1.PodStatus{Phase: corev1.PodRunning}}) + objects = append(objects, &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: "server-0", Namespace: "ns", UID: "game-pod-uid", OwnerReferences: []metav1.OwnerReference{{APIVersion: "apps/v1", Kind: "StatefulSet", Name: "server", UID: "workload-uid", Controller: ownerBoolPtr(true)}}}, Spec: corev1.PodSpec{NodeName: "node-a"}, Status: corev1.PodStatus{Phase: corev1.PodRunning}}) for _, b := range backups { objects = append(objects, b) } cl := fake.NewClientBuilder().WithScheme(s).WithObjects(objects...).WithStatusSubresource(&gameplanev1alpha1.Backup{}, &gameplanev1alpha1.GameServer{}, &batchv1.Job{}, &snapshotv1.VolumeSnapshot{}).Build() + cl = interceptor.NewClient(cl, interceptor.Funcs{Create: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.CreateOption) error { + if obj.GetUID() == "" { + obj.SetUID(types.UID(obj.GetName() + "-generated-uid")) + } + if err := c.Create(ctx, obj, opts...); err != nil { + return err + } + job, ok := obj.(*batchv1.Job) + if !automatic || !ok || job.Annotations[backupWorkerProtocolAnnotation] != "v1" { + return nil + } + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: job.Name + "-automatic", Namespace: job.Namespace, UID: types.UID(job.Name + "-automatic-uid"), Labels: job.Spec.Template.Labels, + OwnerReferences: []metav1.OwnerReference{{APIVersion: "batch/v1", Kind: "Job", Name: job.Name, UID: job.UID, Controller: ownerBoolPtr(true)}}}, Spec: job.Spec.Template.Spec, + Status: corev1.PodStatus{Phase: corev1.PodRunning, Conditions: []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionTrue}}}} + pod.Spec.NodeName = "node-a" + return c.Create(ctx, pod) + }, Delete: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.DeleteOption) error { + if err := c.Delete(ctx, obj, opts...); err != nil { + return err + } + if job, ok := obj.(*batchv1.Job); automatic && ok { + return client.IgnoreNotFound(c.Delete(ctx, &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: job.Name + "-automatic", Namespace: job.Namespace}})) + } + return nil + }}) q := &guardQuiescer{} return &BackupReconciler{Client: cl, Scheme: s, AgentClient: q, LogReader: scrapeLogReader{body: `{"message_type":"summary","snapshot_id":"abc123"}`}}, q } @@ -81,6 +112,14 @@ func guardFixture(t *testing.T, backups ...*gameplanev1alpha1.Backup) (*BackupRe func guardPass(t *testing.T, r *BackupReconciler, name string) { t.Helper() for i := 0; i < 3; i++ { + // Model a completed worker after tests advance the Job's terminal + // status. Explicitly created live/terminating pods are never removed. + var job batchv1.Job + if err := r.Get(context.Background(), client.ObjectKey{Namespace: "ns", Name: name}, &job); err == nil && (job.Status.Succeeded > 0 || jobPermanentlyFailed(&job)) { + if err := r.Delete(context.Background(), &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Namespace: "ns", Name: name + "-automatic"}}); err != nil && !apierrors.IsNotFound(err) { + t.Fatal(err) + } + } if _, err := r.Reconcile(context.Background(), ctrl.Request{NamespacedName: types.NamespacedName{Namespace: "ns", Name: name}}); err != nil { t.Fatal(err) } @@ -541,7 +580,7 @@ func TestBackupGuardRecoveryLineageSurvivesRepeatedOwnerDeletion(t *testing.T) { } } -func TestBackupGuardPendingSnapshotCleanupWaitsForCapture(t *testing.T) { +func TestBackupGuardPendingSnapshotCancellationInvalidatesBeforeSaveOn(t *testing.T) { for _, mode := range []string{"delete", "orphan", "legacy"} { t.Run(mode, func(t *testing.T) { a := guardBackup("snapshot") @@ -562,6 +601,19 @@ func TestBackupGuardPendingSnapshotCleanupWaitsForCapture(t *testing.T) { guardPass(t, r, "snapshot") } guardPass(t, r, "next") + if mode != "legacy" { + if q.unquiesced != 1 { + t.Fatalf("cancelled CSI capture did not restore saving: %+v", q) + } + var jobs batchv1.JobList + if err := r.List(context.Background(), &jobs); err != nil { + t.Fatal(err) + } + if len(jobs.Items) != 1 { + t.Fatal("next copy did not start after explicit CSI abort") + } + return + } if q.unquiesced != 0 { t.Fatalf("save-on during pending CSI capture: %+v", q) } @@ -635,11 +687,8 @@ func TestBackupSnapshotAmbiguousCreateStillCleansUp(t *testing.T) { t.Fatalf("ambiguous create lost abort cleanup: %+v", q) } var vs snapshotv1.VolumeSnapshot - if err := r.Get(context.Background(), types.NamespacedName{Namespace: "ns", Name: "snapshot"}, &vs); err != nil { - t.Fatal(err) - } - if vs.Status != nil && vs.Status.ReadyToUse != nil && *vs.Status.ReadyToUse { - t.Fatal("expected still-pending ambiguous capture") + if err := r.Get(context.Background(), types.NamespacedName{Namespace: "ns", Name: "snapshot"}, &vs); !apierrors.IsNotFound(err) { + t.Fatalf("ambiguous aborted capture was not reclaimed: %v", err) } } diff --git a/operator/internal/controller/backup_manifest.go b/operator/internal/controller/backup_manifest.go index c2c2afb70..210ab286a 100644 --- a/operator/internal/controller/backup_manifest.go +++ b/operator/internal/controller/backup_manifest.go @@ -15,8 +15,8 @@ import ( gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" ) -// Called after lifecycle ownership and the workload fence, before creating the -// copy worker. Requeues and later template edits never rewrite this manifest. +// Called after lifecycle ownership and the workload fence, before a copy is +// authorized. Requeues and later template edits never rewrite this manifest. func (r *BackupReconciler) recordBackupManifest(ctx context.Context, b *gameplanev1alpha1.Backup, gs *gameplanev1alpha1.GameServer) error { if b.Status.CaptureManifest != nil { return nil @@ -27,10 +27,25 @@ func (r *BackupReconciler) recordBackupManifest(ctx context.Context, b *gameplan worker = &snapshotv1.VolumeSnapshot{} } if err := reader.Get(ctx, client.ObjectKeyFromObject(b), worker); err == nil { - // Existing workers predate this metadata. Current layout cannot prove - // what they captured, even if their requested quiesce was successful. - b.Status.CaptureManifest = &gameplanev1alpha1.BackupCaptureManifest{State: "LegacyUnknown"} - return r.Status().Update(ctx, b) + // Current prepared workers exist before capture but cannot read data + // without authorization. Its durable annotation is written before the + // projected token and never cleared, including during cancellation. + // All other existing workers retain unknown historical coverage. + waiting := false + if job, ok := worker.(*batchv1.Job); ok && metav1.IsControlledBy(job, b) && job.Annotations[backupWorkerProtocolAnnotation] == "v1" && + b.Annotations[backupPreparationDeadlineAnnotation] != "" && b.Annotations[backupAuthorizedWorkerAnnotation] == "" && + job.Status.Succeeded == 0 && !jobPermanentlyFailed(job) { + var gate corev1.ConfigMap + if err := reader.Get(ctx, client.ObjectKey{Namespace: b.Namespace, Name: backupStartConfigMapName(b)}, &gate); err == nil { + waiting = metav1.IsControlledBy(&gate, b) && gate.Data[backupStartTokenKey] == "" + } else if !apierrors.IsNotFound(err) { + return err + } + } + if !waiting { + b.Status.CaptureManifest = &gameplanev1alpha1.BackupCaptureManifest{State: "LegacyUnknown"} + return r.Status().Update(ctx, b) + } } else if !apierrors.IsNotFound(err) { return err } diff --git a/operator/internal/controller/backup_manifest_test.go b/operator/internal/controller/backup_manifest_test.go index 89b6a00fd..69b471bd5 100644 --- a/operator/internal/controller/backup_manifest_test.go +++ b/operator/internal/controller/backup_manifest_test.go @@ -141,6 +141,46 @@ func TestBackupManifestDoesNotInventHistoryForExistingWorker(t *testing.T) { } } +func TestBackupManifestDoesNotInventHistoryForAuthorizedPreparedWorker(t *testing.T) { + for _, mode := range []string{"authorized", "projected-token", "missing-gate", "foreign-gate"} { + t.Run(mode, func(t *testing.T) { + b := guardBackup("copy") + b.Annotations = map[string]string{backupPreparationDeadlineAnnotation: "persisted-before-preparation"} + r, _ := guardFixtureWorkers(t, false, b) + job := &batchv1.Job{ObjectMeta: metav1.ObjectMeta{Name: b.Name, Namespace: b.Namespace, Annotations: map[string]string{backupWorkerProtocolAnnotation: "v1"}, + OwnerReferences: []metav1.OwnerReference{*metav1.NewControllerRef(b, gameplanev1alpha1.GroupVersion.WithKind("Backup"))}}} + if err := r.Create(t.Context(), job); err != nil { + t.Fatal(err) + } + gate := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: backupStartConfigMapName(b), Namespace: b.Namespace, + OwnerReferences: job.OwnerReferences}, Data: map[string]string{backupStartTokenKey: ""}} + if mode == "authorized" { + b.Annotations[backupAuthorizedWorkerAnnotation] = "previous-worker" + if err := r.Update(t.Context(), b); err != nil { + t.Fatal(err) + } + } + if mode == "projected-token" { + gate.Data[backupStartTokenKey] = "previous-authorization" + } + if mode == "foreign-gate" { + gate.OwnerReferences = nil + } + if mode != "missing-gate" { + if err := r.Create(t.Context(), gate); err != nil { + t.Fatal(err) + } + } + if err := r.recordBackupManifest(t.Context(), b, &gameplanev1alpha1.GameServer{}); err != nil { + t.Fatal(err) + } + if b.Status.CaptureManifest == nil || b.Status.CaptureManifest.State != "LegacyUnknown" || len(b.Status.CaptureManifest.Volumes) != 0 { + t.Fatalf("invented history for potentially started copy: %+v", b.Status.CaptureManifest) + } + }) + } +} + func TestBackupConsistencyDescribesAchievedResult(t *testing.T) { for _, tc := range []struct { strategy, outcome, reason string diff --git a/operator/internal/controller/backup_snapshot_cleanup.go b/operator/internal/controller/backup_snapshot_cleanup.go new file mode 100644 index 000000000..6e07b7c0c --- /dev/null +++ b/operator/internal/controller/backup_snapshot_cleanup.go @@ -0,0 +1,215 @@ +package controller + +import ( + "context" + "encoding/json" + "fmt" + "strings" + "time" + + snapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + + gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" +) + +const backupSnapshotIdentityAnnotation = "backup.gameplane.local/snapshot-identity" +const backupSnapshotAbortMessageAnnotation = "backup.gameplane.local/snapshot-abort-reason" + +type backupSnapshotIdentity struct { + UID types.UID `json:"uid"` + Contents map[string]types.UID `json:"contents,omitempty"` +} + +func (r *BackupReconciler) rememberBackupSnapshot(ctx context.Context, b *gameplanev1alpha1.Backup, vs *snapshotv1.VolumeSnapshot) error { + var id backupSnapshotIdentity + if value := b.Annotations[backupSnapshotIdentityAnnotation]; value != "" { + if err := json.Unmarshal([]byte(value), &id); err != nil { + return fmt.Errorf("invalid snapshot identity: %w", err) + } + } + if !metav1.IsControlledBy(vs, b) || (id.UID != "" && id.UID != vs.UID) { + return fmt.Errorf("VolumeSnapshot %s/%s identity changed", vs.Namespace, vs.Name) + } + if id.UID == vs.UID { + return nil + } + id.UID = vs.UID + encoded, err := json.Marshal(id) + if err != nil { + return err + } + patchBackupAnnotations(b, map[string]string{backupSnapshotIdentityAnnotation: string(encoded)}) + return r.Update(ctx, b) +} + +func (r *BackupReconciler) recordSnapshotAbort(ctx context.Context, b *gameplanev1alpha1.Backup, message string) error { + if b.Annotations[backupSnapshotAbortAnnotation] != "true" { + patchBackupAnnotations(b, map[string]string{backupSnapshotAbortAnnotation: "true", backupSnapshotAbortMessageAnnotation: message}) + if err := r.Update(ctx, b); err != nil { + return err + } + } + if stored := b.Annotations[backupSnapshotAbortMessageAnnotation]; stored != "" { + message = stored + } + if b.Status.Phase == gameplanev1alpha1.BackupPhaseFailed && b.Status.SnapshotID == "" && b.Status.VolumeSnapshotContentName == "" && b.Status.Message == message { + return nil + } + b.Status.SnapshotID, b.Status.VolumeSnapshotContentName = "", "" + _, err := r.fail(ctx, b, message) + return err +} + +func (r *BackupReconciler) snapshotCleanupCondition(ctx context.Context, b *gameplanev1alpha1.Backup, status metav1.ConditionStatus, reason, message string) error { + conditions := upsertCondition(b.Status.Conditions, metav1.Condition{Type: "SnapshotCleaned", Status: status, Reason: reason, Message: message, ObservedGeneration: b.Generation}) + if sameConditions(b.Status.Conditions, conditions) { + return nil + } + b.Status.Conditions = conditions + if status != metav1.ConditionTrue { + r.recordEvent(b, corev1.EventTypeWarning, reason, message) + } + return r.Status().Update(ctx, b) +} + +// Cleanup is independent of save-on once the capture is durably invalidated. +// The original snapshot UID and each content UID are retained on the Backup; +// resource replacements are reported, never adopted or deleted. +func (r *BackupReconciler) cleanupAbortedSnapshot(ctx context.Context, b *gameplanev1alpha1.Backup) (ctrl.Result, error) { + res, err := r.reclaimAbortedSnapshot(ctx, b) + if err == nil { + return res, nil + } + if conditionErr := r.snapshotCleanupCondition(ctx, b, metav1.ConditionFalse, "SnapshotCleanupPending", err.Error()); conditionErr != nil { + return ctrl.Result{}, conditionErr + } + return ctrl.Result{RequeueAfter: 10 * time.Second}, nil +} + +func (r *BackupReconciler) reclaimAbortedSnapshot(ctx context.Context, b *gameplanev1alpha1.Backup) (ctrl.Result, error) { + var id backupSnapshotIdentity + if value := b.Annotations[backupSnapshotIdentityAnnotation]; value != "" { + if err := json.Unmarshal([]byte(value), &id); err != nil { + return ctrl.Result{}, err + } + } + var vs snapshotv1.VolumeSnapshot + err := r.backupAPIReader().Get(ctx, client.ObjectKeyFromObject(b), &vs) + if meta.IsNoMatchError(err) { + return ctrl.Result{}, fmt.Errorf("snapshot API unavailable; inspect VolumeSnapshot %s/%s and recorded identity %s", b.Namespace, b.Name, b.Annotations[backupSnapshotIdentityAnnotation]) + } + if err != nil && !apierrors.IsNotFound(err) { + return ctrl.Result{}, fmt.Errorf("read VolumeSnapshot %s/%s (recorded UID %s): %w", b.Namespace, b.Name, id.UID, err) + } + present := err == nil + if present { + if !metav1.IsControlledBy(&vs, b) || (id.UID != "" && id.UID != vs.UID) { + return ctrl.Result{}, r.snapshotCleanupCondition(ctx, b, metav1.ConditionFalse, "SnapshotCleanupUnverified", fmt.Sprintf("refusing replacement VolumeSnapshot %s/%s UID %s; original UID %s", vs.Namespace, vs.Name, vs.UID, id.UID)) + } + if id.UID == "" { + if vs.UID == "" { + return ctrl.Result{}, fmt.Errorf("VolumeSnapshot %s/%s has no verifiable UID", vs.Namespace, vs.Name) + } + if err := r.rememberBackupSnapshot(ctx, b, &vs); err != nil { + return ctrl.Result{}, err + } + id.UID = vs.UID + } + if vs.Annotations == nil { + vs.Annotations = map[string]string{} + } + if vs.Annotations[backupSnapshotAbortAnnotation] != "true" { + vs.Annotations[backupSnapshotAbortAnnotation] = "true" + if err := r.Update(ctx, &vs); err != nil { + return ctrl.Result{}, err + } + } + } + var contents snapshotv1.VolumeSnapshotContentList + if err := r.backupAPIReader().List(ctx, &contents); err != nil { + return ctrl.Result{}, fmt.Errorf("list contents for VolumeSnapshot %s/%s UID %s: %w", b.Namespace, b.Name, id.UID, err) + } + if id.Contents == nil { + id.Contents = map[string]types.UID{} + } + var retained []string + unverified := false + boundName := "" + if present && vs.Status != nil && vs.Status.BoundVolumeSnapshotContentName != nil { + boundName = *vs.Status.BoundVolumeSnapshotContentName + } + var owned []*snapshotv1.VolumeSnapshotContent + for i := range contents.Items { + content := &contents.Items[i] + ref := content.Spec.VolumeSnapshotRef + if known, ok := id.Contents[content.Name]; ok && known != content.UID { + unverified = true + retained = append(retained, fmt.Sprintf("replacement content %s UID %s (original %s)", content.Name, content.UID, known)) + continue + } + if id.UID == "" || ref.UID != id.UID || ref.Name != b.Name || ref.Namespace != b.Namespace { + if _, known := id.Contents[content.Name]; known || content.Name == boundName { + // Retain the rejected candidate's UID so deleting the valid + // snapshot cannot erase the administrator's follow-up identity. + id.Contents[content.Name] = content.UID + unverified = true + retained = append(retained, fmt.Sprintf("unverified content %s UID %s references snapshot %s/%s UID %s", content.Name, content.UID, ref.Namespace, ref.Name, ref.UID)) + } + continue + } + id.Contents[content.Name] = content.UID + if content.Spec.DeletionPolicy == snapshotv1.VolumeSnapshotContentRetain { + retained = append(retained, fmt.Sprintf("Retain content %s UID %s for snapshot UID %s", content.Name, content.UID, id.UID)) + } else { + owned = append(owned, content) + } + } + encoded, err := json.Marshal(id) + if err != nil { + return ctrl.Result{}, err + } + if string(encoded) != b.Annotations[backupSnapshotIdentityAnnotation] { + patchBackupAnnotations(b, map[string]string{backupSnapshotIdentityAnnotation: string(encoded)}) + if err := r.Update(ctx, b); err != nil { + return ctrl.Result{}, err + } + } + if present && vs.DeletionTimestamp.IsZero() { + uid := vs.UID + if err := r.Delete(ctx, &vs, &client.DeleteOptions{Preconditions: &metav1.Preconditions{UID: &uid}}); err != nil && !apierrors.IsNotFound(err) { + return ctrl.Result{}, fmt.Errorf("delete VolumeSnapshot %s/%s UID %s: %w", vs.Namespace, vs.Name, uid, err) + } + } + for _, content := range owned { + if !content.DeletionTimestamp.IsZero() { + continue + } + uid := content.UID + if uid == "" { + return ctrl.Result{}, fmt.Errorf("content %s has no verifiable UID", content.Name) + } + if err := r.Delete(ctx, content, &client.DeleteOptions{Preconditions: &metav1.Preconditions{UID: &uid}}); err != nil && !apierrors.IsNotFound(err) { + return ctrl.Result{}, fmt.Errorf("delete VolumeSnapshotContent %s UID %s for snapshot UID %s: %w", content.Name, uid, id.UID, err) + } + } + if len(retained) > 0 { + if unverified { + return ctrl.Result{}, r.snapshotCleanupCondition(ctx, b, metav1.ConditionFalse, "SnapshotCleanupUnverified", strings.Join(retained, "; ")+"; administrator verification required") + } + return ctrl.Result{}, r.snapshotCleanupCondition(ctx, b, metav1.ConditionFalse, "SnapshotCleanupRetained", strings.Join(retained, "; ")+"; administrator reclamation required") + } + if present || len(owned) > 0 { + if err := r.snapshotCleanupCondition(ctx, b, metav1.ConditionFalse, "SnapshotCleanupPending", fmt.Sprintf("deletion requested for snapshot %s/%s UID %s and contents %v", b.Namespace, b.Name, id.UID, id.Contents)); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: 10 * time.Second}, nil + } + return ctrl.Result{}, r.snapshotCleanupCondition(ctx, b, metav1.ConditionTrue, "SnapshotCleanupComplete", "aborted snapshot resources are absent") +} diff --git a/operator/internal/controller/backup_snapshot_source.go b/operator/internal/controller/backup_snapshot_source.go new file mode 100644 index 000000000..160619c22 --- /dev/null +++ b/operator/internal/controller/backup_snapshot_source.go @@ -0,0 +1,49 @@ +package controller + +import ( + "context" + "encoding/json" + "fmt" + + snapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" +) + +// A raw GameServer dataSource must not revive an aborted operator capture while +// a CSI driver still exposes its object. Run only before creating a new PVC: +// existing volumes remain usable after retention deletes their source Backup. +// External snapshots without a Gameplane Backup owner keep the direct-CRD +// administrator contract; API authorization is enforced separately. +func validateBackupSnapshotSource(ctx context.Context, reader client.Reader, namespace, name string) error { + var snapshot snapshotv1.VolumeSnapshot + if err := reader.Get(ctx, client.ObjectKey{Namespace: namespace, Name: name}, &snapshot); err != nil { + return fmt.Errorf("resolve snapshot source: %w", err) + } + if snapshot.Annotations[backupSnapshotAbortAnnotation] == "true" { + return fmt.Errorf("snapshot %s was aborted and cannot seed a volume", name) + } + owner := metav1.GetControllerOf(&snapshot) + if owner == nil || owner.APIVersion != gameplanev1alpha1.GroupVersion.String() || owner.Kind != "Backup" { + return nil + } + var backup gameplanev1alpha1.Backup + if err := reader.Get(ctx, client.ObjectKey{Namespace: namespace, Name: owner.Name}, &backup); err != nil { + return fmt.Errorf("resolve snapshot Backup owner: %w", err) + } + if owner.UID == "" || owner.UID != backup.UID || !backup.DeletionTimestamp.IsZero() || !snapshot.DeletionTimestamp.IsZero() || + backup.Annotations[backupSnapshotAbortAnnotation] == "true" || backup.Annotations[backupCopyAbortAnnotation] != "" || + backup.Status.Phase != gameplanev1alpha1.BackupPhaseSucceeded || backup.Status.SnapshotID != snapshot.Name || + snapshot.Status == nil || snapshot.Status.ReadyToUse == nil || !*snapshot.Status.ReadyToUse { + return fmt.Errorf("snapshot %s is not a completed, valid capture of its current Backup owner", name) + } + if value := backup.Annotations[backupSnapshotIdentityAnnotation]; value != "" { + var id backupSnapshotIdentity + if err := json.Unmarshal([]byte(value), &id); err != nil || id.UID == "" || id.UID != snapshot.UID { + return fmt.Errorf("snapshot %s differs from the recorded capture UID", name) + } + } + return nil +} diff --git a/operator/internal/controller/backup_snapshot_source_test.go b/operator/internal/controller/backup_snapshot_source_test.go new file mode 100644 index 000000000..2daa8f82d --- /dev/null +++ b/operator/internal/controller/backup_snapshot_source_test.go @@ -0,0 +1,69 @@ +package controller + +import ( + "encoding/json" + "testing" + + snapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" +) + +func TestSnapshotPVCRefusesIncompleteAbortedAndReplacedCapture(t *testing.T) { + for _, mode := range []string{"valid", "pending", "running", "failed", "abort-before-status", "copy-aborted", "snapshot-aborted", "owner-replaced", "snapshot-replaced", "not-ready", "external"} { + t.Run(mode, func(t *testing.T) { + b := guardBackup("snapshot") + b.Spec.Strategy = "volume-snapshot" + b.Status.Phase, b.Status.SnapshotID = gameplanev1alpha1.BackupPhaseSucceeded, b.Name + identity, err := json.Marshal(backupSnapshotIdentity{UID: "snapshot-uid"}) + if err != nil { + t.Fatal(err) + } + b.Annotations = map[string]string{backupSnapshotIdentityAnnotation: string(identity)} + vs := &snapshotv1.VolumeSnapshot{ObjectMeta: metav1.ObjectMeta{Name: b.Name, Namespace: b.Namespace, UID: "snapshot-uid", OwnerReferences: []metav1.OwnerReference{{APIVersion: gameplanev1alpha1.GroupVersion.String(), Kind: "Backup", Name: b.Name, UID: b.UID, Controller: ownerBoolPtr(true)}}}, Status: &snapshotv1.VolumeSnapshotStatus{ReadyToUse: ownerBoolPtr(true)}} + switch mode { + case "pending": + b.Status.Phase = gameplanev1alpha1.BackupPhasePending + case "running": + b.Status.Phase = gameplanev1alpha1.BackupPhaseRunning + case "failed": + b.Status.Phase = gameplanev1alpha1.BackupPhaseFailed + case "abort-before-status": + b.Annotations[backupSnapshotAbortAnnotation] = "true" + case "copy-aborted": + b.Annotations[backupCopyAbortAnnotation] = "cancelled" + case "snapshot-aborted": + vs.Annotations = map[string]string{backupSnapshotAbortAnnotation: "true"} + case "owner-replaced": + vs.OwnerReferences[0].UID = "previous-backup" + case "snapshot-replaced": + vs.UID = "replacement-snapshot" + case "not-ready": + vs.Status = nil + case "external": + vs.OwnerReferences = nil + vs.Status = nil + } + r, _ := guardFixture(t, b) + if err := r.Create(t.Context(), vs); err != nil { + t.Fatal(err) + } + gs := &gameplanev1alpha1.GameServer{ObjectMeta: metav1.ObjectMeta{Name: "restored", Namespace: b.Namespace, UID: "restored-uid"}, Spec: gameplanev1alpha1.GameServerSpec{Storage: &gameplanev1alpha1.GameStorageSpec{DataSource: &gameplanev1alpha1.GameDataSource{Kind: "VolumeSnapshot", Name: b.Name}}}} + gr := &GameServerReconciler{Client: r.Client, APIReader: r.Client, Scheme: r.Scheme} + err = gr.reconcilePVC(t.Context(), gs, &gameplanev1alpha1.GameTemplate{}) + allowed := mode == "valid" || mode == "external" + if (err == nil) != allowed { + t.Fatalf("PVC source validation mode=%s err=%v", mode, err) + } + var pvc corev1.PersistentVolumeClaim + getErr := r.Get(t.Context(), client.ObjectKey{Namespace: b.Namespace, Name: "restored-data"}, &pvc) + if !allowed && !apierrors.IsNotFound(getErr) { + t.Fatalf("invalid source created PVC: %v", getErr) + } + }) + } +} diff --git a/operator/internal/controller/backup_stopped_test.go b/operator/internal/controller/backup_stopped_test.go index eb07407b8..914752522 100644 --- a/operator/internal/controller/backup_stopped_test.go +++ b/operator/internal/controller/backup_stopped_test.go @@ -200,8 +200,12 @@ func TestBackupLiveTimeoutRetainsOwnerUntilOriginalPodGone(t *testing.T) { t.Fatalf("timeout did not fail closed: %+v", first.Status) } var job batchv1.Job - if err := r.Get(t.Context(), client.ObjectKey{Namespace: "ns", Name: "first"}, &job); !apierrors.IsNotFound(err) { - t.Fatal("live timeout created copy") + var gate corev1.ConfigMap + if err := r.Get(t.Context(), client.ObjectKey{Namespace: "ns", Name: backupStartConfigMapName(first)}, &gate); err != nil { + t.Fatal(err) + } + if gate.Data[backupStartTokenKey] != "" { + t.Fatal("live timeout authorized prepared worker") } var gs gameplanev1alpha1.GameServer if err := r.Get(t.Context(), client.ObjectKey{Namespace: "ns", Name: "server"}, &gs); err != nil { diff --git a/operator/internal/controller/backup_volumesnapshot.go b/operator/internal/controller/backup_volumesnapshot.go index e90a0b6c9..4eea866d1 100644 --- a/operator/internal/controller/backup_volumesnapshot.go +++ b/operator/internal/controller/backup_volumesnapshot.go @@ -2,11 +2,13 @@ package controller import ( "context" + "errors" "fmt" "time" snapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" @@ -29,7 +31,7 @@ func (r *BackupReconciler) reconcileVolumeSnapshot( ctx context.Context, b *gameplanev1alpha1.Backup, gs *gameplanev1alpha1.GameServer, ) (ctrl.Result, error) { if b.Annotations[backupSnapshotAbortAnnotation] == "true" { - return r.failVolumeSnapshot(ctx, b, "volume snapshot aborted after an API or creation failure") + return r.abortVolumeSnapshot(ctx, b, "volume snapshot capture was aborted") } // An uncached read verifies that the snapshot API is served before // disabling autosave. Missing CRDs (including NoMatch) and authorization @@ -38,22 +40,43 @@ func (r *BackupReconciler) reconcileVolumeSnapshot( if err := r.backupAPIReader().Get(ctx, types.NamespacedName{Namespace: b.Namespace, Name: b.Name}, &existing); err != nil && !apierrors.IsNotFound(err) { return r.abortVolumeSnapshot(ctx, b, fmt.Sprintf("volume snapshot API unavailable: %v", err)) } + if existing.Name != "" { + if err := r.rememberBackupSnapshot(ctx, b, &existing); err != nil { + return r.abortVolumeSnapshot(ctx, b, err.Error()) + } + } + if !legacyBackupOwesUnquiesce(b) { + if res, err := r.prepareBackupWorkload(ctx, b); err != nil || res.RequeueAfter > 0 { + return res, err + } + } + deadline, err := r.backupDeadline(ctx, b, backupCSIDeadlineAnnotation, backupTimeout(r.CSICaptureTimeout, DefaultBackupCSICaptureTimeout)) + if err != nil { + return r.abortVolumeSnapshot(ctx, b, err.Error()) + } + if !time.Now().Before(deadline) { + return r.abortVolumeSnapshot(ctx, b, "CSI capture deadline exceeded; incomplete snapshot invalidated") + } if legacyBackupOwesUnquiesce(b) && metav1.IsControlledBy(&existing, b) { return r.observeVolumeSnapshot(ctx, b, &existing) } - // Quiesce first (idempotent across requeues). A CSI snapshot is only - // crash-consistent; flushing the game to disk first (RCON save-all, - // etc.) makes it application-consistent. No-op when the backup opts out - // or no agent mTLS is configured. + // The durable capture budget starts before any save-off RPC. Quiesce + // confirmation describes the agent action, not universal game consistency. if res, err := r.prepareBackupQuiesce(ctx, b); err != nil || res.RequeueAfter > 0 { return res, err } + if !time.Now().Before(deadline) { + return r.abortVolumeSnapshot(ctx, b, "CSI capture deadline exceeded during quiesce") + } pvcName := gs.Name + "-data" vs := &snapshotv1.VolumeSnapshot{ ObjectMeta: metav1.ObjectMeta{Name: b.Name, Namespace: b.Namespace}, } if _, err := controllerutil.CreateOrUpdate(ctx, r.Client, vs, func() error { + if !vs.CreationTimestamp.IsZero() && !metav1.IsControlledBy(vs, b) { + return fmt.Errorf("VolumeSnapshot is not owned by this Backup") + } // Source + class are immutable on a VolumeSnapshot, so only set // them on first creation; SetControllerReference is idempotent. if vs.CreationTimestamp.IsZero() { @@ -71,16 +94,28 @@ func (r *BackupReconciler) reconcileVolumeSnapshot( // independently until save-on is confirmed. return r.abortVolumeSnapshot(ctx, b, fmt.Sprintf("volume snapshot creation failed: %v", err)) } + if err := r.rememberBackupSnapshot(ctx, b, vs); err != nil { + return r.abortVolumeSnapshot(ctx, b, err.Error()) + } return r.observeVolumeSnapshot(ctx, b, vs) } func (r *BackupReconciler) observeVolumeSnapshot(ctx context.Context, b *gameplanev1alpha1.Backup, vs *snapshotv1.VolumeSnapshot) (ctrl.Result, error) { + if b.Annotations[backupSnapshotAbortAnnotation] == "true" { + return r.abortVolumeSnapshot(ctx, b, "volume snapshot capture was aborted") + } + if value := b.Annotations[backupCSIDeadlineAnnotation]; value != "" { + deadline, err := time.Parse(time.RFC3339Nano, value) + if err != nil || !time.Now().Before(deadline) { + return r.abortVolumeSnapshot(ctx, b, "CSI capture deadline exceeded; late readiness is invalid") + } + } switch { case vs.Status != nil && vs.Status.Error != nil && vs.Status.Error.Message != nil: // The CSI driver rejected the snapshot. Surface it and unquiesce so // the game doesn't stay frozen. - return r.failVolumeSnapshot(ctx, b, fmt.Sprintf("volume snapshot failed: %s", *vs.Status.Error.Message)) + return r.abortVolumeSnapshot(ctx, b, fmt.Sprintf("volume snapshot failed: %s", *vs.Status.Error.Message)) case vs.Status != nil && vs.Status.ReadyToUse != nil && *vs.Status.ReadyToUse: return r.completeVolumeSnapshot(ctx, b, vs) default: @@ -93,11 +128,15 @@ func (r *BackupReconciler) observeVolumeSnapshot(ctx context.Context, b *gamepla // created a snapshot. Persist that abort decision before terminal cleanup so // worker-drain checks cannot trap this failed/nonrestorable Backup forever. func (r *BackupReconciler) abortVolumeSnapshot(ctx context.Context, b *gameplanev1alpha1.Backup, msg string) (ctrl.Result, error) { - patchBackupAnnotations(b, map[string]string{backupSnapshotAbortAnnotation: "true"}) - if err := r.Update(ctx, b); err != nil { + if err := r.recordSnapshotAbort(ctx, b, msg); err != nil { return ctrl.Result{}, err } - return r.failVolumeSnapshot(ctx, b, msg) + unquiesce, unquiesceErr := r.runUnquiesce(ctx, b) + cleanup, cleanupErr := r.cleanupAbortedSnapshot(ctx, b) + if unquiesce.RequeueAfter > 0 && (cleanup.RequeueAfter == 0 || unquiesce.RequeueAfter < cleanup.RequeueAfter) { + cleanup = unquiesce + } + return cleanup, errors.Join(unquiesceErr, cleanupErr) } // markVolumeSnapshotRunning flips the Backup to Running on first observation @@ -124,6 +163,9 @@ func (r *BackupReconciler) markVolumeSnapshotRunning( return ctrl.Result{}, err } } + if err := r.setBackupProgress(ctx, b, "SnapshotCapturePending", "waiting for CSI capture; deadline "+b.Annotations[backupCSIDeadlineAnnotation]); err != nil { + return ctrl.Result{}, err + } return ctrl.Result{RequeueAfter: 10 * time.Second}, nil } @@ -132,9 +174,13 @@ func (r *BackupReconciler) markVolumeSnapshotRunning( func (r *BackupReconciler) completeVolumeSnapshot( ctx context.Context, b *gameplanev1alpha1.Backup, vs *snapshotv1.VolumeSnapshot, ) (ctrl.Result, error) { + if b.Annotations[backupSnapshotAbortAnnotation] == "true" { + return r.abortVolumeSnapshot(ctx, b, "volume snapshot capture was aborted") + } if b.Status.Phase != gameplanev1alpha1.BackupPhaseSucceeded { now := metav1.Now() b.Status.Phase = gameplanev1alpha1.BackupPhaseSucceeded + meta.RemoveStatusCondition(&b.Status.Conditions, "CaptureReady") b.Status.ObservedGeneration = b.Generation b.Status.SnapshotID = vs.Name if vs.Status.BoundVolumeSnapshotContentName != nil { @@ -167,14 +213,3 @@ func (r *BackupReconciler) completeVolumeSnapshot( } return r.runUnquiesce(ctx, b) } - -// failVolumeSnapshot marks the Backup Failed and still runs the terminal -// unquiesce so a quiesced game doesn't stay frozen after a snapshot error. -func (r *BackupReconciler) failVolumeSnapshot( - ctx context.Context, b *gameplanev1alpha1.Backup, msg string, -) (ctrl.Result, error) { - if _, err := r.fail(ctx, b, msg); err != nil { - return ctrl.Result{}, err - } - return r.runUnquiesce(ctx, b) -} diff --git a/operator/internal/controller/backup_volumesnapshot_envtest_test.go b/operator/internal/controller/backup_volumesnapshot_envtest_test.go index 6e8bb7e01..d182eb696 100644 --- a/operator/internal/controller/backup_volumesnapshot_envtest_test.go +++ b/operator/internal/controller/backup_volumesnapshot_envtest_test.go @@ -9,8 +9,10 @@ import ( "testing" "time" + snapshotv1 "github.com/kubernetes-csi/external-snapshotter/client/v8/apis/volumesnapshot/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/types" gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" @@ -210,6 +212,10 @@ func TestRestore_VolumeSnapshotRejectsExistingTarget(t *testing.T) { func TestGameServer_DataSourceSeedsPVC(t *testing.T) { ns := newNamespace(t) startMgr(t, ns, withGameServerReconciler(t, ns)) + sourcePVC := "external-source-data" + if err := k8sClient.Create(context.Background(), &snapshotv1.VolumeSnapshot{ObjectMeta: metav1.ObjectMeta{Name: "snap-1", Namespace: ns}, Spec: snapshotv1.VolumeSnapshotSpec{Source: snapshotv1.VolumeSnapshotSource{PersistentVolumeClaimName: &sourcePVC}}}); err != nil { + t.Fatal(err) + } tmpl := buildGameTemplate(uniqueName("minecraft")) if err := k8sClient.Create(context.Background(), tmpl); err != nil { diff --git a/operator/internal/controller/backup_worker.go b/operator/internal/controller/backup_worker.go new file mode 100644 index 000000000..61d55055a --- /dev/null +++ b/operator/internal/controller/backup_worker.go @@ -0,0 +1,358 @@ +package controller + +import ( + "context" + "crypto/sha256" + "encoding/json" + "errors" + "fmt" + "strconv" + "time" + + appsv1 "k8s.io/api/apps/v1" + batchv1 "k8s.io/api/batch/v1" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + + gameplanev1alpha1 "github.com/GameplanePanel/gameplane/operator/api/v1alpha1" +) + +const ( + backupWorkerSourceAnnotation = "backup.gameplane.local/worker-source" + backupAuthorizedWorkerAnnotation = "backup.gameplane.local/authorized-worker" + backupStartTokenKey = "start" + backupWorkerProtocolAnnotation = "backup.gameplane.local/worker-protocol" +) + +// The waiting worker never reads source files until the operator authorizes +// this exact Backup/pod/source generation. Repository preparation takes a +// shared restic lock; later lock contention is bounded by the copy timeout. +// Values and tags are arguments/environment, never interpolated shell code. +const resticWorkerScript = `set -eu +for utility in timeout date cat touch sleep restic; do command -v "$utility" >/dev/null; done +timeout -s TERM -k 5 1 /bin/sh -c ':' +test -d /data && test -r /data && test -x /data +restic snapshots --json --retry-lock 10m >/dev/null +touch /tmp/prepared +while :; do + now=$(date +%s) + if [ -r /start/start ]; then + token=$(cat /start/start) + if [ -n "$token" ]; then + read -r version backup pod source expires extra < 0 { + return res, err + } + return r.mirrorJobStatus(ctx, b, &job) + } + // Queueing behind another lifecycle owner is not worker preparation and + // must not consume its budget before this Backup can start preparing. + if b.Annotations[backupPreparationDeadlineAnnotation] == "" { + claimed, err := r.claimBackupTarget(ctx, b) + if err != nil { + if errors.Is(err, errLifecycleGenerationChanged) { + return r.abortResticCopy(ctx, b, err.Error()) + } + return ctrl.Result{}, err + } + if !claimed { + return ctrl.Result{RequeueAfter: 10 * time.Second}, nil + } + } + preparation, err := r.backupDeadline(ctx, b, backupPreparationDeadlineAnnotation, backupTimeout(r.PreparationTimeout, DefaultBackupPreparationTimeout)) + if err != nil { + return r.abortResticCopy(ctx, b, err.Error()) + } + if b.Annotations[backupAuthorizedWorkerAnnotation] == "" && time.Now().After(preparation) { + return r.abortResticCopy(ctx, b, "worker preparation deadline exceeded before copy authorization") + } + if value := b.Annotations[backupQuiesceDeadlineAnnotation]; value != "" { + deadline, err := time.Parse(time.RFC3339Nano, value) + if err != nil { + return r.abortResticCopy(ctx, b, "invalid persisted quiesce deadline") + } + finishedInTime := job.Status.Succeeded > 0 && job.Status.CompletionTime != nil && !job.Status.CompletionTime.After(deadline) + if time.Now().After(deadline) && !finishedInTime { + return r.abortResticCopy(ctx, b, "maximum quiesce duration exceeded; copy invalidated") + } + } + if res, err := r.prepareBackupWorkload(ctx, b); err != nil || res.RequeueAfter > 0 { + return res, err + } + // Admission refreshes the target. Read its effective content only after + // owning the workload fence, so a rollout completed before admission + // cannot leave the worker using an earlier filesystem layout. + bound, err := r.ownedBackupTarget(ctx, b) + if err != nil { + return ctrl.Result{}, err + } + if bound == nil { + return ctrl.Result{RequeueAfter: 5 * time.Second}, nil + } + tmpl, err := effectiveGameTemplate(ctx, r.backupAPIReader(), bound) + if err != nil && (bound.Annotations[effectiveTemplateAnnotation] != "" || !apierrors.IsNotFound(err)) { + return ctrl.Result{}, err + } + source, err := r.prepareBackupWorkerSource(ctx, b) + if err != nil { + return r.abortResticCopy(ctx, b, err.Error()) + } + if source == nil { + if err := r.setBackupProgress(ctx, b, "WorkerPreparing", "waiting for the source workload to be created and scheduled"); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: 5 * time.Second}, nil + } + gate := &corev1.ConfigMap{ObjectMeta: metav1.ObjectMeta{Name: backupStartConfigMapName(b), Namespace: b.Namespace}} + if _, err := controllerutil.CreateOrUpdate(ctx, r.Client, gate, func() error { + if !gate.CreationTimestamp.IsZero() && !metav1.IsControlledBy(gate, b) { + return fmt.Errorf("copy authorization ConfigMap is not owned by this Backup") + } + if gate.Data == nil { + gate.Data = map[string]string{backupStartTokenKey: ""} + } + return controllerutil.SetControllerReference(b, gate, r.Scheme) + }); err != nil { + return ctrl.Result{}, err + } + if job.Name == "" { + backoff, deadline := resolveJobLimits(r.JobBackoffLimit, r.JobActiveDeadlineSeconds) + job = batchv1.Job{ObjectMeta: metav1.ObjectMeta{Name: b.Name, Namespace: b.Namespace, Annotations: map[string]string{backupWorkerProtocolAnnotation: "v1"}}, Spec: batchv1.JobSpec{BackoffLimit: &backoff, ActiveDeadlineSeconds: &deadline, + Template: corev1.PodTemplateSpec{ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{backupRestoreJobLabel: backupRestoreJobValue, backupUIDLabel: string(b.UID)}}, Spec: r.buildPreparedBackupPodSpec(b, tmpl, source, preparation)}}} + job.Spec.Template.Spec.RestartPolicy = corev1.RestartPolicyNever + if err := controllerutil.SetControllerReference(b, &job, r.Scheme); err != nil { + return ctrl.Result{}, err + } + if err := r.Create(ctx, &job); err != nil { + return ctrl.Result{}, err + } + } + if jobPermanentlyFailed(&job) { + return r.abortResticCopy(ctx, b, "backup job reported Failed") + } + if job.Status.Succeeded > 0 { + if b.Annotations[backupAuthorizedWorkerAnnotation] == "" { + return r.abortResticCopy(ctx, b, "backup worker completed without copy authorization") + } + return r.mirrorJobStatus(ctx, b, &job) + } + worker, err := r.preparedBackupWorker(ctx, b, &job) + if err != nil { + return ctrl.Result{}, err + } + if worker == nil { + if err := r.setBackupProgress(ctx, b, "WorkerPreparing", "waiting for a scheduled worker with mounted source and validated repository"); err != nil { + return ctrl.Result{}, err + } + return ctrl.Result{RequeueAfter: 2 * time.Second}, nil + } + // Raw and already-stopped copies have no save-off window. Their tokens use + // the full Job deadline, preserving long-copy support independently. + _, copySeconds := resolveJobLimits(r.JobBackoffLimit, r.JobActiveDeadlineSeconds) + if job.Spec.ActiveDeadlineSeconds != nil && *job.Spec.ActiveDeadlineSeconds > 0 { + copySeconds = *job.Spec.ActiveDeadlineSeconds + } + const maxDurationSeconds = int64((1<<63 - 1) / time.Second) + if copySeconds > maxDurationSeconds { + copySeconds = maxDurationSeconds + } + copyBudget := time.Duration(copySeconds) * time.Second + protected := b.Spec.Quiesce && agentQuiesceAvailable(r.AgentClient) + if prepared, err := parseBackupPreparation(b); err != nil { + return ctrl.Result{}, err + } else if prepared != nil && prepared.Outcome == "already-stopped" { + protected = false + } + deadlineKey := backupQuiesceDeadlineAnnotation + if protected { + copyBudget = backupTimeout(r.QuiesceTimeout, DefaultBackupQuiesceTimeout) + } + deadline, err := r.backupDeadline(ctx, b, deadlineKey, copyBudget) + if err != nil { + return ctrl.Result{}, err + } + if res, err := r.prepareBackupQuiesce(ctx, b); err != nil || res.RequeueAfter > 0 { + return res, err + } + if _, err := r.prepareBackupWorkerSource(ctx, b); err != nil { + return r.abortResticCopy(ctx, b, err.Error()) + } + // Re-read readiness after the RPC. A replacement worker cannot inherit + // the first pod's authorization, including during a Job retry. + current, err := r.preparedBackupWorker(ctx, b, &job) + if err != nil { + return ctrl.Result{}, err + } + if current == nil || current.UID != worker.UID { + return ctrl.Result{RequeueAfter: 2 * time.Second}, nil + } + if !time.Now().Before(deadline) { + return r.abortResticCopy(ctx, b, "maximum quiesce duration exceeded before start authorization") + } + if b.Annotations[backupAuthorizedWorkerAnnotation] != string(worker.UID) { + patchBackupAnnotations(b, map[string]string{backupAuthorizedWorkerAnnotation: string(worker.UID)}) + if err := r.Update(ctx, b); err != nil { + return ctrl.Result{}, err + } + } + token := fmt.Sprintf("v1 %s %s %s %d\n", b.UID, worker.UID, backupWorkerSourceID(b), deadline.Unix()) + if gate.Data[backupStartTokenKey] != token { + gate.Data[backupStartTokenKey] = token + if err := r.Update(ctx, gate); err != nil { + return ctrl.Result{}, err + } + } + if err := r.setBackupProgress(ctx, b, "WorkerAuthorized", "prepared worker authorized; copy deadline "+deadline.UTC().Format(time.RFC3339)); err != nil { + return ctrl.Result{}, err + } + return r.mirrorJobStatus(ctx, b, &job) +} diff --git a/operator/internal/controller/gameserver_controller.go b/operator/internal/controller/gameserver_controller.go index 7f3115ea0..1ef3a0148 100644 --- a/operator/internal/controller/gameserver_controller.go +++ b/operator/internal/controller/gameserver_controller.go @@ -608,6 +608,13 @@ func (r *GameServerReconciler) reconcilePVC( // size is >= the snapshot's restoreSize by construction (the // snapshot came from a PVC of that size). if gs.Spec.Storage != nil && gs.Spec.Storage.DataSource != nil { + reader := r.APIReader + if reader == nil { + reader = r.Client + } + if err := validateBackupSnapshotSource(ctx, reader, gs.Namespace, gs.Spec.Storage.DataSource.Name); err != nil { + return err + } apiGroup := "snapshot.storage.k8s.io" pvc.Spec.DataSource = &corev1.TypedLocalObjectReference{ APIGroup: &apiGroup, diff --git a/operator/internal/controller/helpers_envtest_test.go b/operator/internal/controller/helpers_envtest_test.go index 8069878a8..0a18108ad 100644 --- a/operator/internal/controller/helpers_envtest_test.go +++ b/operator/internal/controller/helpers_envtest_test.go @@ -468,6 +468,25 @@ func patchJobStatus(t *testing.T, ns, name string, mut func(s *batchv1.JobStatus t.Fatalf("get job %s/%s: %v", ns, name, err) } mut(&job.Status) + if job.Annotations[backupWorkerProtocolAnnotation] == "v1" { + if job.Status.Succeeded > 0 || job.Status.Active > 0 { + prepareEnvtestBackupWorker(t, &job) + } + if job.Status.Succeeded > 0 || job.Status.Failed > 0 { + var worker corev1.Pod + if err := k8sClient.Get(context.Background(), client.ObjectKey{Namespace: ns, Name: name + "-test-worker"}, &worker); err == nil { + worker.Status.Phase = corev1.PodSucceeded + if job.Status.Failed > 0 { + worker.Status.Phase = corev1.PodFailed + } + if err := k8sClient.Status().Update(context.Background(), &worker); err != nil { + t.Fatal(err) + } + } else if !apierrors.IsNotFound(err) { + t.Fatal(err) + } + } + } now := metav1.Now() @@ -494,6 +513,34 @@ func patchJobStatus(t *testing.T, ns, name string, mut func(s *batchv1.JobStatus } } +// envtest has neither Job controller nor kubelet. This models their actual +// readiness contract and waits for operator authorization before reporting a +// successful copy. It never grants a token itself. +func prepareEnvtestBackupWorker(t *testing.T, job *batchv1.Job) { + t.Helper() + ctx := context.Background() + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: job.Name + "-test-worker", Namespace: job.Namespace, Labels: job.Spec.Template.Labels, + OwnerReferences: []metav1.OwnerReference{{APIVersion: "batch/v1", Kind: "Job", Name: job.Name, UID: job.UID, Controller: ownerBoolPtr(true)}}}, Spec: job.Spec.Template.Spec} + pod.Spec.NodeName = "test-node" + if err := k8sClient.Create(ctx, pod); err != nil { + if !apierrors.IsAlreadyExists(err) { + t.Fatal(err) + } + if err := k8sClient.Get(ctx, client.ObjectKeyFromObject(pod), pod); err != nil { + t.Fatal(err) + } + } + pod.Status.Phase = corev1.PodRunning + pod.Status.Conditions = []corev1.PodCondition{{Type: corev1.PodReady, Status: corev1.ConditionTrue}} + if err := k8sClient.Status().Update(ctx, pod); err != nil { + t.Fatal(err) + } + eventually(t, func() (bool, string) { + backup := getBackup(t, job.Namespace, job.Name) + return backup.Annotations[backupAuthorizedWorkerAnnotation] == string(pod.UID), "prepared worker is not yet authorized: " + describeBackupStatus(backup) + }) +} + // upsertJobCondition sets/replaces the JobCondition of type t to // Status: True in conds, returning the updated slice. Unlike // upsertCondition (helpers.go), which operates on the CRD condition @@ -838,4 +885,15 @@ func seedBackupWorkload(t *testing.T, gs *gameplanev1alpha1.GameServer) { if err := k8sClient.Status().Update(context.Background(), ss); err != nil { t.Fatalf("observe workload: %v", err) } + // Envtest has no StatefulSet controller or scheduler. Model their initial + // pod placement before a worker binds its immutable source identity. + pod := &corev1.Pod{ObjectMeta: metav1.ObjectMeta{Name: name + "-0", Namespace: ns, OwnerReferences: []metav1.OwnerReference{{APIVersion: "apps/v1", Kind: "StatefulSet", Name: name, UID: ss.UID, Controller: ownerBoolPtr(true)}}}, + Spec: corev1.PodSpec{NodeName: "fixture-node", Containers: []corev1.Container{{Name: "game", Image: "test/game:fixture"}}}} + if err := k8sClient.Create(context.Background(), pod); err != nil { + t.Fatalf("seed scheduled source pod: %v", err) + } + pod.Status.Phase = corev1.PodRunning + if err := k8sClient.Status().Update(context.Background(), pod); err != nil { + t.Fatalf("observe running source pod: %v", err) + } } diff --git a/operator/internal/controller/restore_controller.go b/operator/internal/controller/restore_controller.go index e979ee712..41839e2a6 100644 --- a/operator/internal/controller/restore_controller.go +++ b/operator/internal/controller/restore_controller.go @@ -80,13 +80,13 @@ func (r *RestoreReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct // the rug out from under us mid-restore. if rs.Status.SnapshotID == "" { var src gameplanev1alpha1.Backup - if err := r.Get(ctx, types.NamespacedName{Name: rs.Spec.BackupRef.Name, Namespace: rs.Namespace}, &src); err != nil { + if err := r.apiReader().Get(ctx, types.NamespacedName{Name: rs.Spec.BackupRef.Name, Namespace: rs.Namespace}, &src); err != nil { if apierrors.IsNotFound(err) { return r.fail(ctx, &rs, fmt.Sprintf("source backup %q not found", rs.Spec.BackupRef.Name)) } return ctrl.Result{}, err } - if src.Status.Phase == gameplanev1alpha1.BackupPhaseFailed { + if src.Status.Phase == gameplanev1alpha1.BackupPhaseFailed || src.Annotations[backupSnapshotAbortAnnotation] == "true" || src.Annotations[backupCopyAbortAnnotation] != "" { // Terminal: a Failed backup will never grow a snapshotID, so // waiting would leave the Restore Pending forever. return r.fail(ctx, &rs, fmt.Sprintf( @@ -125,7 +125,7 @@ func (r *RestoreReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct // touch an existing server — they provision a new one seeded from the // CSI snapshot — so they branch off here, before the suspend/Job flow. var src gameplanev1alpha1.Backup - if err := r.Get(ctx, types.NamespacedName{Name: rs.Spec.BackupRef.Name, Namespace: rs.Namespace}, &src); err != nil { + if err := r.apiReader().Get(ctx, types.NamespacedName{Name: rs.Spec.BackupRef.Name, Namespace: rs.Namespace}, &src); err != nil { if apierrors.IsNotFound(err) { return r.fail(ctx, &rs, fmt.Sprintf("source backup %q disappeared", rs.Spec.BackupRef.Name)) } @@ -135,6 +135,9 @@ func (r *RestoreReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct // preparation path checks that binding before awaiting startup, repairing // references and enforcing the deadline on every subsequent reconcile. if src.Spec.Strategy == "volume-snapshot" { + if src.Annotations[backupSnapshotAbortAnnotation] == "true" || src.Status.Phase != gameplanev1alpha1.BackupPhaseSucceeded || src.Status.SnapshotID != rs.Status.SnapshotID { + return r.fail(ctx, &rs, "source CSI capture is no longer valid") + } return r.reconcileVolumeSnapshotRestore(ctx, &rs, &src) } diff --git a/operator/specs.md b/operator/specs.md index d2c150b4c..fa67de76c 100644 --- a/operator/specs.md +++ b/operator/specs.md @@ -333,18 +333,20 @@ Primary reconcilers register with the manager in `cmd/main.go` and handle CRD li ### BackupReconciler - **Responsibility:** Drive Backup to completion: coordinate quiesce (if agent available), spawn restic Job, track snapshot ID. - **Key functions:** - - Quiesce orchestration: call agent Quiesce before Job, Unquiesce after completion. + - Restic preparation precedes quiesce: claim/fence the source, persist a preparation deadline, create the waiting worker, and require an uncached Running/Ready pod controlled by the exact Job UID. Before binding the first worker, a source StatefulSet requesting replicas must have an owned pod assigned to a node; initial creation/scheduling waits within the preparation budget, while a replacement after binding fails the copy. An intentionally stopped raw source remains eligible without a pod. The worker mounts the PVC, initializes/probes the repository with a shared lock, then waits for an owned ConfigMap start token. Only after readiness does the operator persist the quiesce deadline, perform the pod-bound RPC and source postcheck, record the capture manifest, and grant the Backup UID/worker UID/source-fingerprint/expiry-bound token. Replacement pods cannot reuse it. Workers have no service-account token, use the shared backup resource settings and source tolerations, and prefer the observed game node; hard same-node affinity is opt-in. + - Preparation defaults to 10m and live quiesce attempts to 5m (`--backup-preparation-timeout`, `--backup-quiesce-timeout`; matching Helm `operator.backupPreparationTimeout`, `operator.backupQuiesceTimeout`). Deadlines are annotations written once and survive restart/configuration changes, including when an attempted quiesce is unsupported. Copies requested raw, without an agent, or from an already-stopped source retain the full Job deadline. The waiting shell validates token expiry and bounds Restic with `timeout -s TERM -k 5`. On expiry/deletion, invalidation and token revocation precede foreground cancellation; actual live/terminating workers still block save-on and owner release. A timeout is not evidence of drainage. Existing pre-upgrade Jobs retain their observation protocol. - With quiesce and an agent client enabled, preparation is recorded once in `backup.gameplane.local/preparation`, bound to the target UID and StatefulSet UID/generation. `already-stopped` requires uncached proof of desired and observed zero replicas, caught-up observed generation, and no live game/PVC user pods or pending writer Jobs. Current copy workers are excluded by Job ownership. This path sends no agent calls and preserves suspend/idle/wake intent under the shared workload fence. It applies equally to restic and CSI copies. - Live preparation also pins the game Pod UID. The operator checks the agent's read-only `/v1/pods/{uid}/identity` endpoint before recording the first cleanup intent, then uses pod-bound quiesce/unquiesce routes; the agent rejects a different UID before RCON. A missing endpoint or identity mismatch fails closed without a legacy-route fallback. Ambiguous quiesce errors retain cleanup intent and the original Pod UID on both the Backup and shared owner. Only confirmed save-on or uncached evidence that the original pod is absent/replaced/terminal permits cleanup; an HTTP timeout, unready pod, or terminating pod is insufficient. Legacy attempts without a UID retire only on absence/terminal proof and never adopt a new UID. `Unquiesced=True, Reason=SourceProcessGone` distinguishes process exit from an actual save-on RPC. - Source proof is checked before and after quiesce and on each copy reconcile. Observed workload/process replacement fails the Backup before it can report a restorable success. Owner recovery retains the pod UID even if the Backup is force-deleted, drains predecessor workers, and never sends its cleanup to a successor pod. - Imported legacy recovery records and backups without a recorded Pod UID cannot call unbound save-on, even if an agent is available. Their cleanup intent remains durable until an uncached read proves the game Pod absent or terminal and not terminating; a live or unready replacement is not exit proof. - Upgrade the operator and agent image together and allow existing GameServer pods to roll out the new agent and `GAMEPLANE_POD_UID` DownwardAPI env. Older agents fail the identity preflight before cleanup debt, so the workload can continue its upgrade after terminal cleanup releases the owner. Retry failed copies after rollout; preflight never bypasses protection. This does not automatically stop a running game. - Every copy, including `quiesce=false` and CSI snapshots, acquires the shared lifecycle owner before starting. The Backup pins its target in `backup.gameplane.local/target`; `backup.gameplane.local/owner` remains a compatibility marker. The shared owner is described below. Quiesce cleanup intent is recorded on both the Backup and the surviving GameServer before calling the agent, including partial failures. Cleanup waits for live copy workers; deletion cancels the copy Job with foreground propagation first. Every Backup carries the cleanup finalizer. Pre-upgrade quiesced workers are observed through completion even when overlapping legacy copies cannot acquire the new exclusive owner. - - Orphan recovery preserves predecessor operation identities in the shared owner's `recovery` array, atomically with ownership transfer. Existing `backup.gameplane.local/recovering-owners` lineage is imported during migration. Deleting a recovering owner cannot hide an earlier worker. Normal deletion, orphan recovery, and legacy cleanup also wait for pending owned CSI captures to report ready or an explicit error; VolumeSnapshots are reclaimed by owner-reference GC after cleanup. - - VolumeSnapshot API availability is checked before quiescing. Snapshot creation errors fail the Backup and run unquiesce cleanup immediately instead of retrying creation indefinitely with saving disabled. API/creation failures persist `backup.gameplane.local/snapshot-aborted` to allow cleanup without waiting for a failed/nonrestorable capture. Failed unquiesce remains visible and retried. A transport failure can leave a CSI snapshot object despite a Failed Backup; this does not make that Backup restorable. + - Orphan recovery preserves predecessor operation identities in the shared owner's `recovery` array, atomically with ownership transfer. Existing `backup.gameplane.local/recovering-owners` lineage is imported during migration. Deleting a recovering owner cannot hide an earlier worker. Normal CSI observation waits for ready/error, but cancellation durably invalidates incomplete captures before save-on. Force-deleted Backup recovery marks the exact owned snapshot aborted and requests UID-bound deletion; a missing Backup cannot expose that snapshot as a valid Gameplane capture. + - CSI capture has a persisted deadline before quiesce, default 10m (`--backup-csi-capture-timeout`, Helm `operator.backupCSICaptureTimeout`), independent of Restic. API/creation errors, deadline expiry and deletion persist `backup.gameplane.local/snapshot-aborted` as a terminal decision, clear snapshot identity and fail the Backup before save-on. Late readiness cannot revive it. Snapshot/content identity is retained separately for reclamation. Cleanup verifies snapshot ownership and recorded UID, content reference UID/name/namespace and content UID, and deletes with UID preconditions. `Retain` or unverified resources remain for administrator action with names/UIDs in `SnapshotCleaned` conditions/events. Cleanup errors retry independently of save-on; deletion retains the Backup finalizer for pending reclamation but does not retain the game owner after confirmed save-on. Cluster RBAC adds content delete for verified `Delete`-policy resources. + - New PVCs referencing a Gameplane Backup-owned VolumeSnapshot require uncached successful-owner/snapshotID/UID/readiness checks and reject abort markers, including raw GameServer dataSource references. Existing PVCs do not depend on retained source metadata. External snapshots without a Gameplane Backup owner preserve the administrator direct-CRD contract; API authorization is separate. - Restic Job: mount data PVC, run restic backup to configured destination (S3, B2, rest-server, local fs). Job carries BackoffLimit and ActiveDeadlineSeconds from the reconciler's JobBackoffLimit / JobActiveDeadlineSeconds fields (operator flags --backup-job-backoff-limit / --backup-job-active-deadline-seconds, Helm operator.backupJobBackoffLimit / operator.backupJobActiveDeadlineSeconds; defaults 2 and 86400, applied when unset; spec 018 OD-027); only a Failed=True Job condition (set by jobPermanentlyFailed check) marks the Backup Failed. Pod SecurityContext takes FSGroup from the target GameTemplate's spec.security.fsGroup (fallback 65532 if template missing). - Snapshot tracking: parse restic JSON output from container logs to extract snapshot ID. - - Capture metadata: after ownership, workload fencing and quiesce admission, record `status.captureManifest` once before creating the first copy worker. It pins the GameServer UID, strategy, observed Pod/StatefulSet mount layout, PVC names/UIDs, included primary claim and excluded extra/mod/retained claims. `storage.extra[].backupRole` optionally declares `data` or `rebuildable`; absent declarations remain unclassified and excluded. Later template edits never rewrite the manifest. Existing workers and old terminal Backups have unknown historical coverage, never reconstructed from the current template. + - Capture metadata: after ownership, workload fencing and quiesce admission, record `status.captureManifest` once before authorizing the prepared copy worker or starting CSI capture. It pins the GameServer UID, strategy, observed Pod/StatefulSet mount layout, PVC names/UIDs, included primary claim and excluded extra/mod/retained claims. `storage.extra[].backupRole` optionally declares `data` or `rebuildable`; absent declarations remain unclassified and excluded. Later template edits never rewrite the manifest. Legacy workers and old terminal Backups have unknown historical coverage, never reconstructed from the current template. - Achieved consistency: the `Consistency` condition is separate from copy phase and volume coverage. It stays Unknown until a usable snapshot identity is recorded; failed copies report `CaptureFailed`. Agent-confirmed quiesce, already-stopped/drained sources, unsupported/no-agent raw copy and unquiesced CSI snapshots have distinct reasons. Quiesce depends on the module commands and does not imply that excluded volumes were captured. The API exposes the stored status and the backup drawer shows consistency and included/excluded claims. `StoppedSource` records an observation, without adding a stop/copy/restart policy or multi-volume backup. - Snapshot cleanup on delete (`backup_forget.go`): restic-strategy Backups carry `BackupSnapshotFinalizer` (`gameplane.local/backup-snapshot-finalizer`, added on first reconcile, also for pre-existing non-Failed Backups; not added for volume-snapshot or Failed Backups). `finalizeDelete` runs the unquiesce stage (`BackupFinalizer`) first, then `finalizeSnapshot`: it deletes the Backup's own still-running Job (foreground, awaited), waits while a non-terminal Restore pins the Backup or its snapshot id, then runs Job `-forget` (`restic unlock || true; restic forget --prune --retry-lock 10m`, backoff 3, deadline 1800s, 50m/128Mi requests and 512Mi memory limit, no PVC; the snapshot id is passed through env `FORGET_SNAPSHOT_ID` and must match `^[0-9a-f]{8,64}$`). Success sets `SnapshotForgotten=True` and a Normal `SnapshotForgotten` event; a Failed Job, a missing/incomplete repo Secret, a terminating namespace, or `maxSnapshotForgetWait` (45m, from annotation `backup.gameplane.local/forget-started-at`) sets `SnapshotForgotten=False` with a Warning `SnapshotForgetAbandoned` event and releases the Backup, leaving the snapshot in the repository. A Backup with no recorded snapshot id is released with `SnapshotForgetSkipped` when it may have written one. Backup Jobs run `restic backup /data --json --retry-lock 10m ...`. RBAC markers on `backup_controller.go` grant `backups` update/patch, `backups/status`, `backups/finalizers` and `jobs` create/update/patch/delete. Tests: `backup_forget_test.go`, `backup_forget_envtest_test.go`, `retention_terminating_test.go`; e2e `TestBackup_DeleteForgetsSnapshot` and the extended `TestBackupSchedule_RetentionTrimsPast`. Spec: `specs/done_020-backup-snapshot-forget/`. - Status: report phase (Pending/Running/Succeeded/Failed), error details. @@ -545,6 +547,10 @@ different copy under the old owner. See [operator operation recovery](../docs/op | `--agent-image-pull-policy` | string | `` | ImagePullPolicy override (Always/IfNotPresent/Never); empty leaves unset. | | `--config-init-image` | string | `controller.DefaultConfigInitImage` | Init container for rendering config files onto data volume. | | `--restic-image` | string | `controller.DefaultResticImage` | restic backup/restore Job image. | +| `--backup-preparation-timeout` | duration | `10m` | Maximum preparation before copy authorization. | +| `--backup-quiesce-timeout` | duration | `5m` | Live quiesced Restic copy budget; drainage precedes save-on. | +| `--backup-csi-capture-timeout` | duration | `10m` | CSI capture budget before terminal invalidation. | +| `--backup-hard-same-node` | bool | `false` | Require the observed game node instead of preferring it. | | `--backup-job-backoff-limit` | int64 | 2 | BackoffLimit of restic backup/restore Jobs; must be >= 0. | | `--backup-job-active-deadline-seconds` | int64 | 86400 | ActiveDeadlineSeconds of restic backup/restore Jobs; must be > 0. | | `--sentinel-image` | string | `controller.DefaultSentinelImage` | Wake-on-connect sentinel container image. | diff --git a/test/e2e/backup_preparation_e2e_test.go b/test/e2e/backup_preparation_e2e_test.go new file mode 100644 index 000000000..fd5ec305c --- /dev/null +++ b/test/e2e/backup_preparation_e2e_test.go @@ -0,0 +1,167 @@ +//go:build e2e + +package e2e + +import ( + "context" + "fmt" + "regexp" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +// Real scheduler, RWO PVC and repository failure/recovery on two schedulable +// nodes. Kind's local-path volume supplies node affinity; attachable CSI +// drivers still need their own multi-attach qualification. Zero API logins. +func TestBackup_PreparationAndSameNodeCopy(t *testing.T) { + t.Parallel() + ctx := t.Context() + nodes, err := envInstance.K8s.CoreV1().Nodes().List(ctx, metav1.ListOptions{}) + if err != nil { + t.Fatal(err) + } + eligible := 0 + for _, node := range nodes.Items { + ready, tainted := false, node.Spec.Unschedulable + for _, condition := range node.Status.Conditions { + ready = ready || (condition.Type == corev1.NodeReady && condition.Status == corev1.ConditionTrue) + } + for _, taint := range node.Spec.Taints { + tainted = tainted || taint.Effect == corev1.TaintEffectNoSchedule || taint.Effect == corev1.TaintEffectNoExecute + } + if ready && !tainted { + eligible++ + } + } + if eligible < 2 { + t.Fatal("backup preparation requires the management Kind profile with two schedulable workers") + } + suffix := fmt.Sprintf("%d", time.Now().UnixNano()) + ns, gs, template, name := "gameplane-games", "prepared-source-"+suffix, "prepared-template-"+suffix, "prepared-copy-"+suffix + ensureResticRepo(t) + applyBusyboxTemplate(t, template) + applyBusyboxGameServer(t, ns, gs, template) + waitPVCBound(t, ns, gs+"-data", 90*time.Second) + var sourceNode string + envInstance.Eventually(t, 2*time.Minute, func() (bool, string) { + pod, err := envInstance.K8s.CoreV1().Pods(ns).Get(ctx, gs+"-0", metav1.GetOptions{}) + if err != nil { + return false, err.Error() + } + for _, status := range pod.Status.ContainerStatuses { + if status.Name == "game" && status.Ready { + sourceNode = pod.Spec.NodeName + return sourceNode != "", "source has no node" + } + } + return false, "game container not ready" + }) + pvc, err := envInstance.K8s.CoreV1().PersistentVolumeClaims(ns).Get(ctx, gs+"-data", metav1.GetOptions{}) + if err != nil || len(pvc.Spec.AccessModes) != 1 || pvc.Spec.AccessModes[0] != corev1.ReadWriteOnce { + t.Fatalf("fixture is not RWO: %v %+v", err, pvc) + } + if out, err := envInstance.KubectlExec(t, ns, "pod/"+gs+"-0", "sh", "-c", "echo prepared-copy-marker > /data/prepared-marker.txt && sync"); err != nil { + t.Fatalf("write marker: %v %s", err, out) + } + repo, err := envInstance.K8s.CoreV1().Secrets(ns).Get(ctx, "e2e-restic-creds", metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + repo = repo.DeepCopy() + repo.ObjectMeta = metav1.ObjectMeta{Name: "prepared-repo-" + suffix, Namespace: ns} + // Authentication failure against the warm repository is deterministic. + // A refused HTTP endpoint exercises Restic's network retry budget and may + // keep the init container running longer than this preparation assertion. + validPassword := append([]byte(nil), repo.Data["password"]...) + repo.Data["password"] = []byte("incorrect-repository-password-" + suffix) + if _, err := envInstance.K8s.CoreV1().Secrets(ns).Create(ctx, repo, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + cleanupCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second) + defer cancel() + _ = envInstance.K8s.CoreV1().Secrets(ns).Delete(cleanupCtx, repo.Name, metav1.DeleteOptions{}) + }) + backup := &unstructured.Unstructured{Object: map[string]any{"apiVersion": "gameplane.local/v1alpha1", "kind": "Backup", "metadata": map[string]any{"name": name, "namespace": ns}, + "spec": map[string]any{"serverRef": map[string]any{"name": gs}, "repoRef": map[string]any{"name": repo.Name, "key": "repo"}, "quiesce": true}}} + if _, err := envInstance.Dyn.Resource(backupGVR).Namespace(ns).Create(ctx, backup, metav1.CreateOptions{}); err != nil { + t.Fatal(err) + } + t.Cleanup(func() { deleteBackupCR(t, ns, name) }) + var failedWorker *corev1.Pod + envInstance.Eventually(t, 2*time.Minute, func() (bool, string) { + got, err := envInstance.Dyn.Resource(backupGVR).Namespace(ns).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return false, err.Error() + } + if got.GetAnnotations()["backup.gameplane.local/quiesce-attempted"] != "" || got.GetAnnotations()["backup.gameplane.local/authorized-worker"] != "" { + t.Fatal("repository preparation failure disabled saving or authorized copy") + } + pods, err := envInstance.K8s.CoreV1().Pods(ns).List(ctx, metav1.ListOptions{LabelSelector: "job-name=" + name}) + if err != nil { + return false, err.Error() + } + for i := range pods.Items { + pod := &pods.Items[i] + if pod.Spec.NodeName != "" && pod.Spec.NodeName != sourceNode { + t.Fatalf("RWO worker scheduled away from source: %s vs %s", pod.Spec.NodeName, sourceNode) + } + for _, status := range pod.Status.InitContainerStatuses { + if status.Name == "restic-init" && ((status.State.Terminated != nil && status.State.Terminated.ExitCode != 0) || (status.LastTerminationState.Terminated != nil && status.LastTerminationState.Terminated.ExitCode != 0)) { + failedWorker = pod + return true, "" + } + } + } + return false, "waiting for actual repository preparation failure" + }) + repo, err = envInstance.K8s.CoreV1().Secrets(ns).Get(ctx, repo.Name, metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + repo.Data["password"] = validPassword + if _, err := envInstance.K8s.CoreV1().Secrets(ns).Update(ctx, repo, metav1.UpdateOptions{}); err != nil { + t.Fatal(err) + } + uid := failedWorker.UID + if err := envInstance.K8s.CoreV1().Pods(ns).Delete(ctx, failedWorker.Name, metav1.DeleteOptions{Preconditions: &metav1.Preconditions{UID: &uid}}); err != nil { + t.Fatal(err) + } + var snapshotID string + envInstance.Eventually(t, 5*time.Minute, func() (bool, string) { + got, err := envInstance.Dyn.Resource(backupGVR).Namespace(ns).Get(ctx, name, metav1.GetOptions{}) + if err != nil { + return false, err.Error() + } + phase, _, _ := unstructured.NestedString(got.Object, "status", "phase") + if phase == "Failed" { + t.Fatalf("copy failed after repository recovery: %+v", got.Object["status"]) + } + snapshotID, _, _ = unstructured.NestedString(got.Object, "status", "snapshotID") + return phase == "Succeeded" && snapshotID != "", "phase=" + phase + }) + pods, err := envInstance.K8s.CoreV1().Pods(ns).List(ctx, metav1.ListOptions{LabelSelector: "job-name=" + name}) + if err != nil { + t.Fatal(err) + } + completed := false + for _, pod := range pods.Items { + if pod.Status.Phase == corev1.PodSucceeded { + completed = true + if pod.Spec.NodeName != sourceNode || pod.Spec.AutomountServiceAccountToken == nil || *pod.Spec.AutomountServiceAccountToken { + t.Fatal("completed worker violated placement or credential boundary") + } + } + } + if !completed { + t.Fatal("no completed real restic worker") + } + if !regexp.MustCompile(`^[0-9a-f]{8,64}$`).MatchString(snapshotID) { + t.Fatalf("unexpected snapshot identity %q", snapshotID) + } + runResticJob(t, fmt.Sprintf("restic restore %s --target /tmp/prepared-restore --retry-lock 5m && test \"$(cat /tmp/prepared-restore/data/prepared-marker.txt)\" = prepared-copy-marker", snapshotID)) +} diff --git a/test/e2e/buckets.sh b/test/e2e/buckets.sh index ba9f800ec..ad448fa5d 100755 --- a/test/e2e/buckets.sh +++ b/test/e2e/buckets.sh @@ -428,10 +428,15 @@ TestManagementPlacementAndDisruption EOF } +bucket_backup_preparation() { cat <<'EOF' +TestBackup_PreparationAndSameNodeCopy +EOF +} + unbucketed() { :; } bucket_names() { - printf '%s\n' operator api-auth api-roles api-rbac api-agent api-mods ratelimit bot-fast bot-heavy multicluster upgrade postgres-availability telemetry capture-overhead optional-components restricted-workloads management-placement + printf '%s\n' operator api-auth api-roles api-rbac api-agent api-mods ratelimit bot-fast bot-heavy multicluster upgrade postgres-availability telemetry capture-overhead optional-components restricted-workloads management-placement backup-preparation } list_bucket() { @@ -453,6 +458,7 @@ list_bucket() { optional-components) bucket_optional_components ;; restricted-workloads) bucket_restricted_workloads ;; management-placement) bucket_management_placement ;; + backup-preparation) bucket_backup_preparation ;; *) echo "unknown bucket: $1 (known: $(bucket_names | tr '\n' ' '))" >&2 exit 2 diff --git a/test/e2e/ci_selection_e2e_test.go b/test/e2e/ci_selection_e2e_test.go index 02949e334..de2b09a51 100644 --- a/test/e2e/ci_selection_e2e_test.go +++ b/test/e2e/ci_selection_e2e_test.go @@ -22,7 +22,7 @@ import ( // ciSelectionBuckets mirrors E2E_BUCKETS in hack/ci_scope.py (the e2e-go matrix). var ciSelectionBuckets = []string{ "operator", "api-auth", "api-roles", "api-rbac", "api-agent", "api-mods", - "telemetry", "optional-components", "restricted-workloads", "management-placement", + "telemetry", "optional-components", "restricted-workloads", "management-placement", "backup-preparation", } // ciSelection is the decoded output of one `ci_scope.py e2e` run. diff --git a/test/e2e/failure_paths_e2e_test.go b/test/e2e/failure_paths_e2e_test.go index 6cf00a068..e497340a4 100644 --- a/test/e2e/failure_paths_e2e_test.go +++ b/test/e2e/failure_paths_e2e_test.go @@ -123,7 +123,8 @@ func TestBackup_FailsOnBadCredentials(t *testing.T) { } phase, _, _ := unstructured.NestedString(got.Object, "status", "phase") if phase != "Succeeded" { - return false, "seed phase=" + phase + message, _, _ := unstructured.NestedString(got.Object, "status", "message") + return false, "seed phase=" + phase + ": " + message } return true, "" }) diff --git a/test/e2e/specs.md b/test/e2e/specs.md index 9cd5bd94d..0fdf94e2c 100644 --- a/test/e2e/specs.md +++ b/test/e2e/specs.md @@ -84,6 +84,17 @@ a cordon, readiness of the surviving replica, and singleton API eviction. Node labels and cordons are restored in cleanup. This validates scheduler/PDB behavior, not application HA or storage migration. +### Backup preparation acceptance + +The `backup-preparation` bucket also uses the `management` Kind profile and zero +API logins. It verifies a real RWO copy on the game's node across two schedulable +workers, repository authentication failure before quiesce/authorization, retry after +repository recovery, token-gated Restic execution and restoration of a marker. +Kind local-path PV affinity supplies the storage placement constraint; this is +not an attachable CSI driver or multi-attach qualification. Unit/envtest regressions +cover stale worker/source identity, persisted budgets, delayed drainage, CSI +abort/reclamation, retained/replaced content and raw snapshot-source rejection. + ## Non-goals / boundaries - Does **not** replace the unit or envtest tiers for any other module — it exercises integration behavior, not the exhaustive branch coverage those tiers own (this module carries no `.testcoverage.yml` coverage gate).