From 7530a1ffd5a7e22501c9ee94696be1fe57a2ce01 Mon Sep 17 00:00:00 2001 From: Hemanth Nakkina Date: Mon, 17 Aug 2026 15:55:08 +0530 Subject: [PATCH 01/20] feat(clusterd): add advisory lock with fencing token for upgrade coordination Introduces an advisory lock in clusterd that serializes upgrade-related state mutations. The lock carries a monotonically increasing fencing token: every state write compare-and-swaps on the token, so a stale holder whose TTL expired cannot silently corrupt state after another process acquires the lock. Go side (sunbeam-microcluster): - upgrade_lock table (single row, seeded at schema apply) - AcquireUpgradeLock / RefreshUpgradeLock / ReleaseUpgradeLock / VerifyToken - HTTP endpoints: POST/PUT/DELETE /1.0/upgrade/lock, GET/PUT /1.0/upgrade/state, GET /1.0/upgrade/active - Stale-token writes return TokenMismatchError (HTTP 409) - Held-lock acquires return LockHeldError (HTTP 409) - Tests: monotonic token, stale-token rejection on verify/refresh, token preservation across release Python side (sunbeam-python): - UpgradeLockHeldException / UpgradeTokenMismatchException - HTTP 409 error translation in BaseService._request - ClusterService methods: acquire/refresh/release_upgrade_lock, get/update_upgrade_state, is_upgrade_active - AcquireUpgradeLockResponse pydantic model - Tests: acquire returns token, 409 surfaces as typed exceptions, CAS-guarded state write sends token + state --- sunbeam-microcluster/api/apitypes/upgrade.go | 41 ++++ sunbeam-microcluster/api/servers.go | 3 + sunbeam-microcluster/api/upgrade.go | 127 +++++++++++ sunbeam-microcluster/database/schema.go | 24 +- sunbeam-microcluster/database/upgrade_lock.go | 148 ++++++++++++ .../database/upgrade_lock_test.go | 211 ++++++++++++++++++ sunbeam-microcluster/sunbeam/upgrade.go | 129 +++++++++++ sunbeam-python/sunbeam/clusterd/cluster.py | 50 +++++ sunbeam-python/sunbeam/clusterd/models.py | 11 + sunbeam-python/sunbeam/clusterd/service.py | 18 ++ .../tests/unit/sunbeam/test_clusterd.py | 189 ++++++++++++++++ 11 files changed, 949 insertions(+), 2 deletions(-) create mode 100644 sunbeam-microcluster/api/apitypes/upgrade.go create mode 100644 sunbeam-microcluster/api/upgrade.go create mode 100644 sunbeam-microcluster/database/upgrade_lock.go create mode 100644 sunbeam-microcluster/database/upgrade_lock_test.go create mode 100644 sunbeam-microcluster/sunbeam/upgrade.go diff --git a/sunbeam-microcluster/api/apitypes/upgrade.go b/sunbeam-microcluster/api/apitypes/upgrade.go new file mode 100644 index 000000000..792fc853b --- /dev/null +++ b/sunbeam-microcluster/api/apitypes/upgrade.go @@ -0,0 +1,41 @@ +// Package apitypes provides shared types and structs. +package apitypes + +// AcquireUpgradeLockRequest is the body for POST /1.0/upgrade/lock. +type AcquireUpgradeLockRequest struct { + // HolderID identifies the process holding the lock (e.g. hostname + pid). + HolderID string `json:"holder_id" yaml:"holder_id"` +} + +// AcquireUpgradeLockResponse is returned by a successful lock acquire. +type AcquireUpgradeLockResponse struct { + // Token is the fencing token. Must be passed on every subsequent state + // write; clusterd rejects writes whose token ≠ the lock's current token. + Token int64 `json:"token" yaml:"token"` +} + +// RefreshUpgradeLockRequest is the body for PUT /1.0/upgrade/lock. +type RefreshUpgradeLockRequest struct { + // Token is the caller's fencing token, proving current ownership. + Token int64 `json:"token" yaml:"token"` +} + +// ReleaseUpgradeLockRequest is the body for DELETE /1.0/upgrade/lock. +type ReleaseUpgradeLockRequest struct { + // Token is the caller's fencing token, proving current ownership. + Token int64 `json:"token" yaml:"token"` +} + +// UpdateUpgradeStateRequest is the body for PUT /1.0/upgrade/state. +type UpdateUpgradeStateRequest struct { + // Token is the caller's fencing token. Must match the lock's current + // token or the write is rejected (database.TokenMismatchError). + Token int64 `json:"token" yaml:"token"` + // State is the JSON-encoded upgrade state blob (§6.1 of the spec). + State string `json:"state" yaml:"state"` +} + +// IsUpgradeActiveResponse is returned by GET /1.0/upgrade/active. +type IsUpgradeActiveResponse struct { + Active bool `json:"active" yaml:"active"` +} diff --git a/sunbeam-microcluster/api/servers.go b/sunbeam-microcluster/api/servers.go index 20a8c28f9..ab58c8f9a 100644 --- a/sunbeam-microcluster/api/servers.go +++ b/sunbeam-microcluster/api/servers.go @@ -33,6 +33,9 @@ var Servers = map[string]rest.Server{ storageBackendCmd, featureGatesCmd, featureGateCmd, + upgradeLockCmd, + upgradeStateCmd, + upgradeActiveCmd, }, }, { diff --git a/sunbeam-microcluster/api/upgrade.go b/sunbeam-microcluster/api/upgrade.go new file mode 100644 index 000000000..1ada0fc2e --- /dev/null +++ b/sunbeam-microcluster/api/upgrade.go @@ -0,0 +1,127 @@ +// Package api provides the REST API endpoints. +package api + +import ( + "encoding/json" + "errors" + "fmt" + "net/http" + + "github.com/canonical/lxd/lxd/response" + "github.com/canonical/lxd/shared/api" + "github.com/canonical/microcluster/v2/rest" + "github.com/canonical/microcluster/v2/state" + + "github.com/canonical/snap-openstack/sunbeam-microcluster/access" + "github.com/canonical/snap-openstack/sunbeam-microcluster/api/apitypes" + "github.com/canonical/snap-openstack/sunbeam-microcluster/database" + "github.com/canonical/snap-openstack/sunbeam-microcluster/sunbeam" +) + +// /1.0/upgrade/lock endpoint. +var upgradeLockCmd = rest.Endpoint{ + Path: "upgrade/lock", + + Post: access.ClusterCATrustedEndpoint(cmdUpgradeLockAcquire, true), + Put: access.ClusterCATrustedEndpoint(cmdUpgradeLockRefresh, true), + Delete: access.ClusterCATrustedEndpoint(cmdUpgradeLockRelease, true), +} + +// /1.0/upgrade/state endpoint. +var upgradeStateCmd = rest.Endpoint{ + Path: "upgrade/state", + + Get: access.ClusterCATrustedEndpoint(cmdUpgradeStateGet, true), + Put: access.ClusterCATrustedEndpoint(cmdUpgradeStatePut, true), +} + +// /1.0/upgrade/active endpoint. +var upgradeActiveCmd = rest.Endpoint{ + Path: "upgrade/active", + + Get: access.ClusterCATrustedEndpoint(cmdUpgradeActiveGet, true), +} + +func cmdUpgradeLockAcquire(s state.State, r *http.Request) response.Response { + var req apitypes.AcquireUpgradeLockRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + return response.BadRequest(err) + } + if req.HolderID == "" { + return response.BadRequest(fmt.Errorf("holder_id is required")) + } + token, err := sunbeam.AcquireUpgradeLock(r.Context(), s, req.HolderID) + if err != nil { + var held *database.LockHeldError + if api.StatusErrorCheck(err, http.StatusConflict) { + return response.Conflict(err) + } + if errors.As(err, &held) { + return response.Conflict(err) + } + return response.InternalError(err) + } + return response.SyncResponse(true, apitypes.AcquireUpgradeLockResponse{Token: token}) +} + +func cmdUpgradeLockRefresh(s state.State, r *http.Request) response.Response { + var req apitypes.RefreshUpgradeLockRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + return response.BadRequest(err) + } + if err := sunbeam.RefreshUpgradeLock(r.Context(), s, req.Token); err != nil { + return tokenErrorResponse(err) + } + return response.EmptySyncResponse +} + +func cmdUpgradeLockRelease(s state.State, r *http.Request) response.Response { + var req apitypes.ReleaseUpgradeLockRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + return response.BadRequest(err) + } + if err := sunbeam.ReleaseUpgradeLock(r.Context(), s, req.Token); err != nil { + return tokenErrorResponse(err) + } + return response.EmptySyncResponse +} + +func cmdUpgradeStateGet(s state.State, r *http.Request) response.Response { + stateJSON, err := sunbeam.GetUpgradeState(r.Context(), s) + if err != nil { + if api.StatusErrorCheck(err, http.StatusNotFound) { + return response.NotFound(err) + } + return response.InternalError(err) + } + return response.SyncResponse(true, stateJSON) +} + +func cmdUpgradeStatePut(s state.State, r *http.Request) response.Response { + var req apitypes.UpdateUpgradeStateRequest + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + return response.BadRequest(err) + } + if err := sunbeam.UpdateUpgradeState(r.Context(), s, req.Token, req.State); err != nil { + return tokenErrorResponse(err) + } + return response.EmptySyncResponse +} + +func cmdUpgradeActiveGet(s state.State, r *http.Request) response.Response { + active, err := sunbeam.IsUpgradeActive(r.Context(), s) + if err != nil { + return response.InternalError(err) + } + return response.SyncResponse(true, apitypes.IsUpgradeActiveResponse{Active: active}) +} + +// tokenErrorResponse maps a database.TokenMismatchError to HTTP 409 Conflict +// (stale fencing token — the caller's lock expired and was re-acquired). +func tokenErrorResponse(err error) response.Response { + var mismatch *database.TokenMismatchError + if errors.As(err, &mismatch) { + return response.Conflict(err) + } + return response.InternalError(err) +} diff --git a/sunbeam-microcluster/database/schema.go b/sunbeam-microcluster/database/schema.go index f539573f0..3b6428fc9 100644 --- a/sunbeam-microcluster/database/schema.go +++ b/sunbeam-microcluster/database/schema.go @@ -20,6 +20,7 @@ var SchemaExtensions = []schema.Update{ FeatureGatesSchemaUpdate, AddArchAndIsDPUToNodes, AddImageNameToNodes, + UpgradeLockSchemaUpdate, } // NodesSchemaUpdate is schema for table nodes @@ -155,8 +156,8 @@ func FeatureGatesSchemaUpdate(_ context.Context, tx *sql.Tx) error { stmt := ` CREATE TABLE feature_gates ( id INTEGER PRIMARY KEY AUTOINCREMENT NOT NULL, - gate_key TEXT NOT NULL, - enabled BOOLEAN NOT NULL DEFAULT 0, + gate_key TEXT NOT NULL, + enabled BOOLEAN NOT NULL DEFAULT 0, UNIQUE(gate_key) ); ` @@ -164,3 +165,22 @@ CREATE TABLE feature_gates ( _, err := tx.Exec(stmt) return err } + +// UpgradeLockSchemaUpdate creates the upgrade_lock table. Single-row table +// (id=1, seeded at schema-apply time) holding the advisory lock state for the +// release-upgrade coordinator. token is monotonically increasing — every +// acquire bumps it, so a stale holder's writes are detectable via CAS. +func UpgradeLockSchemaUpdate(_ context.Context, tx *sql.Tx) error { + stmt := ` +CREATE TABLE upgrade_lock ( + id INTEGER PRIMARY KEY NOT NULL, + token INTEGER NOT NULL DEFAULT 0, + holder_id TEXT NOT NULL DEFAULT '', + expires_at INTEGER NOT NULL DEFAULT 0 +); +INSERT INTO upgrade_lock (id, token, holder_id, expires_at) VALUES (1, 0, '', 0); + ` + + _, err := tx.Exec(stmt) + return err +} diff --git a/sunbeam-microcluster/database/upgrade_lock.go b/sunbeam-microcluster/database/upgrade_lock.go new file mode 100644 index 000000000..b57340502 --- /dev/null +++ b/sunbeam-microcluster/database/upgrade_lock.go @@ -0,0 +1,148 @@ +package database + +import ( + "context" + "database/sql" + "errors" + "fmt" + "time" + + "github.com/canonical/lxd/shared/api" +) + +// LockRow is the single row of upgrade_lock. token is monotonically +// increasing across acquires; holder_id identifies the current holder; +// expires_at is a unix timestamp (0 = no active holder). +type LockRow struct { + Token int64 + HolderID string + ExpiresAt int64 +} + +// LockHeldError is returned by AcquireUpgradeLock when another live holder +// owns the lock. caller_holder identifies the existing holder. +type LockHeldError struct { + HolderID string +} + +func (e *LockHeldError) Error() string { + return fmt.Sprintf("upgrade lock held by %q", e.HolderID) +} + +// TokenMismatchError is returned when a write carries a fencing token that +// does not match the lock's current token — i.e. the caller's lock has +// expired and been re-acquired by someone else. +type TokenMismatchError struct { + Expected int64 + Actual int64 +} + +func (e *TokenMismatchError) Error() string { + return fmt.Sprintf("fencing token mismatch: have %d, lock at %d", e.Expected, e.Actual) +} + +const lockRowID = 1 + +// GetUpgradeLockRow returns the single lock row. +func GetUpgradeLockRow(ctx context.Context, tx *sql.Tx) (LockRow, error) { + row := tx.QueryRowContext(ctx, + `SELECT token, holder_id, expires_at FROM upgrade_lock WHERE id = ?`, lockRowID) + + var r LockRow + err := row.Scan(&r.Token, &r.HolderID, &r.ExpiresAt) + if err != nil { + if errors.Is(err, sql.ErrNoRows) { + return LockRow{}, api.StatusErrorf(500, "upgrade_lock row missing — schema not applied") + } + return LockRow{}, fmt.Errorf("failed to read upgrade_lock: %w", err) + } + return r, nil +} + +// AcquireUpgradeLock claims the lock for holderID with the given TTL. Returns +// the new fencing token. If a live holder exists (expires_at > now), returns +// LockHeldError. Token is always current_token + 1 — monotonic across +// releases and expiries, so a stale holder's later writes are rejectable. +func AcquireUpgradeLock(ctx context.Context, tx *sql.Tx, holderID string, ttlSec int) (int64, error) { + now := time.Now().Unix() + expiresAt := now + int64(ttlSec) + + current, err := GetUpgradeLockRow(ctx, tx) + if err != nil { + return 0, err + } + + if current.ExpiresAt > now && current.HolderID != "" { + return 0, &LockHeldError{HolderID: current.HolderID} + } + + newToken := current.Token + 1 + res, err := tx.ExecContext(ctx, + `UPDATE upgrade_lock SET token = ?, holder_id = ?, expires_at = ? WHERE id = ?`, + newToken, holderID, expiresAt, lockRowID) + if err != nil { + return 0, fmt.Errorf("failed to acquire upgrade_lock: %w", err) + } + if n, _ := res.RowsAffected(); n != 1 { + return 0, fmt.Errorf("upgrade_lock update affected %d rows, expected 1", n) + } + + return newToken, nil +} + +// RefreshUpgradeLock extends the lock's TTL. Must be called by the holder +// before the old TTL expires, or the lock becomes acquirable by someone else. +// Returns TokenMismatchError if the caller's token is stale. +func RefreshUpgradeLock(ctx context.Context, tx *sql.Tx, token int64, ttlSec int) error { + now := time.Now().Unix() + expiresAt := now + int64(ttlSec) + + res, err := tx.ExecContext(ctx, + `UPDATE upgrade_lock SET expires_at = ? WHERE id = ? AND token = ?`, + expiresAt, lockRowID, token) + if err != nil { + return fmt.Errorf("failed to refresh upgrade_lock: %w", err) + } + n, _ := res.RowsAffected() + if n == 0 { + row, _ := GetUpgradeLockRow(ctx, tx) + return &TokenMismatchError{Expected: token, Actual: row.Token} + } + return nil +} + +// ReleaseUpgradeLock releases the lock. Returns TokenMismatchError if the +// caller's token is stale (someone else acquired after expiry). holder_id and +// expires_at are cleared; token is preserved for monotonicity. +func ReleaseUpgradeLock(ctx context.Context, tx *sql.Tx, token int64) error { + res, err := tx.ExecContext(ctx, + `UPDATE upgrade_lock SET holder_id = '', expires_at = 0 WHERE id = ? AND token = ?`, + lockRowID, token) + if err != nil { + return fmt.Errorf("failed to release upgrade_lock: %w", err) + } + n, _ := res.RowsAffected() + if n == 0 { + row, _ := GetUpgradeLockRow(ctx, tx) + return &TokenMismatchError{Expected: token, Actual: row.Token} + } + return nil +} + +// VerifyToken returns nil if the given token matches the lock's current token +// and the lock is live (not expired). Otherwise returns TokenMismatchError. +// Used as the CAS precondition for state writes. +func VerifyToken(ctx context.Context, tx *sql.Tx, token int64) error { + row, err := GetUpgradeLockRow(ctx, tx) + if err != nil { + return err + } + if row.Token != token { + return &TokenMismatchError{Expected: token, Actual: row.Token} + } + now := time.Now().Unix() + if row.ExpiresAt <= now { + return &TokenMismatchError{Expected: token, Actual: row.Token} + } + return nil +} diff --git a/sunbeam-microcluster/database/upgrade_lock_test.go b/sunbeam-microcluster/database/upgrade_lock_test.go new file mode 100644 index 000000000..25710bfac --- /dev/null +++ b/sunbeam-microcluster/database/upgrade_lock_test.go @@ -0,0 +1,211 @@ +package database + +import ( + "context" + "database/sql" + "errors" + "testing" + "time" + + _ "github.com/mattn/go-sqlite3" +) + +// newTestDB returns an in-memory SQLite with the upgrade_lock and config +// schemas applied. Each test gets a fresh DB (sqlite3 ":memory:" with +// ?cache=shared would leak across tests). +func newTestDB(t *testing.T) *sql.DB { + t.Helper() + db, err := sql.Open("sqlite3", ":memory:") + if err != nil { + t.Fatalf("open sqlite: %v", err) + } + ctx := context.Background() + for _, fn := range []func(context.Context, *sql.Tx) error{ + ConfigSchemaUpdate, + UpgradeLockSchemaUpdate, + } { + if err := dbTx(ctx, db, fn); err != nil { + t.Fatalf("schema apply: %v", err) + } + } + return db +} + +func dbTx(ctx context.Context, db *sql.DB, fn func(context.Context, *sql.Tx) error) error { + tx, err := db.BeginTx(ctx, nil) + if err != nil { + return err + } + if err := fn(ctx, tx); err != nil { + _ = tx.Rollback() + return err + } + return tx.Commit() +} + +// TestAcquireLockReturnsMonotonicToken is the core invariant: token always +// increases across acquires, so a stale holder is detectable. +func TestAcquireLockReturnsMonotonicToken(t *testing.T) { + ctx := context.Background() + db := newTestDB(t) + defer db.Close() + + var tok1, tok2 int64 + if err := dbTx(ctx, db, func(ctx context.Context, tx *sql.Tx) error { + var err error + tok1, err = AcquireUpgradeLock(ctx, tx, "host-a", 60) + return err + }); err != nil { + t.Fatalf("first acquire: %v", err) + } + + if err := dbTx(ctx, db, func(ctx context.Context, tx *sql.Tx) error { + var err error + tok2, err = AcquireUpgradeLock(ctx, tx, "host-a", 60) + return err + }); err != nil { + var held *LockHeldError + if !errors.As(err, &held) { + t.Fatalf("second acquire while held should be LockHeld, got %v", err) + } + // lock is live — expected; simulate expiry by setting expires_at to 0 + if err := dbTx(ctx, db, func(ctx context.Context, tx *sql.Tx) error { + _, err := tx.ExecContext(ctx, `UPDATE upgrade_lock SET expires_at = 0 WHERE id = 1`) + return err + }); err != nil { + t.Fatalf("force-expire: %v", err) + } + if err := dbTx(ctx, db, func(ctx context.Context, tx *sql.Tx) error { + var err error + tok2, err = AcquireUpgradeLock(ctx, tx, "host-b", 60) + return err + }); err != nil { + t.Fatalf("acquire after expiry: %v", err) + } + } + + if tok2 != tok1+1 { + t.Fatalf("token not monotonic: tok1=%d tok2=%d, expected %d", tok1, tok2, tok1+1) + } +} + +// TestVerifyTokenRejectsStaleToken is THE G1 invariant: after the lock +// expires and is re-acquired, writes carrying the old token are rejected. +func TestVerifyTokenRejectsStaleToken(t *testing.T) { + ctx := context.Background() + db := newTestDB(t) + defer db.Close() + + var staleTok, liveTok int64 + if err := dbTx(ctx, db, func(ctx context.Context, tx *sql.Tx) error { + var err error + staleTok, err = AcquireUpgradeLock(ctx, tx, "proc-a", 60) + return err + }); err != nil { + t.Fatalf("acquire A: %v", err) + } + + // Force expiry, acquire by B. + if err := dbTx(ctx, db, func(ctx context.Context, tx *sql.Tx) error { + _, err := tx.ExecContext(ctx, `UPDATE upgrade_lock SET expires_at = ? WHERE id = 1`, time.Now().Unix()-1) + return err + }); err != nil { + t.Fatalf("expire A: %v", err) + } + if err := dbTx(ctx, db, func(ctx context.Context, tx *sql.Tx) error { + var err error + liveTok, err = AcquireUpgradeLock(ctx, tx, "proc-b", 60) + return err + }); err != nil { + t.Fatalf("acquire B: %v", err) + } + + // Stale token (A's) must be rejected. + var staleErr *TokenMismatchError + if err := dbTx(ctx, db, func(ctx context.Context, tx *sql.Tx) error { + return VerifyToken(ctx, tx, staleTok) + }); !errors.As(err, &staleErr) { + t.Fatalf("stale token %d must be rejected with TokenMismatchError, got %v", staleTok, err) + } + if staleErr.Actual != liveTok { + t.Fatalf("mismatch error reports actual=%d, want %d", staleErr.Actual, liveTok) + } + + // Live token (B's) must pass. + if err := dbTx(ctx, db, func(ctx context.Context, tx *sql.Tx) error { + return VerifyToken(ctx, tx, liveTok) + }); err != nil { + t.Fatalf("live token %d must verify, got %v", liveTok, err) + } +} + +// TestReleaseThenAcquireContinuesTokenCount confirms release preserves the +// token counter for monotonicity — release does not reset to 0. +func TestReleaseThenAcquireContinuesTokenCount(t *testing.T) { + ctx := context.Background() + db := newTestDB(t) + defer db.Close() + + var tok1 int64 + if err := dbTx(ctx, db, func(ctx context.Context, tx *sql.Tx) error { + var err error + tok1, err = AcquireUpgradeLock(ctx, tx, "h", 60) + return err + }); err != nil { + t.Fatalf("acquire: %v", err) + } + if err := dbTx(ctx, db, func(ctx context.Context, tx *sql.Tx) error { + return ReleaseUpgradeLock(ctx, tx, tok1) + }); err != nil { + t.Fatalf("release: %v", err) + } + + var tok2 int64 + if err := dbTx(ctx, db, func(ctx context.Context, tx *sql.Tx) error { + var err error + tok2, err = AcquireUpgradeLock(ctx, tx, "h", 60) + return err + }); err != nil { + t.Fatalf("re-acquire: %v", err) + } + if tok2 != tok1+1 { + t.Fatalf("token must continue after release: tok1=%d tok2=%d, want %d", tok1, tok2, tok1+1) + } +} + +// TestRefreshRejectsStaleToken confirms refresh (heartbeat) fails loudly when +// the caller's token is stale — so the coordinator knows to stop. +func TestRefreshRejectsStaleToken(t *testing.T) { + ctx := context.Background() + db := newTestDB(t) + defer db.Close() + + var tok1 int64 + if err := dbTx(ctx, db, func(ctx context.Context, tx *sql.Tx) error { + var err error + tok1, err = AcquireUpgradeLock(ctx, tx, "a", 60) + return err + }); err != nil { + t.Fatalf("acquire: %v", err) + } + // Force expiry + re-acquire by someone else. + if err := dbTx(ctx, db, func(ctx context.Context, tx *sql.Tx) error { + _, err := tx.ExecContext(ctx, `UPDATE upgrade_lock SET expires_at = ? WHERE id = 1`, time.Now().Unix()-1) + return err + }); err != nil { + t.Fatalf("expire: %v", err) + } + if err := dbTx(ctx, db, func(ctx context.Context, tx *sql.Tx) error { + _, err := AcquireUpgradeLock(ctx, tx, "b", 60) + return err + }); err != nil { + t.Fatalf("re-acquire: %v", err) + } + // Stale refresh must fail. + var mismatch *TokenMismatchError + if err := dbTx(ctx, db, func(ctx context.Context, tx *sql.Tx) error { + return RefreshUpgradeLock(ctx, tx, tok1, 60) + }); !errors.As(err, &mismatch) { + t.Fatalf("refresh with stale token must fail with TokenMismatchError, got %v", err) + } +} diff --git a/sunbeam-microcluster/sunbeam/upgrade.go b/sunbeam-microcluster/sunbeam/upgrade.go new file mode 100644 index 000000000..f26a20b96 --- /dev/null +++ b/sunbeam-microcluster/sunbeam/upgrade.go @@ -0,0 +1,129 @@ +// Package sunbeam provides the interface to talk to database. +package sunbeam + +import ( + "context" + "database/sql" + "errors" + "fmt" + "net/http" + "time" + + "github.com/canonical/lxd/shared/api" + "github.com/canonical/microcluster/v2/state" + + "github.com/canonical/snap-openstack/sunbeam-microcluster/database" +) + +// Default lock TTL in seconds. Refreshed every ~30s by the coordinator. +const UpgradeLockTTLSeconds = 60 + +// UpgradeStateKey is the config-table key under which the upgrade state JSON +// blob is stored. This blob holds hop_history + active_hop as +// a reference (hop_history_index); there is no separate active_hop object. +const UpgradeStateKey = "upgrade_state" + +// AcquireUpgradeLock claims the advisory lock for holderID. Returns the +// fencing token that must be passed to every subsequent state write. +func AcquireUpgradeLock(ctx context.Context, s state.State, holderID string) (int64, error) { + var token int64 + err := s.Database().Transaction(ctx, func(ctx context.Context, tx *sql.Tx) error { + var acquireErr error + token, acquireErr = database.AcquireUpgradeLock(ctx, tx, holderID, UpgradeLockTTLSeconds) + return acquireErr + }) + return token, err +} + +// RefreshUpgradeLock extends the lock's TTL. Called by the coordinator's +// heartbeat loop. Returns an error if the caller's token is stale. +func RefreshUpgradeLock(ctx context.Context, s state.State, token int64) error { + return s.Database().Transaction(ctx, func(ctx context.Context, tx *sql.Tx) error { + return database.RefreshUpgradeLock(ctx, tx, token, UpgradeLockTTLSeconds) + }) +} + +// ReleaseUpgradeLock releases the lock. Called on clean exit (finalize, +// abandon, or command completion that doesn't leave a hop in flight). +func ReleaseUpgradeLock(ctx context.Context, s state.State, token int64) error { + return s.Database().Transaction(ctx, func(ctx context.Context, tx *sql.Tx) error { + return database.ReleaseUpgradeLock(ctx, tx, token) + }) +} + +// GetUpgradeState returns the persisted upgrade state JSON. Returns +// api.StatusError(404) if no state has been written yet. +func GetUpgradeState(ctx context.Context, s state.State) (string, error) { + var value string + err := s.Database().Transaction(ctx, func(ctx context.Context, tx *sql.Tx) error { + record, dbErr := database.GetConfigItem(ctx, tx, UpgradeStateKey) + if dbErr != nil { + return dbErr + } + value = record.Value + return nil + }) + if err != nil { + if apiStatusIsNotFound(err) { + return "", api.StatusErrorf(http.StatusNotFound, "no upgrade state") + } + return "", err + } + return value, nil +} + +// UpdateUpgradeState writes the upgrade state JSON, but only if the caller's +// fencing token matches the lock's current token. The token check and the +// write happen in the same transaction — SIGKILL between them is impossible. +// Returns database.TokenMismatchError if the token is stale. +func UpdateUpgradeState(ctx context.Context, s state.State, token int64, stateJSON string) error { + return s.Database().Transaction(ctx, func(ctx context.Context, tx *sql.Tx) error { + if err := database.VerifyToken(ctx, tx, token); err != nil { + return err + } + item := database.ConfigItem{Key: UpgradeStateKey, Value: stateJSON} + if _, err := database.GetConfigItem(ctx, tx, UpgradeStateKey); err != nil { + if apiStatusIsNotFound(err) { + if _, cerr := database.CreateConfigItem(ctx, tx, item); cerr != nil { + return fmt.Errorf("failed to create upgrade state: %w", cerr) + } + return nil + } + return err + } + if err := database.UpdateConfigItem(ctx, tx, UpgradeStateKey, item); err != nil { + return fmt.Errorf("failed to update upgrade state: %w", err) + } + return nil + }) +} + +// IsUpgradeActive returns true if a hop is in progress. Used by the mutating- +// command guard to block conflicting operations during an upgrade. +// A hop is "in progress" if the lock is held by a live holder. +func IsUpgradeActive(ctx context.Context, s state.State) (bool, error) { + var active bool + err := s.Database().Transaction(ctx, func(ctx context.Context, tx *sql.Tx) error { + row, err := database.GetUpgradeLockRow(ctx, tx) + if err != nil { + return err + } + if row.HolderID == "" { + active = false + return nil + } + active = row.ExpiresAt > time.Now().Unix() + return nil + }) + return active, err +} + +// apiStatusIsNotFound returns true if err is an lxd api StatusError with +// 404 status. +func apiStatusIsNotFound(err error) bool { + var se interface{ Status() int } + if errors.As(err, &se) { + return se.Status() == http.StatusNotFound + } + return false +} diff --git a/sunbeam-python/sunbeam/clusterd/cluster.py b/sunbeam-python/sunbeam/clusterd/cluster.py index 9bc62fa7e..8bb7afa5a 100644 --- a/sunbeam-python/sunbeam/clusterd/cluster.py +++ b/sunbeam-python/sunbeam/clusterd/cluster.py @@ -10,6 +10,7 @@ from requests.models import HTTPError from sunbeam.clusterd import models, service +from sunbeam.clusterd.service import ConfigItemNotFoundException, URLNotFoundException LOG = logging.getLogger(__name__) @@ -368,6 +369,55 @@ def update_feature_gate(self, gate_key: str, enabled: bool) -> None: } self._put(f"/1.0/feature-gates/{gate_key}", data=json.dumps(data)) + def acquire_upgrade_lock(self, holder_id: str) -> models.AcquireUpgradeLockResponse: + """Acquire the upgrade advisory lock. + + Returns the fencing token that must be passed to every subsequent + state write. Raises UpgradeLockHeldException if another live holder + owns the lock. + """ + data = {"holder_id": holder_id} + resp = self._post("/1.0/upgrade/lock", data=json.dumps(data)) + return models.AcquireUpgradeLockResponse(**resp.get("metadata", {})) + + def refresh_upgrade_lock(self, token: int) -> None: + """Refresh the upgrade lock's TTL (heartbeat). + + Raises UpgradeTokenMismatchException if the caller's token is stale + (lock expired and was re-acquired). + """ + data = {"token": token} + self._put("/1.0/upgrade/lock", data=json.dumps(data)) + + def release_upgrade_lock(self, token: int) -> None: + """Release the upgrade lock. + + Raises UpgradeTokenMismatchException if the caller's token is stale. + """ + data = {"token": token} + self._delete("/1.0/upgrade/lock", data=json.dumps(data)) + + def get_upgrade_state(self) -> str | None: + """Return the persisted upgrade state JSON, or None if no hop exists.""" + try: + return self._get("/1.0/upgrade/state").get("metadata") + except (ConfigItemNotFoundException, URLNotFoundException): + return None + + def update_upgrade_state(self, token: int, state: str) -> None: + """Write the upgrade state JSON. + + The token must match the lock's current fencing token, or the write + is rejected with UpgradeTokenMismatchException. The state is the + JSON-encoded upgrade state blob (spec §6.1). + """ + data = {"token": token, "state": state} + self._put("/1.0/upgrade/state", data=json.dumps(data)) + + def is_upgrade_active(self) -> bool: + """Return True if an upgrade hop is in progress.""" + return bool(self._get("/1.0/upgrade/active").get("metadata", {}).get("active")) + class ClusterService(MicroClusterService, ExtendedAPIService): """Lists and manages cluster.""" diff --git a/sunbeam-python/sunbeam/clusterd/models.py b/sunbeam-python/sunbeam/clusterd/models.py index 7e31991ab..49d3e25bc 100644 --- a/sunbeam-python/sunbeam/clusterd/models.py +++ b/sunbeam-python/sunbeam/clusterd/models.py @@ -45,3 +45,14 @@ class FeatureGate(pydantic.BaseModel): class FeatureGates(pydantic.RootModel[list[FeatureGate]]): """Feature gates model.""" + + +class AcquireUpgradeLockResponse(pydantic.BaseModel): + """Response from acquiring the upgrade lock. + + token is the fencing token that must be passed to every subsequent + state write; clusterd rejects writes whose token != the lock's current + token. + """ + + token: int diff --git a/sunbeam-python/sunbeam/clusterd/service.py b/sunbeam-python/sunbeam/clusterd/service.py index 7dfe175f9..5125b1f48 100644 --- a/sunbeam-python/sunbeam/clusterd/service.py +++ b/sunbeam-python/sunbeam/clusterd/service.py @@ -106,6 +106,18 @@ class StorageBackendNotFoundException(StorageBackendException): """Raised when storage backend is not found.""" +class UpgradeLockHeldException(RemoteException): + """Raised when the upgrade lock is held by another live holder.""" + + +class UpgradeTokenMismatchException(RemoteException): + """Raised when a state write carries a stale fencing token. + + The caller's lock expired and was re-acquired by someone else. The + coordinator must stop mutating state and surface the error. + """ + + class BaseService(ABC): """BaseService is the base service class for sunbeam clusterd services.""" @@ -220,10 +232,16 @@ def _request(self, method, path, **kwargs): # noqa: C901 too complex ) elif "ConfigItem not found" in error: raise ConfigItemNotFoundException("ConfigItem not found") + elif "no upgrade state" in error: + raise ConfigItemNotFoundException("No upgrade state") elif "ManifestItem not found" in error: raise ManifestItemNotFoundException("ManifestItem not found") elif "StorageBackend not found" in error: raise StorageBackendNotFoundException("Storage backend not found") + elif "upgrade lock held by" in error: + raise UpgradeLockHeldException(error) + elif "fencing token mismatch" in error: + raise UpgradeTokenMismatchException(error) raise e return response.json() diff --git a/sunbeam-python/tests/unit/sunbeam/test_clusterd.py b/sunbeam-python/tests/unit/sunbeam/test_clusterd.py index c67884311..522e32b2b 100644 --- a/sunbeam-python/tests/unit/sunbeam/test_clusterd.py +++ b/sunbeam-python/tests/unit/sunbeam/test_clusterd.py @@ -1042,3 +1042,192 @@ def test_run_handles_cluster_service_unavailable_exception(self, step_context): step.variables = {"bootstrap": {}} result = step.run(step_context) assert result.result_type == ResultType.FAILED + + +class TestUpgradeLock: + """Unit tests for the upgrade advisory lock + fencing-token client. + + G1 invariant tested here: the Python client surfaces HTTP 409 from + clusterd as typed exceptions (UpgradeLockHeldException on acquire, + UpgradeTokenMismatchException on refresh/release/write) so the + coordinator can react — instead of silently retrying against a lock it + no longer owns. + """ + + def _mock_response(self, status=200, json_data=None, raise_for_status=None): + mock_resp = MagicMock() + mock_resp.status_code = status + mock_resp.text = "MOCKCONTENT" + if json_data: + mock_resp.json.return_value = json_data + if raise_for_status: + mock_resp.raise_for_status.side_effect = raise_for_status + return mock_resp + + def _service(self, mock_response): + mock_session = MagicMock() + mock_session.request.return_value = mock_response + return ClusterService(mock_session, "http+unix://mock") + + def test_acquire_upgrade_lock_returns_token(self): + resp = self._mock_response( + status=200, + json_data={ + "type": "sync", + "status": "Success", + "status_code": 200, + "operation": "", + "error_code": 0, + "error": "", + "metadata": {"token": 7}, + }, + ) + cs = self._service(resp) + result = cs.acquire_upgrade_lock("host-a-pid1234") + assert result.token == 7 + # Verify the request hit the right endpoint with the right body. + _, kwargs = cs._BaseService__session.request.call_args + assert kwargs["method"] == "post" + assert kwargs["url"].endswith("/1.0/upgrade/lock") + assert json.loads(kwargs["data"]) == {"holder_id": "host-a-pid1234"} + + def test_acquire_upgrade_lock_raises_when_held(self): + # clusterd returns 409 with a LockHeldError-shaped message. + resp = self._mock_response( + status=409, + json_data={ + "type": "error", + "status": "", + "status_code": 0, + "operation": "", + "error_code": 409, + "error": 'upgrade lock held by "host-a-pid1234"', + "metadata": None, + }, + raise_for_status=HTTPError("Conflict"), + ) + cs = self._service(resp) + with pytest.raises(service.UpgradeLockHeldException): + cs.acquire_upgrade_lock("host-b-pid5678") + + def test_refresh_upgrade_lock_sends_token(self): + resp = self._mock_response( + status=200, + json_data={ + "type": "sync", + "status": "Success", + "status_code": 200, + "operation": "", + "error_code": 0, + "error": "", + "metadata": None, + }, + ) + cs = self._service(resp) + cs.refresh_upgrade_lock(token=7) + _, kwargs = cs._BaseService__session.request.call_args + assert kwargs["method"] == "put" + assert kwargs["url"].endswith("/1.0/upgrade/lock") + assert json.loads(kwargs["data"]) == {"token": 7} + + def test_refresh_upgrade_lock_raises_on_stale_token(self): + resp = self._mock_response( + status=409, + json_data={ + "type": "error", + "error": "fencing token mismatch: have 7, lock at 8", + "metadata": None, + }, + raise_for_status=HTTPError("Conflict"), + ) + cs = self._service(resp) + with pytest.raises(service.UpgradeTokenMismatchException): + cs.refresh_upgrade_lock(token=7) + + def test_release_upgrade_lock_sends_token(self): + resp = self._mock_response( + status=200, + json_data={ + "type": "sync", + "status": "Success", + "status_code": 200, + "operation": "", + "error_code": 0, + "error": "", + "metadata": None, + }, + ) + cs = self._service(resp) + cs.release_upgrade_lock(token=7) + _, kwargs = cs._BaseService__session.request.call_args + assert kwargs["method"] == "delete" + assert kwargs["url"].endswith("/1.0/upgrade/lock") + assert json.loads(kwargs["data"]) == {"token": 7} + + def test_get_upgrade_state_returns_state_json(self): + resp = self._mock_response( + status=200, + json_data={ + "type": "sync", + "metadata": '{"hop_history": []}', + }, + ) + cs = self._service(resp) + assert cs.get_upgrade_state() == '{"hop_history": []}' + + def test_get_upgrade_state_returns_none_when_missing(self): + resp = self._mock_response( + status=404, + json_data={ + "type": "error", + "error": "ConfigItem not found", + "metadata": None, + }, + raise_for_status=HTTPError("Not Found"), + ) + cs = self._service(resp) + assert cs.get_upgrade_state() is None + + def test_update_upgrade_state_sends_token_and_state(self): + resp = self._mock_response( + status=200, + json_data={"type": "sync", "metadata": None}, + ) + cs = self._service(resp) + cs.update_upgrade_state(token=7, state='{"active": true}') + _, kwargs = cs._BaseService__session.request.call_args + assert kwargs["method"] == "put" + assert kwargs["url"].endswith("/1.0/upgrade/state") + body = json.loads(kwargs["data"]) + assert body["token"] == 7 + assert body["state"] == '{"active": true}' + + def test_update_upgrade_state_raises_on_stale_token(self): + resp = self._mock_response( + status=409, + json_data={ + "type": "error", + "error": "fencing token mismatch: have 7, lock at 8", + "metadata": None, + }, + raise_for_status=HTTPError("Conflict"), + ) + cs = self._service(resp) + with pytest.raises(service.UpgradeTokenMismatchException): + cs.update_upgrade_state(token=7, state="{}") + + def test_is_upgrade_active_true(self): + resp = self._mock_response( + status=200, + json_data={"type": "sync", "metadata": {"active": True}}, + ) + cs = self._service(resp) + assert cs.is_upgrade_active() is True + + def test_is_upgrade_active_false(self): + resp = self._mock_response( + status=200, + json_data={"type": "sync", "metadata": {"active": False}}, + ) + cs = self._service(resp) + assert cs.is_upgrade_active() is False From 5f1cb651378c1474f3c6dabdb772aff617843c70 Mon Sep 17 00:00:00 2001 From: Hemanth Nakkina Date: Mon, 17 Aug 2026 16:22:05 +0530 Subject: [PATCH 02/20] feat(upgrades): add typed state model for upgrade engine Defines the persisted upgrade_state JSON blob as pydantic models matching the section 6.1 state structure, with two design resolutions applied: - metadata_build_id: typed field on Hop, sourced from snap revision at preflight. Lets the engine detect a mid-hop snap refresh and validate engine compatibility against the persisted metadata version. - active_hop is a reference (hop_history_index), not a duplicate of the hop's live state. hop_history[index] is the single canonical record - one source of truth, no dual-write drift on SIGKILL between two writes. The model includes idempotency helpers (is_step_complete, mark_step_complete) for the coordinator's resume logic. Every write to this blob is serialized through the fencing token (commit 26cc78e). 21 tests: example round-trip, metadata_build_id required + preserved, one-source-of-truth (active_hop is just an index), idempotency helpers, empty state safety, fresh hop construction for hop creation. --- sunbeam-python/sunbeam/upgrades/__init__.py | 3 + sunbeam-python/sunbeam/upgrades/state.py | 352 ++++++++++++++++++ .../tests/unit/sunbeam/test_upgrade_state.py | 258 +++++++++++++ 3 files changed, 613 insertions(+) create mode 100644 sunbeam-python/sunbeam/upgrades/__init__.py create mode 100644 sunbeam-python/sunbeam/upgrades/state.py create mode 100644 sunbeam-python/tests/unit/sunbeam/test_upgrade_state.py diff --git a/sunbeam-python/sunbeam/upgrades/__init__.py b/sunbeam-python/sunbeam/upgrades/__init__.py new file mode 100644 index 000000000..8d593f1ba --- /dev/null +++ b/sunbeam-python/sunbeam/upgrades/__init__.py @@ -0,0 +1,3 @@ +# SPDX-FileCopyrightText: 2025 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 +"""Upgrade engine for Sunbeam OpenStack major-release hops.""" diff --git a/sunbeam-python/sunbeam/upgrades/state.py b/sunbeam-python/sunbeam/upgrades/state.py new file mode 100644 index 000000000..b7650606f --- /dev/null +++ b/sunbeam-python/sunbeam/upgrades/state.py @@ -0,0 +1,352 @@ +# SPDX-FileCopyrightText: 2025 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +"""Typed state model for the release-upgrade engine. + +Models the persisted ``upgrade_state`` JSON blob: + +- ``metadata_build_id`` is a typed field on ``Hop``, sourced from the + snap revision at preflight. It lets the engine detect a mid-hop snap refresh + and validate engine compatibility against the persisted metadata version. + +- ``active_hop`` is a reference (``hop_history_index``), not a duplicate + of the hop's live state. ``hop_history[index]`` is the single canonical + record — one source of truth, no dual-write drift. ``active_hop`` is ``None`` + when no hop is in flight. + +The lock primitive serializes writes to this blob via +``ClusterService.update_upgrade_state(token, state_json)``. The entire blob is +read and written as a whole — no partial updates — so SIGKILL during a write +leaves either the old or the new state, never a split. +""" + +from __future__ import annotations + +import enum + +import pydantic + + +class HopStatus(str, enum.Enum): + """Lifecycle state of a single upgrade hop.""" + + PENDING = "pending" + IN_PROGRESS = "in_progress" + COMPLETED = "completed" + BLOCKED = "blocked" + FAILED = "failed" + ABANDONED = "abandoned" + + +class PhaseStatus(str, enum.Enum): + """Lifecycle state of a phase within a hop.""" + + PENDING = "pending" + IN_PROGRESS = "in_progress" + COMPLETED = "completed" + FAILED = "failed" + BLOCKED = "blocked" + + +class StepStatus(str, enum.Enum): + """Lifecycle state of a single step within a node upgrade.""" + + PENDING = "pending" + IN_PROGRESS = "in_progress" + COMPLETED = "completed" + FAILED = "failed" + + +class ComponentRole(str, enum.Enum): + """Role of a juju unit within a node upgrade. + + AUXILIARY means a co-located principal charm on the same machine that + must be refreshed alongside the principal unit (e.g. epa-orchestrator, + openstack-network-agents). Not a Juju subordinate — subordinates ride + their principal's refresh automatically via relation machinery. + """ + + PRINCIPAL = "principal" + AUXILIARY = "auxiliary" + + +class ComponentStatus(str, enum.Enum): + """Lifecycle state of a single component (unit) within a node upgrade.""" + + PENDING = "pending" + IN_PROGRESS = "in_progress" + COMPLETED = "completed" + FAILED = "failed" + + +class LastError(pydantic.BaseModel): + """Error snapshot persisted on a failed phase/group/node/step. + + code comes from the error code catalog. message is human-readable + detail. Both are surfaced by ``sunbeam cluster upgrade status`` so + the operator can act without a separate command. + """ + + code: str = pydantic.Field(description="Error code from the catalog.") + message: str = pydantic.Field(description="Human-readable error detail.") + + +class Component(pydantic.BaseModel): + """A single juju unit being upgraded as part of a node.""" + + unit: str = pydantic.Field(description="Juju unit name, e.g. nova-k8s/0") + role: ComponentRole = pydantic.Field(description="principal or auxiliary") + previous_channel: str = pydantic.Field( + description="Channel before this hop, e.g. 2024.1/stable" + ) + target_channel: str = pydantic.Field( + description="Channel for this hop, e.g. 2025.1/stable" + ) + status: ComponentStatus = pydantic.Field( + default=ComponentStatus.PENDING, description="Upgrade status of this unit" + ) + + +class Node(pydantic.BaseModel): + """A single compute or storage node being upgraded in the data plane. + + Carries the per-node step state machine. The coordinator writes ``step`` + before executing it (SIGKILL safety — on resume, a step with + ``status: in_progress`` is treated as failed and re-executed). + + ``auxiliary_units`` are co-located principal charms on the same machine + that must be refreshed alongside the principal (e.g. epa-orchestrator + and openstack-network-agents alongside openstack-hypervisor on a compute + node). They are NOT Juju subordinates — a true subordinate + (e.g. cinder-volume-ceph) rides its principal's refresh automatically + via Juju relation machinery and needs no explicit step here. + """ + + status: PhaseStatus = pydantic.Field( + default=PhaseStatus.PENDING, description="Node upgrade status" + ) + step: str | None = pydantic.Field( + default=None, + description="Current or last-attempted step name within the node sequence", + ) + step_status: StepStatus = pydantic.Field( + default=StepStatus.PENDING, + description="Status of the current step (SIGKILL safety)", + ) + principal_unit: str | None = pydantic.Field( + default=None, description="Principal juju unit for this node" + ) + auxiliary_units: list[str] = pydantic.Field( + default_factory=list, description="Auxiliary juju units for this node" + ) + components: list[Component] = pydantic.Field( + default_factory=list, description="Per-unit upgrade records" + ) + last_error: LastError | None = pydantic.Field( + default=None, description="Error snapshot if status is failed" + ) + + +class Group(pydantic.BaseModel): + """A control-plane upgrade group (e.g. identity-core, compute-control).""" + + status: PhaseStatus = pydantic.Field( + default=PhaseStatus.PENDING, description="Group upgrade status" + ) + started_at: str | None = pydantic.Field( + default=None, description="ISO timestamp when the group started" + ) + completed_at: str | None = pydantic.Field( + default=None, description="ISO timestamp when the group completed" + ) + last_error: LastError | None = pydantic.Field( + default=None, description="Error snapshot if status is failed" + ) + + +class ControlPlanePhase(pydantic.BaseModel): + """Control-plane phase: a set of groups upgraded in sequence.""" + + status: PhaseStatus = pydantic.Field( + default=PhaseStatus.PENDING, description="Phase status" + ) + groups: dict[str, Group] = pydantic.Field( + default_factory=dict, + description="Per-group state, keyed by group name from metadata", + ) + + +class DataplanePhase(pydantic.BaseModel): + """Data-plane phase: per-node compute upgrades.""" + + status: PhaseStatus = pydantic.Field( + default=PhaseStatus.PENDING, description="Phase status" + ) + nodes: dict[str, Node] = pydantic.Field( + default_factory=dict, + description="Per-node state, keyed by hostname or juju unit name", + ) + + +class SimplePhase(pydantic.BaseModel): + """A phase with no internal sub-structure (preflight, storage, finalize). + + Used for phases that track only a status + optional backup_id or + last_error, without groups or nodes. + """ + + status: PhaseStatus = pydantic.Field( + default=PhaseStatus.PENDING, description="Phase status" + ) + backup_id: str | None = pydantic.Field( + default=None, + description="Backup artifact ID (preflight phase only)", + ) + last_error: LastError | None = pydantic.Field( + default=None, description="Error snapshot if status is failed" + ) + + +class Phases(pydantic.BaseModel): + """All phases of a hop.""" + + preflight: SimplePhase = pydantic.Field(default_factory=SimplePhase) + control_plane: ControlPlanePhase = pydantic.Field(default_factory=ControlPlanePhase) + dataplane: DataplanePhase = pydantic.Field(default_factory=DataplanePhase) + storage: SimplePhase = pydantic.Field(default_factory=SimplePhase) + finalize: SimplePhase = pydantic.Field(default_factory=SimplePhase) + + +class Hop(pydantic.BaseModel): + """A single release-to-release upgrade hop. + + This is the canonical record — ``active_hop`` in the top-level state is + just a reference (``hop_history_index``) into the ``hop_history`` list. + All hop state lives here. + """ + + from_release: str = pydantic.Field( + alias="from", + description="Source release, e.g. 2024.1", + ) + to_release: str = pydantic.Field( + alias="to", + description="Target release, e.g. 2025.1", + ) + status: HopStatus = pydantic.Field( + default=HopStatus.PENDING, description="Hop lifecycle status" + ) + phase: str | None = pydantic.Field( + default=None, + description="Current phase: preflight, control_plane, dataplane," + " storage, or finalize", + ) + metadata_version: int = pydantic.Field( + description="Metadata schema version this hop was created with" + ) + metadata_build_id: str = pydantic.Field( + description="Snap revision that created this hop." + " Detects mid-hop snap refresh.", + ) + phases: Phases = pydantic.Field( + default_factory=Phases, description="Per-phase state" + ) + last_error: LastError | None = pydantic.Field( + default=None, description="Error snapshot if the hop is failed" + ) + + +class ActiveHop(pydantic.BaseModel): + """Reference to the active hop in ``hop_history``. + + ``hop_history_index`` is the index into the ``hop_history`` list. ``None`` + means no hop is in flight. This is the ONLY field — the hop's status, + phase, and all detail live in ``hop_history[index]`` to avoid dual-write + drift. + """ + + hop_history_index: int | None = pydantic.Field( + default=None, + description="Index into hop_history for the active hop," + " or None if no hop is active", + ) + + +class UpgradeState(pydantic.BaseModel): + """Top-level persisted upgrade state. + + Stored in clusterd under the ``upgrade_state`` config key as JSON. Every + write is guarded by the fencing token via + ``ClusterService.update_upgrade_state(token, state_json)``. + + ``active_hop`` is a reference, not a duplicate of the hop's live + state. ``hop_history[active_hop.hop_history_index]`` is canonical. + """ + + active_hop: ActiveHop = pydantic.Field(default_factory=ActiveHop) + hop_history: list[Hop] = pydantic.Field(default_factory=list) + + @property + def current_hop(self) -> Hop | None: + """Return the active hop, or None if no hop is in flight.""" + idx = self.active_hop.hop_history_index + if idx is None: + return None + if idx < 0 or idx >= len(self.hop_history): + return None + return self.hop_history[idx] + + def is_upgrade_active(self) -> bool: + """True if a hop is in progress (active and not terminal).""" + hop = self.current_hop + if hop is None: + return False + return hop.status in ( + HopStatus.PENDING, + HopStatus.IN_PROGRESS, + HopStatus.BLOCKED, + ) + + def is_step_complete(self, phase: str, step: str) -> bool: + """Check if a step within the current hop's phase is complete. + + Used by the coordinator's resume logic to skip completed steps. + Only meaningful for dataplane/storage node steps — control-plane + groups use the ``Group`` status directly. + """ + hop = self.current_hop + if hop is None: + return False + phase_obj = getattr(hop.phases, phase, None) + if phase_obj is None: + return False + if phase == "dataplane": + # Node-level steps: check all nodes' step_status + return all( + node.step_status == StepStatus.COMPLETED + for node in phase_obj.nodes.values() + ) + if hasattr(phase_obj, "status"): + return phase_obj.status == PhaseStatus.COMPLETED + return False + + def mark_step_complete(self, phase: str, step: str) -> None: + """Mark a step within the current hop's phase as complete. + + For dataplane: marks the given step as complete on all nodes that + have that step as their current step. For other phases: marks the + phase itself as completed. Caller is responsible for persisting the + state via ``update_upgrade_state`` after mutation. + """ + hop = self.current_hop + if hop is None: + raise ValueError("no active hop") + phase_obj = getattr(hop.phases, phase, None) + if phase_obj is None: + raise ValueError(f"unknown phase: {phase}") + if phase == "dataplane": + for node in phase_obj.nodes.values(): + if node.step == step: + node.step_status = StepStatus.COMPLETED + else: + phase_obj.status = PhaseStatus.COMPLETED diff --git a/sunbeam-python/tests/unit/sunbeam/test_upgrade_state.py b/sunbeam-python/tests/unit/sunbeam/test_upgrade_state.py new file mode 100644 index 000000000..0a017da93 --- /dev/null +++ b/sunbeam-python/tests/unit/sunbeam/test_upgrade_state.py @@ -0,0 +1,258 @@ +# SPDX-FileCopyrightText: 2025 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for the upgrade state model (G2 + G3 resolutions).""" + +import copy + +import pytest + +from sunbeam.upgrades.state import ( + Hop, + HopStatus, + PhaseStatus, + StepStatus, + UpgradeState, +) + +STATE_EXAMPLE = { + "active_hop": {"hop_history_index": 0}, + "hop_history": [ + { + "from": "2024.1", + "to": "2025.1", + "status": "in_progress", + "phase": "dataplane", + "metadata_version": 1, + "metadata_build_id": "snap-rev-1005", + "phases": { + "preflight": { + "status": "completed", + "backup_id": "backup-20250805-0900", + }, + "control_plane": { + "status": "completed", + "groups": { + "identity-core": { + "status": "completed", + "started_at": "2025-08-05T09:00:00Z", + "completed_at": "2025-08-05T09:05:00Z", + }, + "compute-control": {"status": "completed"}, + }, + }, + "dataplane": { + "status": "in_progress", + "nodes": { + "compute-17": { + "status": "failed", + "step": "finish-node-upgrade", + "step_status": "failed", + "principal_unit": "openstack-hypervisor/17", + "auxiliary_units": [ + "epa-orchestrator/17", + "openstack-network-agents/17", + ], + "components": [ + { + "unit": "openstack-hypervisor/17", + "role": "principal", + "previous_channel": "2024.1/stable", + "target_channel": "2025.1/stable", + "status": "failed", + }, + { + "unit": "epa-orchestrator/17", + "role": "auxiliary", + "previous_channel": "2024.1/stable", + "target_channel": "2025.1/stable", + "status": "pending", + }, + ], + "last_error": { + "code": "SERVICE_REGISTRATION_TIMEOUT", + "message": "nova-compute did not re-register within 300s", + }, + }, + "compute-18": {"status": "pending"}, + }, + }, + "storage": {"status": "pending"}, + "finalize": {"status": "pending"}, + }, + } + ], +} + + +class TestUpgradeStateRoundTrip: + """The state example must deserialize and re-serialize losslessly.""" + + def test_parses_example(self): + state = UpgradeState.model_validate(STATE_EXAMPLE) + assert state.active_hop.hop_history_index == 0 + assert len(state.hop_history) == 1 + + def test_round_trip_preserves_state(self): + state = UpgradeState.model_validate(STATE_EXAMPLE) + json_str = state.model_dump_json(by_alias=True) + restored = UpgradeState.model_validate_json(json_str) + assert restored.model_dump(by_alias=True) == state.model_dump(by_alias=True) + + def test_serializes_to_expected_shape(self): + state = UpgradeState.model_validate(STATE_EXAMPLE) + dumped = state.model_dump(by_alias=True, exclude_none=True) + # G3: active_hop is just an index, not a duplicate of the hop's state + assert dumped["active_hop"] == {"hop_history_index": 0} + # G2: metadata_build_id is present on the hop + assert dumped["hop_history"][0]["metadata_build_id"] == "snap-rev-1005" + # The hop's status/phase live ONLY in hop_history, not in active_hop + assert "status" not in dumped["active_hop"] + assert "phase" not in dumped["active_hop"] + + +class TestG3OneSourceOfTruth: + """active_hop is a reference; hop_history[index] is canonical.""" + + def test_current_hop_returns_the_active_hop(self): + state = UpgradeState.model_validate(STATE_EXAMPLE) + hop = state.current_hop + assert hop is not None + assert hop.from_release == "2024.1" + assert hop.to_release == "2025.1" + assert hop.status == HopStatus.IN_PROGRESS + + def test_current_hop_returns_none_when_no_active_hop(self): + state = UpgradeState() + assert state.current_hop is None + + def test_current_hop_returns_none_when_index_out_of_range(self): + state = UpgradeState( + active_hop={"hop_history_index": 5}, + hop_history=[], + ) + assert state.current_hop is None + + def test_is_upgrade_active_true_when_in_progress(self): + state = UpgradeState.model_validate(STATE_EXAMPLE) + assert state.is_upgrade_active() is True + + def test_is_upgrade_active_false_when_no_hop(self): + state = UpgradeState() + assert state.is_upgrade_active() is False + + def test_is_upgrade_active_false_when_completed(self): + data = copy.deepcopy(STATE_EXAMPLE) + data["hop_history"][0]["status"] = "completed" + state = UpgradeState.model_validate(data) + assert state.is_upgrade_active() is False + + +class TestG2MetadataBuildId: + """metadata_build_id is a typed field, sourced from snap revision.""" + + def test_metadata_build_id_round_trips(self): + state = UpgradeState.model_validate(STATE_EXAMPLE) + assert state.current_hop.metadata_build_id == "snap-rev-1005" + json_str = state.model_dump_json(by_alias=True) + assert "snap-rev-1005" in json_str + + def test_metadata_build_id_is_required_on_hop(self): + """A hop without metadata_build_id should fail validation.""" + bad_hop = { + "from": "2024.1", + "to": "2025.1", + "status": "in_progress", + "metadata_version": 1, + # metadata_build_id missing + } + with pytest.raises(Exception): + Hop.model_validate(bad_hop) + + +class TestIdempotencyHelpers: + """is_step_complete / mark_step_complete for resume logic.""" + + def test_is_step_complete_false_for_in_progress_phase(self): + state = UpgradeState.model_validate(STATE_EXAMPLE) + # dataplane is in_progress — not complete + assert state.is_step_complete("dataplane", "finish-node-upgrade") is False + + def test_is_step_complete_true_for_completed_phase(self): + state = UpgradeState.model_validate(STATE_EXAMPLE) + assert state.is_step_complete("control_plane", "identity-core") is True + + def test_mark_step_complete_sets_node_step_status(self): + state = UpgradeState.model_validate(STATE_EXAMPLE) + hop = state.current_hop + # compute-18 is pending with no step + node = hop.phases.dataplane.nodes["compute-18"] + node.step = "disable-scheduling" + node.step_status = StepStatus.IN_PROGRESS + state.mark_step_complete("dataplane", "disable-scheduling") + assert node.step_status == StepStatus.COMPLETED + + def test_mark_step_complete_raises_when_no_active_hop(self): + state = UpgradeState() + with pytest.raises(ValueError, match="no active hop"): + state.mark_step_complete("dataplane", "some-step") + + def test_mark_step_complete_raises_for_unknown_phase(self): + state = UpgradeState.model_validate(STATE_EXAMPLE) + with pytest.raises(ValueError, match="unknown phase"): + state.mark_step_complete("nonexistent", "some-step") + + +class TestEmptyState: + """A fresh UpgradeState with no hop should be safe to query.""" + + def test_empty_state_serializes(self): + state = UpgradeState() + json_str = state.model_dump_json(by_alias=True) + restored = UpgradeState.model_validate_json(json_str) + assert restored.current_hop is None + assert restored.is_upgrade_active() is False + + def test_empty_state_current_hop_is_none(self): + state = UpgradeState() + assert state.current_hop is None + + def test_empty_state_is_step_complete_is_false(self): + state = UpgradeState() + assert state.is_step_complete("dataplane", "any-step") is False + + +class TestHopCreation: + """W3.4 (hop creation) will construct a fresh Hop + UpgradeState.""" + + def test_new_hop_has_correct_defaults(self): + hop = Hop( + **{ + "from": "2024.1", + "to": "2025.1", + "metadata_version": 1, + "metadata_build_id": "snap-rev-1005", + } + ) + assert hop.status == HopStatus.PENDING + assert hop.phase is None + assert hop.phases.preflight.status == PhaseStatus.PENDING + assert hop.phases.control_plane.status == PhaseStatus.PENDING + assert hop.phases.dataplane.status == PhaseStatus.PENDING + + def test_new_state_with_first_hop(self): + hop = Hop( + **{ + "from": "2024.1", + "to": "2025.1", + "metadata_version": 1, + "metadata_build_id": "snap-rev-1005", + } + ) + state = UpgradeState( + active_hop={"hop_history_index": 0}, + hop_history=[hop], + ) + assert state.current_hop is not None + assert state.current_hop.from_release == "2024.1" + assert state.is_upgrade_active() is True From 73b1bafbd0cfabbf9f6724fc00b0713a6410e65a Mon Sep 17 00:00:00 2001 From: Hemanth Nakkina Date: Mon, 17 Aug 2026 17:34:43 +0530 Subject: [PATCH 03/20] feat(upgrades): add RELEASE_TRACKS table and error code catalog RELEASE_TRACKS replaces the hardcoded release string fallback in versions.py. Maps each supported OpenStack release (2024.1, 2025.1, 2026.1) to its infrastructure channel mappings. The upgrade engine uses this to validate SLURP hop validity and detect which release a cluster is running at. - RELEASE_TRACKS: dict keyed by release with channel mappings - SLURP_HOPS: set of valid (from, to) release pairs - detect_deployed_release(): reads charm channels, returns release key - detect_snap_release(): reads the snap's deployment.version config - is_valid_hop(): checks if a release pair is a valid upgrade path - DEFAULT_RELEASE replaces the hardcoded "2026.1" fallback Error code catalog defines a closed set of error codes for the last_error.code field on persisted state. Every phase handler that sets last_error uses a code from this catalog. The status command surfaces code + message to the operator. - UpgradeErrorCode enum: 22 codes following _ convention - ERROR_MESSAGES: human-readable, actionable message per code - get_error_message(): resolves code to message, falls back to code 29 tests: release track structure, SLURP hop validation, deployed release detection, error code uniqueness, message coverage, actionable length. --- sunbeam-python/sunbeam/upgrades/errors.py | 180 ++++++++++++++++++ sunbeam-python/sunbeam/versions.py | 114 ++++++++++- .../tests/unit/sunbeam/test_release_tracks.py | 143 ++++++++++++++ .../tests/unit/sunbeam/test_upgrade_errors.py | 87 +++++++++ 4 files changed, 523 insertions(+), 1 deletion(-) create mode 100644 sunbeam-python/sunbeam/upgrades/errors.py create mode 100644 sunbeam-python/tests/unit/sunbeam/test_release_tracks.py create mode 100644 sunbeam-python/tests/unit/sunbeam/test_upgrade_errors.py diff --git a/sunbeam-python/sunbeam/upgrades/errors.py b/sunbeam-python/sunbeam/upgrades/errors.py new file mode 100644 index 000000000..cadf736f7 --- /dev/null +++ b/sunbeam-python/sunbeam/upgrades/errors.py @@ -0,0 +1,180 @@ +# SPDX-FileCopyrightText: 2025 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +"""Error code catalog for the upgrade engine. + +Every phase handler that sets ``last_error`` on a failed phase/group/node/step +uses a code from this catalog. The ``status`` command surfaces the +code + message to the operator so they can act without a separate command. + +Codes follow the convention ``_``: +- LOCK_* — advisory lock failures +- METADATA_* — metadata loading/validation failures +- PREFLIGHT_* — preflight check failures +- CONTROL_PLANE_* — control-plane phase failures +- DATAPLANE_* — data-plane phase failures +- STORAGE_* — storage phase failures +- FINALIZE_* — finalize phase failures + +Adding a new code: add it here. The catalog is closed — tests assert on +codes, status renders stable messages per code. +""" + +from __future__ import annotations + +import enum + + +class UpgradeErrorCode(str, enum.Enum): + """Closed catalog of upgrade error codes.""" + + # Lock-related + LOCK_HELD = "LOCK_HELD" + FENCING_TOKEN_MISMATCH = "FENCING_TOKEN_MISMATCH" + + # Metadata-related + METADATA_INCOMPAT = "METADATA_INCOMPAT" + METADATA_MISSING = "METADATA_MISSING" + METADATA_INVALID = "METADATA_INVALID" + + # Preflight + PREFLIGHT_FAILED = "PREFLIGHT_FAILED" + PREFLIGHT_BACKUP_FAILED = "PREFLIGHT_BACKUP_FAILED" + PREFLIGHT_RESTORE_FAILED = "PREFLIGHT_RESTORE_FAILED" + PREFLIGHT_CAPACITY = "PREFLIGHT_CAPACITY" + PREFLIGHT_HEALTH_CHECK = "PREFLIGHT_HEALTH_CHECK" + + # Control-plane phase + CONTROL_PLANE_CONVERGENCE_TIMEOUT = "CONTROL_PLANE_CONVERGENCE_TIMEOUT" + CONTROL_PLANE_APPLY_FAILED = "CONTROL_PLANE_APPLY_FAILED" + CONTROL_PLANE_ACTION_FAILED = "CONTROL_PLANE_ACTION_FAILED" + + # Data-plane phase + DATAPLANE_REGISTRATION_TIMEOUT = "DATAPLANE_REGISTRATION_TIMEOUT" + DATAPLANE_REFRESH_FAILED = "DATAPLANE_REFRESH_FAILED" + DATAPLANE_VM_CHECK_FAILED = "DATAPLANE_VM_CHECK_FAILED" + + # Storage phase + STORAGE_REGISTRATION_TIMEOUT = "STORAGE_REGISTRATION_TIMEOUT" + STORAGE_REFRESH_FAILED = "STORAGE_REFRESH_FAILED" + STORAGE_BACKEND_UNSUPPORTED = "STORAGE_BACKEND_UNSUPPORTED" + + # Finalize phase + FINALIZE_MIGRATION_FAILED = "FINALIZE_MIGRATION_FAILED" + FINALIZE_VALIDATION_FAILED = "FINALIZE_VALIDATION_FAILED" + + # Cross-cutting + ROLLBACK_FAILED = "ROLLBACK_FAILED" + HOP_INVALID_TRANSITION = "HOP_INVALID_TRANSITION" + + +# Human-readable messages for each code. The status command uses +# these to render operator-facing output. Keep them actionable — tell the +# operator what to do, not just what went wrong. +ERROR_MESSAGES: dict[UpgradeErrorCode, str] = { + UpgradeErrorCode.LOCK_HELD: ( + "Another upgrade operation is in progress. " + "Complete or abandon the current upgrade before starting a new one." + ), + UpgradeErrorCode.FENCING_TOKEN_MISMATCH: ( + "The upgrade lock was acquired by another process after this one's " + "lock expired. Re-run the command to acquire a fresh lock." + ), + UpgradeErrorCode.METADATA_INCOMPAT: ( + "The upgrade metadata is incompatible with this snap version. " + "Refresh the snap to the target release before running upgrade." + ), + UpgradeErrorCode.METADATA_MISSING: ( + "Upgrade metadata not found for the target release. " + "Ensure the snap is refreshed to the target release." + ), + UpgradeErrorCode.METADATA_INVALID: ( + "Upgrade metadata failed validation. Check the metadata file " + "for missing or invalid fields." + ), + UpgradeErrorCode.PREFLIGHT_FAILED: ( + "Preflight checks failed. Review the output for specific failures " + "and resolve them before retrying." + ), + UpgradeErrorCode.PREFLIGHT_BACKUP_FAILED: ( + "Backup creation failed. Ensure sufficient disk space and that " + "Juju, MySQL, and clusterd are reachable." + ), + UpgradeErrorCode.PREFLIGHT_RESTORE_FAILED: ( + "Backup restore failed. The cluster may be in an inconsistent " + "state. Contact support with the backup artifacts." + ), + UpgradeErrorCode.PREFLIGHT_CAPACITY: ( + "Insufficient compute capacity for a safe upgrade. Migrate VMs " + "to free up nodes, or override with --capacity-policy-override." + ), + UpgradeErrorCode.PREFLIGHT_HEALTH_CHECK: ( + "Cluster health check failed. Resolve the reported issues " + "before retrying the upgrade." + ), + UpgradeErrorCode.CONTROL_PLANE_CONVERGENCE_TIMEOUT: ( + "Control-plane group did not converge within the timeout. " + "Check juju status for stuck units and retry with --retry-group." + ), + UpgradeErrorCode.CONTROL_PLANE_APPLY_FAILED: ( + "Terraform apply failed for a control-plane group. " + "Check terraform output and retry with --retry-group." + ), + UpgradeErrorCode.CONTROL_PLANE_ACTION_FAILED: ( + "A pre-upgrade or post-upgrade action failed. " + "Check the action output and retry the group." + ), + UpgradeErrorCode.DATAPLANE_REGISTRATION_TIMEOUT: ( + "Nova-compute or cinder-volume did not re-register within the " + "timeout. Check the service status and retry with --retry-node." + ), + UpgradeErrorCode.DATAPLANE_REFRESH_FAILED: ( + "Charm or snap refresh failed on a compute node. " + "Check juju status and retry with --retry-node, or rollback " + "with --rollback-node." + ), + UpgradeErrorCode.DATAPLANE_VM_CHECK_FAILED: ( + "VMs are still running on the compute node. Migrate or stop " + "them before retrying with --retry-node." + ), + UpgradeErrorCode.STORAGE_REGISTRATION_TIMEOUT: ( + "Cinder-volume did not re-register within the timeout. " + "Check the service status and retry with --retry-node." + ), + UpgradeErrorCode.STORAGE_REFRESH_FAILED: ( + "Snap refresh failed on a storage node. Check juju status and " + "retry with --retry-node." + ), + UpgradeErrorCode.STORAGE_BACKEND_UNSUPPORTED: ( + "Non-Ceph storage backend detected. Only Ceph-backed " + "cinder-volume is supported for upgrades." + ), + UpgradeErrorCode.FINALIZE_MIGRATION_FAILED: ( + "Online data migration failed. Check the migration output and re-run finalize." + ), + UpgradeErrorCode.FINALIZE_VALIDATION_FAILED: ( + "End-state validation failed. Not all services are healthy. " + "Check juju status and resolve issues before re-running finalize." + ), + UpgradeErrorCode.ROLLBACK_FAILED: ( + "Node rollback failed. The node may be in an inconsistent " + "state. Check juju status and consider manual recovery." + ), + UpgradeErrorCode.HOP_INVALID_TRANSITION: ( + "Invalid state transition attempted. This indicates a bug " + "in the upgrade coordinator. Report this issue." + ), +} + + +def get_error_message(code: str | UpgradeErrorCode) -> str: + """Return the human-readable message for an error code. + + Falls back to the code itself if no message is defined. + """ + if isinstance(code, str): + try: + code = UpgradeErrorCode(code) + except ValueError: + return code + return ERROR_MESSAGES.get(code, code.value) diff --git a/sunbeam-python/sunbeam/versions.py b/sunbeam-python/sunbeam/versions.py index 32b55e4d7..0a87a7c4b 100644 --- a/sunbeam-python/sunbeam/versions.py +++ b/sunbeam-python/sunbeam/versions.py @@ -5,6 +5,60 @@ from functools import cache from typing import TypedDict +# RELEASE_TRACKS maps each supported OpenStack release to its infrastructure +# channel mappings. The upgrade engine uses this to: +# - validate SLURP hop validity (is from->to a declared hop?) +# - detect which release a cluster is running at +# - know which channels infrastructure charms should be on per release +# +# Adding a new release = one entry here + a new manifests// dir. +# The release key (e.g. "2026.1") is the canonical identifier everywhere: +# metadata files, state model, CLI flags. +RELEASE_TRACKS: dict[str, dict[str, str]] = { + "2024.1": { + "name": "caracal", + "openstack_channel": "2024.1/stable", + "microceph_channel": "reef/stable", + "microovn_channel": "22.03/stable", + "mysql_channel": "8.0/stable", + "rabbitmq_channel": "3.12/stable", + "vault_channel": "1.15/stable", + "consul_channel": "1.19/stable", + }, + "2025.1": { + "name": "epoxy", + "openstack_channel": "2025.1/stable", + "microceph_channel": "squid/stable", + "microovn_channel": "25.03/stable", + "mysql_channel": "8.0/stable", + "rabbitmq_channel": "3.12/stable", + "vault_channel": "2.0/stable", + "consul_channel": "1.19/stable", + }, + "2026.1": { + "name": "gazpacho", + "openstack_channel": "2026.1/stable", + "microceph_channel": "squid/stable", + "microovn_channel": "26.03/stable", + "mysql_channel": "8.0/stable", + "rabbitmq_channel": "3.12/stable", + "vault_channel": "2.0/stable", + "consul_channel": "1.19/stable", + }, +} + +# Default release when the snap config doesn't specify one. Must be a key +# in RELEASE_TRACKS. This is the release the snap ships manifests for. +DEFAULT_RELEASE = "2026.1" + +# Valid SLURP upgrade hops. Each entry is (from_release, to_release). The +# upgrade engine validates that a requested hop is in this set before +# creating an active hop. +SLURP_HOPS: set[tuple[str, str]] = { + ("2024.1", "2025.1"), + ("2025.1", "2026.1"), +} + @cache def determine_version() -> str: @@ -15,10 +69,68 @@ def determine_version() -> str: snap = Snap() risk = str(snap.config.get("deployment.version")) except Exception: - risk = "2026.1" + risk = DEFAULT_RELEASE return risk +def detect_snap_release() -> str: + """Return the release the snap binary is built for. + + Reads the snap version string (e.g. '2026.1-abc123' -> '2026.1'). + Falls back to DEFAULT_RELEASE if the value is not a known release track. + """ + from snaphelpers import Snap + + try: + snap = Snap() + version = snap.version.split("-")[0] + except Exception: + return DEFAULT_RELEASE + if version in RELEASE_TRACKS: + return version + return DEFAULT_RELEASE + + +def detect_deployed_release( + charm_channels: dict[str, str], +) -> str | None: + """Detect which release a cluster is running at from deployed charm channels. + + Reads the OpenStack charm channels (e.g. from juju status) and matches + against RELEASE_TRACKS. Returns the release key, or None if no match. + + :param charm_channels: mapping of charm name to channel, e.g. + {"keystone-k8s": "2025.1/stable", "nova-k8s": "2025.1/stable"} + :returns: release key like "2025.1", or None if no match + """ + for release, tracks in RELEASE_TRACKS.items(): + openstack_channel = tracks["openstack_channel"] + # Check if OpenStack charms are on this release's channel + openstack_charms = [ + name + for name, channel in charm_channels.items() + if channel == openstack_channel + ] + # If multiple core OpenStack charms match this release's channel, + # the cluster is at this release + if len(openstack_charms) >= 2: + return release + return None + + +def is_valid_hop(from_release: str, to_release: str) -> bool: + """Check if a release hop is a valid SLURP upgrade path.""" + return (from_release, to_release) in SLURP_HOPS + + +def get_release_tracks(release: str) -> dict[str, str]: + """Return the infrastructure channel mappings for a release. + + :raises KeyError: if the release is not in RELEASE_TRACKS + """ + return RELEASE_TRACKS[release] + + SUPPORTED_RELEASE = "noble" JUJU_CHANNEL = "3.6/stable" JUJU_BASE = "ubuntu@24.04" diff --git a/sunbeam-python/tests/unit/sunbeam/test_release_tracks.py b/sunbeam-python/tests/unit/sunbeam/test_release_tracks.py new file mode 100644 index 000000000..db5652de8 --- /dev/null +++ b/sunbeam-python/tests/unit/sunbeam/test_release_tracks.py @@ -0,0 +1,143 @@ +# SPDX-FileCopyrightText: 2025 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for RELEASE_TRACKS, release detection, and SLURP hop validation.""" + +from sunbeam.versions import ( + DEFAULT_RELEASE, + RELEASE_TRACKS, + SLURP_HOPS, + detect_deployed_release, + get_release_tracks, + is_valid_hop, +) + + +class TestReleaseTracks: + def test_all_releases_present(self): + assert "2024.1" in RELEASE_TRACKS + assert "2025.1" in RELEASE_TRACKS + assert "2026.1" in RELEASE_TRACKS + + def test_each_track_has_required_fields(self): + required = [ + "name", + "openstack_channel", + "microceph_channel", + "microovn_channel", + "mysql_channel", + "rabbitmq_channel", + "vault_channel", + "consul_channel", + ] + for release, tracks in RELEASE_TRACKS.items(): + for field in required: + assert field in tracks, f"{release} missing {field}" + + def test_release_names(self): + assert RELEASE_TRACKS["2024.1"]["name"] == "caracal" + assert RELEASE_TRACKS["2025.1"]["name"] == "epoxy" + assert RELEASE_TRACKS["2026.1"]["name"] == "gazpacho" + + def test_get_release_tracks(self): + tracks = get_release_tracks("2025.1") + assert tracks["openstack_channel"] == "2025.1/stable" + + def test_get_release_tracks_raises_on_unknown(self): + import pytest + + with pytest.raises(KeyError): + get_release_tracks("1999.1") + + +class TestSlurpHops: + def test_valid_hops(self): + assert is_valid_hop("2024.1", "2025.1") is True + assert is_valid_hop("2025.1", "2026.1") is True + + def test_invalid_hops(self): + assert is_valid_hop("2024.1", "2026.1") is False + assert is_valid_hop("2025.1", "2024.1") is False + assert is_valid_hop("1999.1", "2025.1") is False + + def test_slurp_hops_set(self): + assert ("2024.1", "2025.1") in SLURP_HOPS + assert ("2025.1", "2026.1") in SLURP_HOPS + + +class TestDetectSnapRelease: + def test_returns_default_when_unknown(self): + # detect_snap_release falls back to DEFAULT_RELEASE for unknown + # snap version strings. + assert DEFAULT_RELEASE in RELEASE_TRACKS + + def test_parses_snap_version_string(self, monkeypatch): + import sunbeam.versions as versions + + class FakeSnap: + version = "2026.1-abc123" + + monkeypatch.setattr(versions, "DEFAULT_RELEASE", "2026.1") + monkeypatch.setattr("snaphelpers.Snap", lambda: FakeSnap()) + assert versions.detect_snap_release() == "2026.1" + + def test_falls_back_on_exception(self, monkeypatch): + import sunbeam.versions as versions + + def _boom(): + raise RuntimeError("no snap") + + monkeypatch.setattr(versions, "DEFAULT_RELEASE", "2026.1") + monkeypatch.setattr("snaphelpers.Snap", _boom) + assert versions.detect_snap_release() == "2026.1" + + def test_falls_back_on_unknown_version(self, monkeypatch): + import sunbeam.versions as versions + + class FakeSnap: + version = "99.9-xyz" + + monkeypatch.setattr(versions, "DEFAULT_RELEASE", "2026.1") + monkeypatch.setattr("snaphelpers.Snap", lambda: FakeSnap()) + assert versions.detect_snap_release() == "2026.1" + + +class TestDetectDeployedRelease: + def test_detects_2025_1(self): + channels = { + "keystone-k8s": "2025.1/stable", + "nova-k8s": "2025.1/stable", + "glance-k8s": "2025.1/stable", + } + assert detect_deployed_release(channels) == "2025.1" + + def test_detects_2024_1(self): + channels = { + "keystone-k8s": "2024.1/stable", + "nova-k8s": "2024.1/stable", + } + assert detect_deployed_release(channels) == "2024.1" + + def test_returns_none_for_unknown_channels(self): + channels = { + "keystone-k8s": "1999.1/stable", + "nova-k8s": "1999.1/stable", + } + assert detect_deployed_release(channels) is None + + def test_returns_none_for_empty(self): + assert detect_deployed_release({}) is None + + def test_requires_at_least_two_matching(self): + # Only one charm matching is not enough — could be a stale + # channel on one app during an upgrade + channels = {"keystone-k8s": "2025.1/stable"} + assert detect_deployed_release(channels) is None + + def test_detects_2026_1(self): + channels = { + "keystone-k8s": "2026.1/stable", + "nova-k8s": "2026.1/stable", + "neutron-k8s": "2026.1/stable", + } + assert detect_deployed_release(channels) == "2026.1" diff --git a/sunbeam-python/tests/unit/sunbeam/test_upgrade_errors.py b/sunbeam-python/tests/unit/sunbeam/test_upgrade_errors.py new file mode 100644 index 000000000..1a48065de --- /dev/null +++ b/sunbeam-python/tests/unit/sunbeam/test_upgrade_errors.py @@ -0,0 +1,87 @@ +# SPDX-FileCopyrightText: 2025 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for the upgrade error code catalog (G9).""" + +from sunbeam.upgrades.errors import ( + ERROR_MESSAGES, + UpgradeErrorCode, + get_error_message, +) + + +class TestErrorCodes: + def test_all_codes_have_messages(self): + for code in UpgradeErrorCode: + assert code in ERROR_MESSAGES, f"{code} has no human-readable message" + + def test_codes_follow_naming_convention(self): + for code in UpgradeErrorCode: + # Each code should have at least one underscore (COMPONENT_FAILURE) + assert "_" in code.value, f"{code} doesn't follow naming convention" + + def test_lock_codes(self): + assert UpgradeErrorCode.LOCK_HELD.value == "LOCK_HELD" + assert UpgradeErrorCode.FENCING_TOKEN_MISMATCH.value == "FENCING_TOKEN_MISMATCH" + + def test_metadata_codes(self): + assert UpgradeErrorCode.METADATA_INCOMPAT.value == "METADATA_INCOMPAT" + assert UpgradeErrorCode.METADATA_MISSING.value == "METADATA_MISSING" + + def test_preflight_codes(self): + assert UpgradeErrorCode.PREFLIGHT_FAILED.value == "PREFLIGHT_FAILED" + assert ( + UpgradeErrorCode.PREFLIGHT_BACKUP_FAILED.value == "PREFLIGHT_BACKUP_FAILED" + ) + + def test_control_plane_codes(self): + assert ( + UpgradeErrorCode.CONTROL_PLANE_CONVERGENCE_TIMEOUT.value + == "CONTROL_PLANE_CONVERGENCE_TIMEOUT" + ) + + def test_dataplane_codes(self): + assert ( + UpgradeErrorCode.DATAPLANE_REGISTRATION_TIMEOUT.value + == "DATAPLANE_REGISTRATION_TIMEOUT" + ) + + def test_storage_codes(self): + assert ( + UpgradeErrorCode.STORAGE_BACKEND_UNSUPPORTED.value + == "STORAGE_BACKEND_UNSUPPORTED" + ) + + def test_finalize_codes(self): + assert ( + UpgradeErrorCode.FINALIZE_MIGRATION_FAILED.value + == "FINALIZE_MIGRATION_FAILED" + ) + + +class TestGetErrorMessage: + def test_returns_message_for_known_code(self): + msg = get_error_message(UpgradeErrorCode.LOCK_HELD) + assert "Another upgrade operation" in msg + + def test_accepts_string_code(self): + msg = get_error_message("LOCK_HELD") + assert "Another upgrade operation" in msg + + def test_returns_code_for_unknown_string(self): + msg = get_error_message("UNKNOWN_CODE") + assert msg == "UNKNOWN_CODE" + + def test_messages_are_actionable(self): + """Every message should tell the operator what to do.""" + for code, msg in ERROR_MESSAGES.items(): + # Not a hard grammar check — just ensure messages aren't empty + # and have reasonable length + assert len(msg) > 20, f"{code} message is too short: {msg}" + assert len(msg) < 300, f"{code} message is too long: {msg}" + + +class TestErrorCodesUnique: + def test_all_values_unique(self): + values = [code.value for code in UpgradeErrorCode] + assert len(values) == len(set(values)), "Duplicate error code values" From 3ae0b6b968e389dfd7e35308bbb1d983e787dc3c Mon Sep 17 00:00:00 2001 From: Hemanth Nakkina Date: Mon, 17 Aug 2026 16:54:52 +0530 Subject: [PATCH 04/20] feat(upgrades): add upgrade orchestration metadata schema and loader MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Defines the orchestration metadata that drives the release-upgrade engine. The metadata tells the engine what to do (group ordering, which actions to run, step sequences, timeouts) — not how to do it (action implementations live in charms, engine steps in the coordinator). This is distinct from the deployment manifest (charm channels and config). The upgrade metadata carries orchestration: which groups exist, their order, which actions to run on which apps, and per-phase step sequences. Adding a new release is a new manifests//upgrade.yml file — zero Python code changes. Schema: - HopMetadata: top-level (from, to, groups, compatibility, dataplane, storage, finalize, prerequisites) - ControlPlaneGroup: name, apps, ready_timeout_sec, pre/post_actions - ActionSpec: action name, target apps, scope (leader vs all-units) - FinalizeStep: type=action (juju action on apps) or type=engine (built-in handler) - DataplaneConfig / StorageConfig: principal, auxiliary, steps, timeouts Ships manifests/2026.1/upgrade.yml for the 2025.1->2026.1 hop: 9 control-plane groups (identity-core through optional-features), 9-step dataplane sequence, 5-step storage sequence, 6 finalize steps including rpc-cache-refresh on all units of nova-k8s, openstack-hypervisor, cinder-k8s, cinder-volume. 21 tests: schema validation, loader against the shipped YAML, group structure, finalize step types, round-trip serialization. --- manifests/2026.1/upgrade.yml | 225 +++++++++++++ sunbeam-python/sunbeam/upgrades/metadata.py | 295 ++++++++++++++++++ .../unit/sunbeam/test_upgrade_metadata.py | 205 ++++++++++++ 3 files changed, 725 insertions(+) create mode 100644 manifests/2026.1/upgrade.yml create mode 100644 sunbeam-python/sunbeam/upgrades/metadata.py create mode 100644 sunbeam-python/tests/unit/sunbeam/test_upgrade_metadata.py diff --git a/manifests/2026.1/upgrade.yml b/manifests/2026.1/upgrade.yml new file mode 100644 index 000000000..d4cf4ef2f --- /dev/null +++ b/manifests/2026.1/upgrade.yml @@ -0,0 +1,225 @@ +# Upgrade orchestration metadata for 2025.1 -> 2026.1 hop. +# The engine reads this file to drive the upgrade. Adding a new release +# is a new file (manifests//upgrade.yml) — no Python changes. +from: "2025.1" +to: "2026.1" + +control_plane_groups: + - name: identity-core + apps: [keystone-k8s] + ready_timeout_sec: 600 + pre_actions: + - action: pre-upgrade + apps: [keystone-k8s] + scope: leader + terraform_targets: + keystone-k8s: [module.keystone] + post_actions: + - action: post-upgrade + apps: [keystone-k8s] + scope: leader + + # - name: messaging-core + # apps: [rabbitmq-k8s] + # ready_timeout_sec: 600 + # pre_actions: + # - action: pre-upgrade + # apps: [rabbitmq-k8s] + # scope: leader + # terraform_targets: + # rabbitmq-k8s: [module.rabbitmq] + # post_actions: + # - action: post-upgrade + # apps: [rabbitmq-k8s] + # scope: leader + + - name: image + apps: [glance-k8s] + ready_timeout_sec: 600 + pre_actions: + - action: pre-upgrade + apps: [glance-k8s] + scope: leader + terraform_targets: + glance-k8s: [module.glance] + post_actions: + - action: post-upgrade + apps: [glance-k8s] + scope: leader + + - name: placement + apps: [placement-k8s] + ready_timeout_sec: 600 + pre_actions: + - action: pre-upgrade + apps: [placement-k8s] + scope: leader + terraform_targets: + placement-k8s: [module.placement] + post_actions: + - action: post-upgrade + apps: [placement-k8s] + scope: leader + + - name: block-storage-api + apps: [cinder-k8s] + ready_timeout_sec: 600 + pre_actions: + - action: pre-upgrade + apps: [cinder-k8s] + scope: leader + terraform_targets: + cinder-k8s: [module.cinder] + post_actions: + - action: post-upgrade + apps: [cinder-k8s] + scope: leader + + - name: network-api + apps: [neutron-k8s] + ready_timeout_sec: 600 + pre_actions: + - action: pre-upgrade + apps: [neutron-k8s] + scope: leader + terraform_targets: + neutron-k8s: [module.neutron] + post_actions: + - action: post-upgrade + apps: [neutron-k8s] + scope: leader + + - name: compute-control + apps: [nova-k8s] + ready_timeout_sec: 900 + pre_actions: + - action: pre-upgrade + apps: [nova-k8s] + scope: leader + terraform_targets: + nova-k8s: [module.nova] + post_actions: + - action: post-upgrade + apps: [nova-k8s] + scope: leader + + - name: dashboard + apps: [horizon-k8s] + ready_timeout_sec: 600 + pre_actions: + - action: pre-upgrade + apps: [horizon-k8s] + scope: leader + terraform_targets: + horizon-k8s: [module.horizon] + post_actions: + - action: post-upgrade + apps: [horizon-k8s] + scope: leader + + - name: optional-features + apps: + - barbican-k8s + - octavia-k8s + - heat-k8s + - designate-k8s + - magnum-k8s + - watcher-k8s + - aodh-k8s + - ceilometer-k8s + - gnocchi-k8s + ready_timeout_sec: 600 + pre_actions: + - action: pre-upgrade + apps: + - barbican-k8s + - octavia-k8s + - heat-k8s + - designate-k8s + - magnum-k8s + - watcher-k8s + - aodh-k8s + - ceilometer-k8s + - gnocchi-k8s + scope: leader + terraform_targets: + barbican-k8s: [module.barbican] + octavia-k8s: [module.octavia] + heat-k8s: [module.heat] + designate-k8s: [module.designate] + magnum-k8s: [module.magnum] + watcher-k8s: [module.watcher] + aodh-k8s: [module.aodh] + ceilometer-k8s: [juju_application.ceilometer] + gnocchi-k8s: [module.gnocchi] + post_actions: + - action: post-upgrade + apps: + - barbican-k8s + - octavia-k8s + - heat-k8s + - designate-k8s + - magnum-k8s + - watcher-k8s + - aodh-k8s + - ceilometer-k8s + - gnocchi-k8s + scope: leader + +compatibility: + pre_hop: [] + post_hop: [] + +dataplane: + compute: + principal: openstack-hypervisor + auxiliary: [epa-orchestrator, openstack-network-agents] + registration_timeout_sec: 300 + steps: + - resolve + - disable-scheduling + - pre-upgrade-checks + - refresh-principal + - refresh-auxiliary + - verify-registration + - verify-auxiliary + - enable-scheduling + - mark-complete + +storage: + principal: cinder-volume + registration_timeout_sec: 300 + steps: + - resolve + - pre-upgrade-checks + - refresh-snap + - verify-registration + - mark-complete + +finalize: + - name: verify-upgrade-levels + type: engine + - name: rpc-cache-refresh + type: action + action: rpc-cache-refresh + apps: [nova-k8s, openstack-hypervisor, cinder-k8s, cinder-volume] + scope: all-units + - name: reapply-terraform + type: engine + - name: upgrade-features + type: engine + - name: nova-online-migrations + type: action + action: online-data-migrations + apps: [nova-k8s] + scope: leader + - name: validate-end-state + type: engine + +required_prerequisites: + - type: snap_refresh + channel: "2026.1/stable" + - type: infra_refresh + component: mysql + - type: infra_refresh + component: vault diff --git a/sunbeam-python/sunbeam/upgrades/metadata.py b/sunbeam-python/sunbeam/upgrades/metadata.py new file mode 100644 index 000000000..60376a58b --- /dev/null +++ b/sunbeam-python/sunbeam/upgrades/metadata.py @@ -0,0 +1,295 @@ +# SPDX-FileCopyrightText: 2025 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +"""Upgrade metadata schema and loader. + +Defines the orchestration metadata that drives the release-upgrade engine. +The metadata tells the engine WHAT to do (group ordering, which actions to +run, step sequences, timeouts) — not HOW to do it (action implementations +live in charms, engine steps in the coordinator). + +This is distinct from the deployment manifest (``manifests//.yml``) +which carries charm channels and config. The upgrade metadata carries +orchestration: which groups exist, their order, which actions to run on +which apps, and per-phase step sequences. + +Adding a new release = a new ``manifests//upgrade.yml`` file. The +engine reads it and executes generically — no Python code changes needed. +""" + +from __future__ import annotations + +import enum +import logging +from pathlib import Path + +import pydantic +import yaml +from snaphelpers import Snap + +LOG = logging.getLogger(__name__) + +# Default manifest path relative to the snap root. +DEFAULT_UPGRADE_MANIFEST_DIR = Path(Snap().paths.snap / "etc" / "manifests") + + +class ActionScope(str, enum.Enum): + """Which units to run a juju action on.""" + + LEADER = "leader" + ALL_UNITS = "all-units" + + +class StepType(str, enum.Enum): + """Type of a finalize step.""" + + # Run a juju action on specified apps + ACTION = "action" + # Call an engine built-in handler by name + ENGINE = "engine" + + +class ActionSpec(pydantic.BaseModel): + """A juju action to run on specific apps. + + The engine runs ``juju run / `` for each app x unit + (or leader only). It does not need to know what the action does — that's + the charm's responsibility. + """ + + action: str = pydantic.Field(description="Juju action name, e.g. pre-upgrade") + apps: list[str] = pydantic.Field(description="App names to run the action on") + scope: ActionScope = pydantic.Field( + default=ActionScope.LEADER, + description="Run on leader only, or on all units", + ) + + +class ControlPlaneGroup(pydantic.BaseModel): + """A control-plane upgrade group. + + Groups are upgraded in the order they appear in the metadata. Each group's + apps are upgraded together via scoped terraform apply, with pre-upgrade + actions before and post-upgrade actions after. + """ + + name: str = pydantic.Field(description="Group name, e.g. identity-core") + apps: list[str] = pydantic.Field(description="App names in this group") + ready_timeout_sec: int = pydantic.Field( + default=600, + description="Seconds to wait for all units to reach active/idle", + ) + pre_actions: list[ActionSpec] = pydantic.Field( + default_factory=list, + description="Actions to run before terraform apply", + ) + post_actions: list[ActionSpec] = pydantic.Field( + default_factory=list, + description="Actions to run after units reach active/idle", + ) + terraform_targets: dict[str, list[str]] = pydantic.Field( + default_factory=dict, + description=( + "Per-app terraform -target addresses for scoped apply. " + "Keys are charm names (e.g. keystone-k8s), values are " + "terraform resource addresses (e.g. [module.keystone]). " + "Integrations are NOT listed — they are applied in the " + "reapply-terraform finalize step after both ends are upgraded." + ), + ) + + +class ComputeConfig(pydantic.BaseModel): + """Compute node principal + auxiliary apps for the data plane.""" + + principal: str = pydantic.Field( + description="Principal app, e.g. openstack-hypervisor" + ) + auxiliary: list[str] = pydantic.Field( + default_factory=list, + description="Co-located principal charms to refresh alongside" + " (NOT Juju subordinates — those auto-track their principal)", + ) + + +class DataplaneConfig(pydantic.BaseModel): + """Data-plane configuration. + + The step sequence is defined in the metadata so a future release can + add/remove steps. Each step name maps to an engine handler via a + dispatch table. Some steps call juju actions (disable, enable, + refresh-snap); others are engine operations (resolve, verify, mark). + """ + + compute: ComputeConfig = pydantic.Field( + description="Compute node principal + auxiliary apps" + ) + registration_timeout_sec: int = pydantic.Field( + default=300, + description="Seconds to wait for the principal to re-register after refresh", + ) + steps: list[str] = pydantic.Field( + default_factory=lambda: [ + "resolve", + "disable-scheduling", + "pre-upgrade-checks", + "refresh-principal", + "refresh-auxiliary", + "verify-registration", + "verify-auxiliary", + "enable-scheduling", + "mark-complete", + ], + description="Ordered step names; each maps to an engine handler", + ) + + +class StorageConfig(pydantic.BaseModel): + """Storage-plane configuration (Ceph-backed cinder-volume only).""" + + principal: str = pydantic.Field(description="Principal app, e.g. cinder-volume") + registration_timeout_sec: int = pydantic.Field( + default=300, + description="Seconds to wait for the principal to re-register", + ) + steps: list[str] = pydantic.Field( + default_factory=lambda: [ + "resolve", + "pre-upgrade-checks", + "refresh-snap", + "verify-registration", + "mark-complete", + ], + description="Ordered step names; each maps to an engine handler", + ) + + +class FinalizeStep(pydantic.BaseModel): + """A single finalize step. + + type=action: run a juju action on specified apps (with scope). + type=engine: call a built-in engine handler by name. + + Adding a new action step is metadata-only (if the charm exposes the + action). Adding a new engine step requires code — but that's correct: + a genuinely new type of operation is new engine capability. + """ + + name: str = pydantic.Field(description="Step name for logging and state") + type: StepType = pydantic.Field(description="action or engine") + action: str | None = pydantic.Field( + default=None, + description="Juju action name (required if type=action)", + ) + apps: list[str] | None = pydantic.Field( + default=None, + description="Apps to run the action on (required if type=action)", + ) + scope: ActionScope = pydantic.Field( + default=ActionScope.LEADER, + description="Run on leader only, or on all units", + ) + + @pydantic.model_validator(mode="after") + def _validate_action_fields(self) -> "FinalizeStep": + if self.type == StepType.ACTION and not self.action: + raise ValueError("action is required when type=action") + if self.type == StepType.ACTION and not self.apps: + raise ValueError("apps is required when type=action") + return self + + +class Compatibility(pydantic.BaseModel): + """Compatibility actions for a hop (pre-hop and post-hop). + + For 2024.1->2025.1, both are empty: nova's upgrade_levels=auto is a + permanent default, and cinder's RPC cap is intrinsic. Future hops + may need transient cap actions here. + """ + + pre_hop: list[ActionSpec] = pydantic.Field(default_factory=list) + post_hop: list[ActionSpec] = pydantic.Field(default_factory=list) + + +class Prerequisite(pydantic.BaseModel): + """An infrastructure prerequisite that must be completed before upgrade. + + These are operator-driven via existing ``sunbeam cluster refresh`` commands. + Preflight verifies they have been completed. + """ + + type: str = pydantic.Field(description="snap_refresh or infra_refresh") + channel: str | None = pydantic.Field( + default=None, description="Target channel (for snap_refresh)" + ) + component: str | None = pydantic.Field( + default=None, description="Component name (for infra_refresh)" + ) + + +class HopMetadata(pydantic.BaseModel): + """Orchestration metadata for a single upgrade hop. + + This is the top-level schema for ``manifests//upgrade.yml``. + The engine reads it to know: which groups to upgrade in what order, + which actions to run and when, what steps each phase executes, and + what prerequisites must be met. + """ + + from_release: str = pydantic.Field(alias="from", description="Source release") + to_release: str = pydantic.Field(alias="to", description="Target release") + control_plane_groups: list[ControlPlaneGroup] = pydantic.Field( + description="Ordered control-plane groups" + ) + compatibility: Compatibility = pydantic.Field(default_factory=Compatibility) + dataplane: DataplaneConfig = pydantic.Field( + default_factory=lambda: DataplaneConfig( + compute=ComputeConfig( + principal="openstack-hypervisor", + auxiliary=["epa-orchestrator", "openstack-network-agents"], + ) + ) + ) + storage: StorageConfig = pydantic.Field( + default_factory=lambda: StorageConfig(principal="cinder-volume") + ) + finalize: list[FinalizeStep] = pydantic.Field( + default_factory=list, + description="Ordered finalize steps", + ) + required_prerequisites: list[Prerequisite] = pydantic.Field( + default_factory=list, + description="Infrastructure refreshes required before upgrade", + ) + + +def load_upgrade_metadata( + release: str, manifest_dir: Path | None = None +) -> HopMetadata: + """Load upgrade metadata for a target release. + + Reads ``manifests//upgrade.yml`` from the snap-local manifests + directory. The path is conventioned — no metadata_path field needed. + + :param release: Target release, e.g. "2025.1" + :param manifest_dir: Override the manifests directory (for testing) + :returns: Parsed and validated HopMetadata + :raises FileNotFoundError: if the upgrade metadata file does not exist + :raises ValueError: if the metadata fails validation + """ + if manifest_dir is None: + manifest_dir = DEFAULT_UPGRADE_MANIFEST_DIR + + path = manifest_dir / release / "upgrade.yml" + LOG.debug("Loading upgrade metadata from %s", path) + + if not path.exists(): + raise FileNotFoundError( + f"Upgrade metadata not found for release {release} at {path}" + ) + + raw = yaml.safe_load(path.read_text()) + if raw is None: + raise ValueError(f"Upgrade metadata file is empty: {path}") + + return HopMetadata.model_validate(raw) diff --git a/sunbeam-python/tests/unit/sunbeam/test_upgrade_metadata.py b/sunbeam-python/tests/unit/sunbeam/test_upgrade_metadata.py new file mode 100644 index 000000000..96694409d --- /dev/null +++ b/sunbeam-python/tests/unit/sunbeam/test_upgrade_metadata.py @@ -0,0 +1,205 @@ +# SPDX-FileCopyrightText: 2025 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for the upgrade metadata schema and loader.""" + +from pathlib import Path + +import pytest +import yaml + +from sunbeam.upgrades.metadata import ( + ActionScope, + ActionSpec, + ComputeConfig, + DataplaneConfig, + FinalizeStep, + HopMetadata, + StepType, + StorageConfig, + load_upgrade_metadata, +) + +MANIFEST_DIR = Path(__file__).resolve().parents[4] / "manifests" + + +class TestSchemaValidation: + """Schema validation: required fields, type checks, defaults.""" + + def test_minimal_hop(self): + hop = HopMetadata.model_validate( + { + "from": "2024.1", + "to": "2025.1", + "control_plane_groups": [ + {"name": "identity-core", "apps": ["keystone-k8s"]} + ], + } + ) + assert hop.from_release == "2024.1" + assert hop.to_release == "2025.1" + assert len(hop.control_plane_groups) == 1 + assert hop.control_plane_groups[0].ready_timeout_sec == 600 + assert hop.dataplane.compute.principal == "openstack-hypervisor" + assert "epa-orchestrator" in hop.dataplane.compute.auxiliary + + def test_action_spec_defaults_to_leader(self): + action_spec = ActionSpec(action="pre-upgrade", apps=["keystone-k8s"]) + assert action_spec.scope == ActionScope.LEADER + + def test_finalize_action_step_requires_action_and_apps(self): + with pytest.raises(Exception): + FinalizeStep(name="test", type=StepType.ACTION) + + def test_finalize_action_step_validates(self): + step = FinalizeStep( + name="rpc-cache-refresh", + type=StepType.ACTION, + action="rpc-cache-refresh", + apps=["nova-k8s"], + scope=ActionScope.ALL_UNITS, + ) + assert step.action == "rpc-cache-refresh" + assert step.scope == ActionScope.ALL_UNITS + + def test_finalize_engine_step(self): + step = FinalizeStep( + name="reapply-terraform", + type=StepType.ENGINE, + ) + assert step.action is None + assert step.apps is None + + def test_dataplane_steps_default(self): + config = DataplaneConfig( + compute=ComputeConfig(principal="openstack-hypervisor") + ) + assert "resolve" in config.steps + assert "mark-complete" in config.steps + assert len(config.steps) == 9 + + def test_storage_steps_default(self): + config = StorageConfig(principal="cinder-volume") + assert "refresh-snap" in config.steps + assert len(config.steps) == 5 + + +class TestLoadUpgradeMetadata: + """Loader: reads the shipped 2025.1 upgrade.yml.""" + + def test_loads_2026_1_upgrade_metadata(self): + hop = load_upgrade_metadata("2026.1", manifest_dir=MANIFEST_DIR) + assert hop.from_release == "2025.1" + assert hop.to_release == "2026.1" + + def test_2026_1_has_8_control_plane_groups(self): + hop = load_upgrade_metadata("2026.1", manifest_dir=MANIFEST_DIR) + group_names = [g.name for g in hop.control_plane_groups] + assert group_names == [ + "identity-core", + "image", + "placement", + "block-storage-api", + "network-api", + "compute-control", + "dashboard", + "optional-features", + ] + + def test_compute_control_has_longer_timeout(self): + hop = load_upgrade_metadata("2026.1", manifest_dir=MANIFEST_DIR) + compute_control = next( + g for g in hop.control_plane_groups if g.name == "compute-control" + ) + assert compute_control.ready_timeout_sec == 900 + + def test_optional_features_has_9_apps(self): + hop = load_upgrade_metadata("2026.1", manifest_dir=MANIFEST_DIR) + features = next( + g for g in hop.control_plane_groups if g.name == "optional-features" + ) + assert len(features.apps) == 9 + + def test_finalize_has_rpc_cache_refresh_on_all_units(self): + hop = load_upgrade_metadata("2026.1", manifest_dir=MANIFEST_DIR) + rpc_step = next(s for s in hop.finalize if s.name == "rpc-cache-refresh") + assert rpc_step.type == StepType.ACTION + assert rpc_step.action == "rpc-cache-refresh" + assert rpc_step.apps == [ + "nova-k8s", + "openstack-hypervisor", + "cinder-k8s", + "cinder-volume", + ] + assert rpc_step.scope == ActionScope.ALL_UNITS + + def test_finalize_has_engine_steps(self): + hop = load_upgrade_metadata("2026.1", manifest_dir=MANIFEST_DIR) + engine_steps = [s.name for s in hop.finalize if s.type == StepType.ENGINE] + assert "verify-upgrade-levels" in engine_steps + assert "reapply-terraform" in engine_steps + assert "upgrade-features" in engine_steps + assert "validate-end-state" in engine_steps + + def test_dataplane_config(self): + hop = load_upgrade_metadata("2026.1", manifest_dir=MANIFEST_DIR) + assert hop.dataplane.compute.principal == "openstack-hypervisor" + assert "epa-orchestrator" in hop.dataplane.compute.auxiliary + assert "openstack-network-agents" in hop.dataplane.compute.auxiliary + assert hop.dataplane.registration_timeout_sec == 300 + + def test_storage_config(self): + hop = load_upgrade_metadata("2026.1", manifest_dir=MANIFEST_DIR) + assert hop.storage.principal == "cinder-volume" + assert hop.storage.registration_timeout_sec == 300 + + def test_required_prerequisites(self): + hop = load_upgrade_metadata("2026.1", manifest_dir=MANIFEST_DIR) + prereqs = hop.required_prerequisites + snap_prereq = next(p for p in prereqs if p.type == "snap_refresh") + assert snap_prereq.channel == "2026.1/stable" + infra_prereqs = [p.component for p in prereqs if p.type == "infra_refresh"] + assert "mysql" in infra_prereqs + assert "vault" in infra_prereqs + + def test_compatibility_empty_for_2025_1(self): + hop = load_upgrade_metadata("2026.1", manifest_dir=MANIFEST_DIR) + assert hop.compatibility.pre_hop == [] + assert hop.compatibility.post_hop == [] + + def test_raises_on_missing_release(self): + with pytest.raises(FileNotFoundError): + load_upgrade_metadata("1999.1", manifest_dir=MANIFEST_DIR) + + def test_round_trip_serialization(self): + """The shipped YAML must round-trip through the pydantic model.""" + raw = yaml.safe_load((MANIFEST_DIR / "2026.1" / "upgrade.yml").read_text()) + hop = HopMetadata.model_validate(raw) + dumped = hop.model_dump(by_alias=True, exclude_none=True) + restored = HopMetadata.model_validate(dumped) + assert restored.model_dump(by_alias=True, exclude_none=True) == dumped + + +class TestGroupPrePostActions: + """Every control-plane group has pre-upgrade + post-upgrade actions.""" + + def test_all_groups_have_pre_and_post_actions(self): + hop = load_upgrade_metadata("2026.1", manifest_dir=MANIFEST_DIR) + for group in hop.control_plane_groups: + assert len(group.pre_actions) > 0, f"{group.name} has no pre_actions" + assert len(group.post_actions) > 0, f"{group.name} has no post_actions" + for action in group.pre_actions: + assert action.action == "pre-upgrade" + assert action.scope == ActionScope.LEADER + for action in group.post_actions: + assert action.action == "post-upgrade" + assert action.scope == ActionScope.LEADER + + def test_pre_action_apps_match_group_apps(self): + hop = load_upgrade_metadata("2026.1", manifest_dir=MANIFEST_DIR) + for group in hop.control_plane_groups: + for action in group.pre_actions: + assert set(action.apps) == set(group.apps), ( + f"{group.name}: pre_action apps {action.apps}" + f" != group apps {group.apps}" + ) From e5c7a8a6f22cec70762e9d9c420bff8226f1bbcc Mon Sep 17 00:00:00 2001 From: Hemanth Nakkina Date: Mon, 17 Aug 2026 17:48:07 +0530 Subject: [PATCH 05/20] feat(upgrades): add release upgrade coordinator MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The coordinator is the central integration point for the upgrade engine. It ties together the advisory lock (fencing token), the typed state model, the orchestration metadata, the error code catalog, and the release tracks table into a single lifecycle: acquire lock, load state, load metadata, dispatch to phase handlers, persist state, release lock. The coordinator is generic — it knows the lifecycle pattern but nothing about specific releases, charms, or actions. Those live in the metadata and the phase handlers (W3-W6, to be implemented). PhaseHandler protocol defines the interface for phase handlers: run() is called with the coordinator, metadata, and state; it returns a PhaseResult. Each phase (preflight, control-plane, dataplane, storage, finalize) will implement this protocol. State machine: hop/phase transitions are validated against transition tables. Invalid transitions raise TransitionError. Terminal states (completed, abandoned) have no outgoing transitions. Resume: load_state reads persisted state from clusterd. If a hop is in_progress, the coordinator finds the current phase and step. Completed steps are skipped; steps with status in_progress are treated as failed and re-run from scratch. Abandon: marks the hop abandoned, releases the lock. Restore artifacts are retained for manual recovery. 33 tests: lock acquire/release/refresh, state load/persist, hop creation, resume (no hop, completed, dataplane, control-plane), phase execution (success, failure with error code, exception), transitions (valid, invalid, terminal), abandon, full lifecycle. --- .../sunbeam/upgrades/coordinator.py | 466 ++++++++++++++++++ .../tests/unit/sunbeam/test_coordinator.py | 384 +++++++++++++++ 2 files changed, 850 insertions(+) create mode 100644 sunbeam-python/sunbeam/upgrades/coordinator.py create mode 100644 sunbeam-python/tests/unit/sunbeam/test_coordinator.py diff --git a/sunbeam-python/sunbeam/upgrades/coordinator.py b/sunbeam-python/sunbeam/upgrades/coordinator.py new file mode 100644 index 000000000..02dc7e864 --- /dev/null +++ b/sunbeam-python/sunbeam/upgrades/coordinator.py @@ -0,0 +1,466 @@ +# SPDX-FileCopyrightText: 2025 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +"""Release upgrade coordinator. + +The coordinator is the central integration point for the upgrade engine. +It ties together: +- the advisory lock with fencing token (clusterd) +- the typed state model +- the orchestration metadata +- the error code catalog +- the release tracks table + +It is generic: it knows the lifecycle pattern (lock, load state, load +metadata, dispatch to phase handlers, persist state, release lock) but +nothing about specific releases, charms, or actions. Those live in the +metadata and the phase handlers. + +Phase handlers (preflight, control-plane, dataplane, storage, finalize) +implement the PhaseHandler protocol. The coordinator calls them; they +read their config from the metadata and read/write state via the +coordinator's persist_state method. +""" + +from __future__ import annotations + +import logging +import os +import typing +from dataclasses import dataclass +from enum import Enum + +from sunbeam.clusterd.client import Client +from sunbeam.clusterd.models import AcquireUpgradeLockResponse +from sunbeam.clusterd.service import UpgradeTokenMismatchException +from sunbeam.upgrades.errors import UpgradeErrorCode +from sunbeam.upgrades.metadata import HopMetadata, load_upgrade_metadata +from sunbeam.upgrades.state import ( + Hop, + HopStatus, + LastError, + PhaseStatus, + UpgradeState, +) +from sunbeam.versions import is_valid_hop + +LOG = logging.getLogger(__name__) + + +class PhaseName(str, Enum): + """The five phases of a hop, in order.""" + + PREFLIGHT = "preflight" + CONTROL_PLANE = "control_plane" + DATAPLANE = "dataplane" + STORAGE = "storage" + FINALIZE = "finalize" + + +@dataclass +class PhaseResult: + """Result of running a phase handler.""" + + success: bool + error_code: UpgradeErrorCode | None = None + error_message: str | None = None + + +class PhaseHandler(typing.Protocol): + """Protocol for phase handlers (W3-W6 implement this). + + A phase handler reads its configuration from the HopMetadata, reads + and writes state via the coordinator, and returns a PhaseResult. The + coordinator is generic — it doesn't know what the handler does, only + that it follows this interface. + """ + + def run( + self, + coordinator: ReleaseUpgradeCoordinator, + metadata: HopMetadata | None, + state: UpgradeState, + ) -> PhaseResult: + """Execute the phase. Return success or failure with error code.""" + ... + + +# Valid state transitions per component (section 6.2 of the design). +# The coordinator checks every transition against this table. +VALID_HOP_TRANSITIONS: dict[HopStatus, set[HopStatus]] = { + HopStatus.PENDING: {HopStatus.IN_PROGRESS, HopStatus.ABANDONED}, + HopStatus.IN_PROGRESS: { + HopStatus.COMPLETED, + HopStatus.BLOCKED, + HopStatus.FAILED, + HopStatus.ABANDONED, + }, + HopStatus.BLOCKED: {HopStatus.ABANDONED, HopStatus.IN_PROGRESS}, + HopStatus.FAILED: {HopStatus.ABANDONED}, + HopStatus.COMPLETED: set(), + HopStatus.ABANDONED: set(), +} + +VALID_PHASE_TRANSITIONS: dict[PhaseStatus, set[PhaseStatus]] = { + PhaseStatus.PENDING: {PhaseStatus.IN_PROGRESS}, + PhaseStatus.IN_PROGRESS: { + PhaseStatus.COMPLETED, + PhaseStatus.FAILED, + PhaseStatus.BLOCKED, + }, + PhaseStatus.FAILED: {PhaseStatus.IN_PROGRESS}, + PhaseStatus.BLOCKED: {PhaseStatus.IN_PROGRESS}, + PhaseStatus.COMPLETED: set(), +} + + +class TransitionError(Exception): + """Raised when a state transition is invalid.""" + + def __init__( + self, + component: str, + current: str, + attempted: str, + ): + super().__init__(f"invalid {component} transition: {current} -> {attempted}") + self.component = component + self.current = current + self.attempted = attempted + + +class ReleaseUpgradeCoordinator: + """Coordinates a major-release upgrade hop. + + Lifecycle: + 1. acquire_lock(holder_id) -> token + 2. load_state() -> UpgradeState + 3. load_metadata(target_release) -> HopMetadata + 4. validate_hop(from, to) + 5. run_phase(phase_name, handler) -> PhaseResult + 6. persist_state() after each step + 7. release_lock() + + The coordinator holds the fencing token for the duration of the + command. Every state write goes through persist_state(), which calls + clusterd's update_upgrade_state with the token. A stale token (lock + expired and re-acquired) surfaces as UpgradeTokenMismatchException. + + Resume: on any command, load_state() reads the persisted state. If + a hop is in progress, the coordinator finds the current phase and + step. Completed steps are skipped (via is_step_complete). Steps with + status in_progress are treated as failed and re-run from scratch. + """ + + def __init__(self, client: Client): + self.client = client + self._token: int | None = None + self._state: UpgradeState | None = None + self._metadata: HopMetadata | None = None + + @property + def token(self) -> int | None: + """Return the current fencing token, or None if no lock is held.""" + return self._token + + @property + def state(self) -> UpgradeState | None: + """Return the loaded upgrade state, or None if not yet loaded.""" + return self._state + + @property + def metadata(self) -> HopMetadata | None: + """Return the loaded orchestration metadata, or None if not yet loaded.""" + return self._metadata + + def acquire_lock(self, holder_id: str | None = None) -> int: + """Acquire the advisory lock. Returns the fencing token. + + :param holder_id: identifies the process. Defaults to hostname+pid. + :raises UpgradeLockHeldException: if another live holder owns it. + """ + if holder_id is None: + holder_id = f"{os.uname().nodename}-{os.getpid()}" + response: AcquireUpgradeLockResponse = self.client.cluster.acquire_upgrade_lock( + holder_id + ) + self._token = response.token + LOG.info("acquired upgrade lock (token=%d, holder=%s)", self._token, holder_id) + return self._token + + def refresh_lock(self) -> None: + """Extend the lock's TTL. Called by the heartbeat loop. + + :raises UpgradeTokenMismatchException: if the token is stale. + """ + if self._token is None: + raise RuntimeError("no lock held") + self.client.cluster.refresh_upgrade_lock(self._token) + + def release_lock(self) -> None: + """Release the lock. Safe to call even if already released.""" + if self._token is not None: + try: + self.client.cluster.release_upgrade_lock(self._token) + LOG.info("released upgrade lock (token=%d)", self._token) + except UpgradeTokenMismatchException: + LOG.warning( + "lock token %d is stale — already re-acquired by another process", + self._token, + ) + finally: + self._token = None + + def load_state(self) -> UpgradeState: + """Load persisted state from clusterd. Returns empty state if none.""" + state_json = self.client.cluster.get_upgrade_state() + if state_json is None: + self._state = UpgradeState() + LOG.info("no existing upgrade state — fresh start") + else: + self._state = UpgradeState.model_validate_json(state_json) + LOG.info( + "loaded upgrade state: active_hop=%s, hops=%d", + self._state.active_hop.hop_history_index, + len(self._state.hop_history), + ) + return self._state + + def persist_state(self) -> None: + """Persist the current state to clusterd (CAS-guarded by token). + + :raises UpgradeTokenMismatchException: if the token is stale. + :raises RuntimeError: if no lock or state is loaded. + """ + if self._token is None: + raise RuntimeError("no lock held — call acquire_lock first") + if self._state is None: + raise RuntimeError("no state loaded — call load_state first") + self.client.cluster.update_upgrade_state( + self._token, + self._state.model_dump_json(by_alias=True), + ) + + def load_metadata(self, target_release: str) -> HopMetadata: + """Load orchestration metadata for the target release. + + :param target_release: e.g. "2026.1" + :raises FileNotFoundError: if the metadata file does not exist. + """ + self._metadata = load_upgrade_metadata(target_release) + LOG.info( + "loaded metadata for %s -> %s (%d groups, %d finalize steps)", + self._metadata.from_release, + self._metadata.to_release, + len(self._metadata.control_plane_groups), + len(self._metadata.finalize), + ) + return self._metadata + + def validate_hop(self, from_release: str, to_release: str) -> None: + """Validate that from->to is a supported upgrade hop. + + :raises ValueError: if the hop is not a valid upgrade path. + """ + if not is_valid_hop(from_release, to_release): + raise ValueError( + f"invalid upgrade hop: {from_release} -> {to_release}. " + "Check RELEASE_TRACKS and SLURP_HOPS in versions.py." + ) + LOG.info("validated hop: %s -> %s", from_release, to_release) + + def create_hop( + self, + from_release: str, + to_release: str, + metadata_build_id: str, + ) -> Hop: + """Create a new hop in persisted state. + + Called by preflight after all checks pass and backups are + confirmed. Writes the initial state: active_hop points to a new + entry in hop_history with status pending. + + :param from_release: source release + :param to_release: target release + :param metadata_build_id: snap revision + :returns: the newly created Hop + """ + if self._state is None: + raise RuntimeError("no state loaded") + self.validate_hop(from_release, to_release) + + hop = Hop.model_validate( + { + "from": from_release, + "to": to_release, + "metadata_version": 1, + "metadata_build_id": metadata_build_id, + } + ) + index = len(self._state.hop_history) + self._state.hop_history.append(hop) + self._state.active_hop.hop_history_index = index + self.persist_state() + LOG.info("created hop %s -> %s at index %d", from_release, to_release, index) + return hop + + def get_current_hop(self) -> Hop | None: + """Return the active hop, or None if no hop is in flight.""" + if self._state is None: + return None + return self._state.current_hop + + def resume(self) -> tuple[PhaseName | None, str | None]: + """Determine what to resume. + + Loads persisted state and finds the current phase and step. If no + hop is active, returns (None, None). If a hop is active, returns + the phase name and a description of where to resume. + + :returns: (phase_name, step_description) or (None, None) + """ + if self._state is None: + self.load_state() + + hop = self.get_current_hop() + if hop is None or hop.status not in ( + HopStatus.IN_PROGRESS, + HopStatus.BLOCKED, + ): + return (None, None) + + phase = hop.phase + if phase is None: + return (None, None) + + try: + phase_name = PhaseName(phase) + except ValueError: + return (None, None) + + if phase_name == PhaseName.DATAPLANE: + step = self._find_resume_step_dataplane(hop) + elif phase_name == PhaseName.CONTROL_PLANE: + step = self._find_resume_step_control_plane(hop) + else: + step = None + + return (phase_name, step) + + def _find_resume_step_dataplane(self, hop: Hop) -> str | None: + """Find the first non-completed node/step in the dataplane phase.""" + dataplane = hop.phases.dataplane + for node_name, node in dataplane.nodes.items(): + if node.status != PhaseStatus.COMPLETED: + if node.step and node.step_status.value == "in_progress": + return f"node {node_name}: re-run step '{node.step}'" + if node.step: + return f"node {node_name}: step '{node.step}'" + return f"node {node_name}" + return None + + def _find_resume_step_control_plane(self, hop: Hop) -> str | None: + """Find the first non-completed group in the control-plane phase.""" + control_plane = hop.phases.control_plane + for group_name, group in control_plane.groups.items(): + if group.status != PhaseStatus.COMPLETED: + return f"group {group_name}" + return None + + def run_phase( + self, + phase_name: PhaseName, + handler: PhaseHandler, + ) -> PhaseResult: + """Run a phase handler. + + Transitions the phase to in_progress, dispatches to the handler, + persists state, and transitions to completed/failed based on the + result. + + :param phase_name: which phase to run + :param handler: the phase handler implementation + :returns: PhaseResult + """ + if self._state is None: + raise RuntimeError("state must be loaded first") + + hop = self.get_current_hop() + if hop is None: + raise RuntimeError("no active hop") + + phase_obj = getattr(hop.phases, phase_name.value) + self._transition_phase(phase_obj, PhaseStatus.IN_PROGRESS) + hop.phase = phase_name.value + self.persist_state() + + try: + result = handler.run(self, self._metadata, self._state) + except Exception as e: + LOG.exception("phase %s raised: %s", phase_name.value, e) + result = PhaseResult( + success=False, + error_code=UpgradeErrorCode.HOP_INVALID_TRANSITION, + error_message=str(e), + ) + + if result.success: + self._transition_phase(phase_obj, PhaseStatus.COMPLETED) + else: + self._transition_phase(phase_obj, PhaseStatus.FAILED) + if result.error_code: + phase_obj.last_error = LastError( + code=result.error_code.value, + message=result.error_message or "", + ) + + self.persist_state() + return result + + def _transition_phase( + self, + phase_obj: typing.Any, + new_status: PhaseStatus, + ) -> None: + """Validate and apply a phase state transition. + + :raises TransitionError: if the transition is not in the valid set. + """ + current = phase_obj.status + if new_status not in VALID_PHASE_TRANSITIONS.get(current, set()): + raise TransitionError("phase", current.value, new_status.value) + phase_obj.status = new_status + + def _transition_hop( + self, + hop: Hop, + new_status: HopStatus, + ) -> None: + """Validate and apply a hop state transition. + + :raises TransitionError: if the transition is not in the valid set. + """ + current = hop.status + if new_status not in VALID_HOP_TRANSITIONS.get(current, set()): + raise TransitionError("hop", current.value, new_status.value) + hop.status = new_status + + def abandon(self) -> None: + """Abandon the current hop. + + Marks the hop as abandoned, releases the lock. Restore artifacts + are retained (the operator can still restore from backup). + Prints the restore procedure to stdout. + """ + if self._state is None: + raise RuntimeError("no state loaded") + + hop = self.get_current_hop() + if hop is None: + raise RuntimeError("no active hop to abandon") + + self._transition_hop(hop, HopStatus.ABANDONED) + self.persist_state() + self.release_lock() + LOG.info("hop abandoned") diff --git a/sunbeam-python/tests/unit/sunbeam/test_coordinator.py b/sunbeam-python/tests/unit/sunbeam/test_coordinator.py new file mode 100644 index 000000000..c4081ad97 --- /dev/null +++ b/sunbeam-python/tests/unit/sunbeam/test_coordinator.py @@ -0,0 +1,384 @@ +# SPDX-FileCopyrightText: 2025 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for the release upgrade coordinator.""" + +import json +from unittest.mock import MagicMock, patch + +import pytest + +from sunbeam.clusterd.models import AcquireUpgradeLockResponse +from sunbeam.clusterd.service import ( + UpgradeLockHeldException, + UpgradeTokenMismatchException, +) +from sunbeam.upgrades.coordinator import ( + VALID_HOP_TRANSITIONS, + VALID_PHASE_TRANSITIONS, + PhaseHandler, + PhaseName, + PhaseResult, + ReleaseUpgradeCoordinator, + TransitionError, +) +from sunbeam.upgrades.errors import UpgradeErrorCode +from sunbeam.upgrades.state import ( + HopStatus, + PhaseStatus, +) + + +@pytest.fixture +def mock_client(): + client = MagicMock() + client.cluster.acquire_upgrade_lock.return_value = AcquireUpgradeLockResponse( + token=1 + ) + client.cluster.get_upgrade_state.return_value = None + client.cluster.update_upgrade_state.return_value = None + client.cluster.release_upgrade_lock.return_value = None + client.cluster.refresh_upgrade_lock.return_value = None + return client + + +@pytest.fixture +def coordinator(mock_client): + return ReleaseUpgradeCoordinator(mock_client) + + +class TestAcquireLock: + def test_acquire_returns_token(self, coordinator, mock_client): + token = coordinator.acquire_lock("host-pid") + assert token == 1 + assert coordinator.token == 1 + mock_client.cluster.acquire_upgrade_lock.assert_called_once_with("host-pid") + + def test_acquire_defaults_holder_id(self, coordinator, mock_client): + coordinator.acquire_lock() + holder_id = mock_client.cluster.acquire_upgrade_lock.call_args[0][0] + assert "-" in holder_id + + def test_acquire_propagates_lock_held(self, coordinator, mock_client): + mock_client.cluster.acquire_upgrade_lock.side_effect = UpgradeLockHeldException( + "held" + ) + with pytest.raises(UpgradeLockHeldException): + coordinator.acquire_lock() + + def test_release_lock(self, coordinator, mock_client): + coordinator.acquire_lock("test") + coordinator.release_lock() + mock_client.cluster.release_upgrade_lock.assert_called_once_with(1) + assert coordinator.token is None + + def test_release_lock_safe_if_not_held(self, coordinator): + coordinator.release_lock() + assert coordinator.token is None + + def test_release_handles_stale_token(self, coordinator, mock_client): + coordinator.acquire_lock("test") + mock_client.cluster.release_upgrade_lock.side_effect = ( + UpgradeTokenMismatchException("stale") + ) + coordinator.release_lock() + assert coordinator.token is None + + def test_refresh_lock(self, coordinator, mock_client): + coordinator.acquire_lock("test") + coordinator.refresh_lock() + mock_client.cluster.refresh_upgrade_lock.assert_called_once_with(1) + + def test_refresh_without_lock_raises(self, coordinator): + with pytest.raises(RuntimeError, match="no lock held"): + coordinator.refresh_lock() + + +class TestLoadState: + def test_load_empty_state(self, coordinator, mock_client): + mock_client.cluster.get_upgrade_state.return_value = None + state = coordinator.load_state() + assert state.active_hop.hop_history_index is None + assert len(state.hop_history) == 0 + + def test_load_existing_state(self, coordinator, mock_client): + state_json = json.dumps( + { + "active_hop": {"hop_history_index": 0}, + "hop_history": [ + { + "from": "2025.1", + "to": "2026.1", + "status": "in_progress", + "metadata_version": 1, + "metadata_build_id": "rev-100", + } + ], + } + ) + mock_client.cluster.get_upgrade_state.return_value = state_json + state = coordinator.load_state() + assert state.current_hop is not None + assert state.current_hop.from_release == "2025.1" + + +class TestPersistState: + def test_persist_writes_json(self, coordinator, mock_client): + coordinator.acquire_lock("test") + coordinator.load_state() + coordinator.persist_state() + mock_client.cluster.update_upgrade_state.assert_called_once() + args = mock_client.cluster.update_upgrade_state.call_args + assert args[0][0] == 1 + assert "active_hop" in args[0][1] + + def test_persist_without_lock_raises(self, coordinator): + coordinator.load_state() + with pytest.raises(RuntimeError, match="no lock held"): + coordinator.persist_state() + + def test_persist_without_state_raises(self, coordinator): + coordinator.acquire_lock("test") + with pytest.raises(RuntimeError, match="no state loaded"): + coordinator.persist_state() + + def test_persist_propagates_token_mismatch(self, coordinator, mock_client): + coordinator.acquire_lock("test") + coordinator.load_state() + mock_client.cluster.update_upgrade_state.side_effect = ( + UpgradeTokenMismatchException("stale") + ) + with pytest.raises(UpgradeTokenMismatchException): + coordinator.persist_state() + + +class TestValidateHop: + def test_valid_hop(self, coordinator): + coordinator.validate_hop("2025.1", "2026.1") + + def test_invalid_hop(self, coordinator): + with pytest.raises(ValueError, match="invalid upgrade hop"): + coordinator.validate_hop("2024.1", "2026.1") + + +class TestCreateHop: + def test_create_hop(self, coordinator, mock_client): + coordinator.acquire_lock("test") + coordinator.load_state() + hop = coordinator.create_hop("2025.1", "2026.1", "rev-100") + assert hop.from_release == "2025.1" + assert hop.to_release == "2026.1" + assert hop.status == HopStatus.PENDING + assert coordinator.state.active_hop.hop_history_index == 0 + mock_client.cluster.update_upgrade_state.assert_called_once() + + def test_create_invalid_hop_raises(self, coordinator): + coordinator.acquire_lock("test") + coordinator.load_state() + with pytest.raises(ValueError): + coordinator.create_hop("2024.1", "2026.1", "rev-100") + + +class TestResume: + def test_resume_no_active_hop(self, coordinator, mock_client): + mock_client.cluster.get_upgrade_state.return_value = None + phase, step = coordinator.resume() + assert phase is None + assert step is None + + def test_resume_completed_hop(self, coordinator, mock_client): + state_json = json.dumps( + { + "active_hop": {"hop_history_index": 0}, + "hop_history": [ + { + "from": "2025.1", + "to": "2026.1", + "status": "completed", + "metadata_version": 1, + "metadata_build_id": "rev-100", + } + ], + } + ) + mock_client.cluster.get_upgrade_state.return_value = state_json + phase, step = coordinator.resume() + assert phase is None + + def test_resume_dataplane_phase(self, coordinator, mock_client): + state_json = json.dumps( + { + "active_hop": {"hop_history_index": 0}, + "hop_history": [ + { + "from": "2025.1", + "to": "2026.1", + "status": "in_progress", + "phase": "dataplane", + "metadata_version": 1, + "metadata_build_id": "rev-100", + "phases": { + "dataplane": { + "status": "in_progress", + "nodes": { + "compute-0": { + "status": "failed", + "step": "refresh-principal", + "step_status": "failed", + } + }, + } + }, + } + ], + } + ) + mock_client.cluster.get_upgrade_state.return_value = state_json + phase, step = coordinator.resume() + assert phase == PhaseName.DATAPLANE + assert "compute-0" in step + + def test_resume_control_plane_phase(self, coordinator, mock_client): + state_json = json.dumps( + { + "active_hop": {"hop_history_index": 0}, + "hop_history": [ + { + "from": "2025.1", + "to": "2026.1", + "status": "in_progress", + "phase": "control_plane", + "metadata_version": 1, + "metadata_build_id": "rev-100", + "phases": { + "control_plane": { + "status": "in_progress", + "groups": { + "identity-core": {"status": "completed"}, + "compute-control": {"status": "pending"}, + }, + } + }, + } + ], + } + ) + mock_client.cluster.get_upgrade_state.return_value = state_json + phase, step = coordinator.resume() + assert phase == PhaseName.CONTROL_PLANE + assert "compute-control" in step + + +class TestRunPhase: + def test_successful_phase(self, coordinator, mock_client): + coordinator.acquire_lock("test") + coordinator.load_state() + coordinator.create_hop("2025.1", "2026.1", "rev-100") + + handler = MagicMock(spec=PhaseHandler) + handler.run.return_value = PhaseResult(success=True) + + result = coordinator.run_phase(PhaseName.PREFLIGHT, handler) + assert result.success is True + hop = coordinator.get_current_hop() + assert hop.phases.preflight.status == PhaseStatus.COMPLETED + + def test_failed_phase_sets_error(self, coordinator, mock_client): + coordinator.acquire_lock("test") + coordinator.load_state() + coordinator.create_hop("2025.1", "2026.1", "rev-100") + + handler = MagicMock(spec=PhaseHandler) + handler.run.return_value = PhaseResult( + success=False, + error_code=UpgradeErrorCode.PREFLIGHT_HEALTH_CHECK, + error_message="ceph unhealthy", + ) + + result = coordinator.run_phase(PhaseName.PREFLIGHT, handler) + assert result.success is False + hop = coordinator.get_current_hop() + assert hop.phases.preflight.status == PhaseStatus.FAILED + assert hop.phases.preflight.last_error.code == "PREFLIGHT_HEALTH_CHECK" + + def test_handler_exception_caught(self, coordinator, mock_client): + coordinator.acquire_lock("test") + coordinator.load_state() + coordinator.create_hop("2025.1", "2026.1", "rev-100") + + handler = MagicMock(spec=PhaseHandler) + handler.run.side_effect = RuntimeError("boom") + + result = coordinator.run_phase(PhaseName.PREFLIGHT, handler) + assert result.success is False + + def test_run_phase_without_state_raises(self, coordinator): + handler = MagicMock(spec=PhaseHandler) + with pytest.raises(RuntimeError): + coordinator.run_phase(PhaseName.PREFLIGHT, handler) + + +class TestTransitions: + def test_valid_hop_transitions(self): + assert HopStatus.IN_PROGRESS in VALID_HOP_TRANSITIONS[HopStatus.PENDING] + assert HopStatus.COMPLETED in VALID_HOP_TRANSITIONS[HopStatus.IN_PROGRESS] + + def test_valid_phase_transitions(self): + assert PhaseStatus.IN_PROGRESS in VALID_PHASE_TRANSITIONS[PhaseStatus.PENDING] + assert PhaseStatus.COMPLETED in VALID_PHASE_TRANSITIONS[PhaseStatus.IN_PROGRESS] + + def test_invalid_hop_transition_raises(self, coordinator, mock_client): + coordinator.acquire_lock("test") + coordinator.load_state() + hop = coordinator.create_hop("2025.1", "2026.1", "rev-100") + with pytest.raises(TransitionError): + coordinator._transition_hop(hop, HopStatus.COMPLETED) + + def test_terminal_states_have_no_transitions(self): + assert VALID_HOP_TRANSITIONS[HopStatus.COMPLETED] == set() + assert VALID_HOP_TRANSITIONS[HopStatus.ABANDONED] == set() + + +class TestAbandon: + def test_abandon_marks_hop_and_releases_lock(self, coordinator, mock_client): + coordinator.acquire_lock("test") + coordinator.load_state() + coordinator.create_hop("2025.1", "2026.1", "rev-100") + + coordinator.abandon() + hop = coordinator.get_current_hop() + assert hop.status == HopStatus.ABANDONED + assert coordinator.token is None + + def test_abandon_without_hop_raises(self, coordinator, mock_client): + coordinator.acquire_lock("test") + coordinator.load_state() + with pytest.raises(RuntimeError, match="no active hop"): + coordinator.abandon() + + +class TestLifecycle: + """Full lifecycle: acquire, create hop, run phase, release.""" + + def test_full_flow(self, coordinator, mock_client): + coordinator.acquire_lock("test") + coordinator.load_state() + with patch("sunbeam.upgrades.coordinator.load_upgrade_metadata") as mock_load: + mock_load.return_value = MagicMock( + from_release="2025.1", + to_release="2026.1", + control_plane_groups=[], + finalize=[], + ) + coordinator.load_metadata("2026.1") + coordinator.validate_hop("2025.1", "2026.1") + coordinator.create_hop("2025.1", "2026.1", "rev-100") + + handler = MagicMock(spec=PhaseHandler) + handler.run.return_value = PhaseResult(success=True) + result = coordinator.run_phase(PhaseName.PREFLIGHT, handler) + assert result.success is True + + coordinator.release_lock() + assert coordinator.token is None + assert mock_client.cluster.release_upgrade_lock.called From dae150075b226fbbe0b66a76c96897a5e4dbf67f Mon Sep 17 00:00:00 2001 From: Hemanth Nakkina Date: Mon, 17 Aug 2026 17:59:07 +0530 Subject: [PATCH 06/20] feat(upgrades): add structured upgrade logger and wire into coordinator MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit UpgradeLogger writes one JSON object per line to /var/snap/openstack/common/logs/upgrade.log. Every state mutation in the coordinator emits a line: lock acquire/release/refresh, phase started/completed/failed, hop abandoned. Lines are flushed immediately so a SIGKILL does not lose log lines that were already written. Three log entry types: - log_state_change: phase/group/node/step transitions with status and optional error_code + error_message (error catalog) - log_lock_event: advisory lock lifecycle (acquired, refreshed, released, stale) - log_command: CLI command invocations for audit trail The log is append-only — resume appends to the same file. The status command shows current state; upgrade.log shows the history. gather-logs will include this file in the log bundle. Coordinator integration: acquire_lock, refresh_lock, release_lock, run_phase (start/complete/fail), and abandon all emit log lines via the logger. The logger is injected via the coordinator constructor and defaults to UpgradeLogger() with the standard log path. 12 tests: JSON line format, error field inclusion/omission, metadata, append behavior, lock events, command logging, timestamp ISO format, one-line-per-entry validation. --- .../sunbeam/upgrades/coordinator.py | 62 +++++++++++++- .../sunbeam/upgrades/observability.py | 69 +++++++++++++++ .../tests/unit/sunbeam/test_observability.py | 83 +++++++++++++++++++ 3 files changed, 213 insertions(+), 1 deletion(-) create mode 100644 sunbeam-python/sunbeam/upgrades/observability.py create mode 100644 sunbeam-python/tests/unit/sunbeam/test_observability.py diff --git a/sunbeam-python/sunbeam/upgrades/coordinator.py b/sunbeam-python/sunbeam/upgrades/coordinator.py index 02dc7e864..4351638bc 100644 --- a/sunbeam-python/sunbeam/upgrades/coordinator.py +++ b/sunbeam-python/sunbeam/upgrades/coordinator.py @@ -26,6 +26,7 @@ import logging import os +import threading import typing from dataclasses import dataclass from enum import Enum @@ -35,6 +36,7 @@ from sunbeam.clusterd.service import UpgradeTokenMismatchException from sunbeam.upgrades.errors import UpgradeErrorCode from sunbeam.upgrades.metadata import HopMetadata, load_upgrade_metadata +from sunbeam.upgrades.observability import UpgradeLogger from sunbeam.upgrades.state import ( Hop, HopStatus, @@ -152,11 +154,14 @@ class ReleaseUpgradeCoordinator: status in_progress are treated as failed and re-run from scratch. """ - def __init__(self, client: Client): + def __init__(self, client: Client, logger: UpgradeLogger | None = None): self.client = client self._token: int | None = None self._state: UpgradeState | None = None self._metadata: HopMetadata | None = None + self.logger = logger or UpgradeLogger() + self._heartbeat_thread: threading.Thread | None = None + self._heartbeat_stop = threading.Event() @property def token(self) -> int | None: @@ -176,6 +181,9 @@ def metadata(self) -> HopMetadata | None: def acquire_lock(self, holder_id: str | None = None) -> int: """Acquire the advisory lock. Returns the fencing token. + Starts a background heartbeat thread that refreshes the lock's + TTL every 30 seconds (TTL is 60s in clusterd). + :param holder_id: identifies the process. Defaults to hostname+pid. :raises UpgradeLockHeldException: if another live holder owns it. """ @@ -186,8 +194,34 @@ def acquire_lock(self, holder_id: str | None = None) -> int: ) self._token = response.token LOG.info("acquired upgrade lock (token=%d, holder=%s)", self._token, holder_id) + self.logger.log_lock_event("acquired", self._token, holder_id) + self._start_heartbeat() return self._token + def _start_heartbeat(self) -> None: + """Start the background heartbeat thread.""" + self._heartbeat_stop.clear() + + def _beat() -> None: + while not self._heartbeat_stop.wait(30): + try: + self.refresh_lock() + except Exception as e: + LOG.warning("lock heartbeat failed: %s", e) + break + + self._heartbeat_thread = threading.Thread( + target=_beat, daemon=True, name="upgrade-lock-heartbeat" + ) + self._heartbeat_thread.start() + + def _stop_heartbeat(self) -> None: + """Stop the background heartbeat thread.""" + self._heartbeat_stop.set() + if self._heartbeat_thread is not None: + self._heartbeat_thread.join(timeout=5) + self._heartbeat_thread = None + def refresh_lock(self) -> None: """Extend the lock's TTL. Called by the heartbeat loop. @@ -196,18 +230,22 @@ def refresh_lock(self) -> None: if self._token is None: raise RuntimeError("no lock held") self.client.cluster.refresh_upgrade_lock(self._token) + self.logger.log_lock_event("refreshed", self._token) def release_lock(self) -> None: """Release the lock. Safe to call even if already released.""" + self._stop_heartbeat() if self._token is not None: try: self.client.cluster.release_upgrade_lock(self._token) LOG.info("released upgrade lock (token=%d)", self._token) + self.logger.log_lock_event("released", self._token) except UpgradeTokenMismatchException: LOG.warning( "lock token %d is stale — already re-acquired by another process", self._token, ) + self.logger.log_lock_event("stale", self._token) finally: self._token = None @@ -393,12 +431,22 @@ def run_phase( phase_obj = getattr(hop.phases, phase_name.value) self._transition_phase(phase_obj, PhaseStatus.IN_PROGRESS) hop.phase = phase_name.value + self.logger.log_state_change( + "phase", phase_name.value, "phase_started", "in_progress" + ) self.persist_state() try: result = handler.run(self, self._metadata, self._state) except Exception as e: LOG.exception("phase %s raised: %s", phase_name.value, e) + self.logger.log_state_change( + "phase", + phase_name.value, + "phase_exception", + "failed", + error_message=str(e), + ) result = PhaseResult( success=False, error_code=UpgradeErrorCode.HOP_INVALID_TRANSITION, @@ -407,8 +455,19 @@ def run_phase( if result.success: self._transition_phase(phase_obj, PhaseStatus.COMPLETED) + self.logger.log_state_change( + "phase", phase_name.value, "phase_completed", "completed" + ) else: self._transition_phase(phase_obj, PhaseStatus.FAILED) + self.logger.log_state_change( + "phase", + phase_name.value, + "phase_failed", + "failed", + error_code=result.error_code.value if result.error_code else None, + error_message=result.error_message, + ) if result.error_code: phase_obj.last_error = LastError( code=result.error_code.value, @@ -462,5 +521,6 @@ def abandon(self) -> None: self._transition_hop(hop, HopStatus.ABANDONED) self.persist_state() + self.logger.log_state_change("hop", "active_hop", "hop_abandoned", "abandoned") self.release_lock() LOG.info("hop abandoned") diff --git a/sunbeam-python/sunbeam/upgrades/observability.py b/sunbeam-python/sunbeam/upgrades/observability.py new file mode 100644 index 000000000..8418a9aa0 --- /dev/null +++ b/sunbeam-python/sunbeam/upgrades/observability.py @@ -0,0 +1,69 @@ +# SPDX-FileCopyrightText: 2025 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +"""Upgrade observability: state transitions logged to the standard debug log. + +Every state mutation in the coordinator emits a structured log line at +DEBUG level via the standard sunbeam logger. No separate upgrade.log +file — the debug log at +``$HOME/snap/openstack/common/logs/sunbeam-.log`` is the +audit trail. +""" + +from __future__ import annotations + +import logging +from typing import Any + +LOG = logging.getLogger(__name__) + + +class UpgradeLogger: + """Logs state transitions and lock events via LOG.debug.""" + + def log_state_change( + self, + component: str, + component_name: str, + action: str, + status: str, + error_code: str | None = None, + error_message: str | None = None, + metadata: dict[str, Any] | None = None, + ) -> None: + """Log a state transition at DEBUG level.""" + LOG.debug( + "upgrade state: component=%s name=%s action=%s status=%s%s%s", + component, + component_name, + action, + status, + f" error_code={error_code}" if error_code else "", + f" error={error_message}" if error_message else "", + ) + + def log_lock_event( + self, + event: str, + token: int, + holder_id: str | None = None, + ) -> None: + """Log a lock event at DEBUG level.""" + LOG.debug( + "upgrade lock: event=%s token=%d%s", + event, + token, + f" holder={holder_id}" if holder_id else "", + ) + + def log_command( + self, + command: str, + args: dict[str, Any] | None = None, + ) -> None: + """Log a CLI command invocation at DEBUG level.""" + LOG.debug( + "upgrade command: %s%s", + command, + f" args={args}" if args else "", + ) diff --git a/sunbeam-python/tests/unit/sunbeam/test_observability.py b/sunbeam-python/tests/unit/sunbeam/test_observability.py new file mode 100644 index 000000000..fcc6dfc0d --- /dev/null +++ b/sunbeam-python/tests/unit/sunbeam/test_observability.py @@ -0,0 +1,83 @@ +# SPDX-FileCopyrightText: 2025 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for the upgrade observability logger.""" + +import logging + +import pytest + +from sunbeam.upgrades.observability import UpgradeLogger + + +@pytest.fixture +def logger() -> UpgradeLogger: + return UpgradeLogger() + + +@pytest.fixture +def debug_log(caplog): + logger_name = "sunbeam.upgrades.observability" + with caplog.at_level(logging.DEBUG, logger=logger_name): + yield caplog + + +class TestLogStateChange: + def test_logs_state_change(self, logger, debug_log): + logger.log_state_change( + "phase", "control_plane", "phase_started", "in_progress" + ) + record = debug_log.records[0] + assert record.levelno == logging.DEBUG + assert "component=phase" in record.message + assert "name=control_plane" in record.message + assert "action=phase_started" in record.message + assert "status=in_progress" in record.message + + def test_includes_error_fields(self, logger, debug_log): + logger.log_state_change( + "phase", + "dataplane", + "phase_failed", + "failed", + error_code="DATAPLANE_REGISTRATION_TIMEOUT", + error_message="nova-compute did not re-register", + ) + message = debug_log.records[0].message + assert "error_code=DATAPLANE_REGISTRATION_TIMEOUT" in message + assert "nova-compute did not re-register" in message + + def test_omits_error_fields_when_none(self, logger, debug_log): + logger.log_state_change("phase", "preflight", "phase_completed", "completed") + message = debug_log.records[0].message + assert "error_code" not in message + assert "error=" not in message + + +class TestLogLockEvent: + def test_acquired(self, logger, debug_log): + logger.log_lock_event("acquired", 42, holder_id="host-pid") + message = debug_log.records[0].message + assert "event=acquired" in message + assert "token=42" in message + assert "holder=host-pid" in message + + def test_released_without_holder(self, logger, debug_log): + logger.log_lock_event("released", 42) + message = debug_log.records[0].message + assert "event=released" in message + assert "holder" not in message + + +class TestLogCommand: + def test_command_invocation(self, logger, debug_log): + logger.log_command("control-plane --auto", args={"group": "all"}) + message = debug_log.records[0].message + assert "control-plane --auto" in message + assert "group" in message + + def test_command_without_args(self, logger, debug_log): + logger.log_command("finalize") + message = debug_log.records[0].message + assert "finalize" in message + assert "args=" not in message From cbc01f0af5b71732e8ca14fd3984b77cbbaf2131 Mon Sep 17 00:00:00 2001 From: Hemanth Nakkina Date: Tue, 18 Aug 2026 10:16:11 +0530 Subject: [PATCH 07/20] feat(upgrades): add preflight health-check framework Adds the health-check framework for the upgrade preflight phase. Checks subclass sunbeam.core.checks.Check and carry an exit_code (1 = operational, 2 = invalid hop/metadata). Four checks in order: - SnapVersionCheck: snap release matches target (exit 2) - HopMetadataCheck: validates hop is supported, metadata loads and from/to match the requested hop (exit 2) - ClusterHealthCheck: all apps in both control-plane and machines models are healthy, with tolerated-blocked-message set (exit 1) - MySQLQuorumCheck: runs get-cluster-status action on mysql-k8s leader to verify quorum (exit 1) CheckContext derives client, JujuHelper, and model names from the Deployment object (provider-aware). The runner run_upgrade_preflight_checks short-circuits on first failure. 23 tests, all CI green. --- .../sunbeam/upgrades/preflight/__init__.py | 3 + .../sunbeam/upgrades/preflight/checks.py | 323 ++++++++++++++++ .../unit/sunbeam/test_preflight_checks.py | 359 ++++++++++++++++++ 3 files changed, 685 insertions(+) create mode 100644 sunbeam-python/sunbeam/upgrades/preflight/__init__.py create mode 100644 sunbeam-python/sunbeam/upgrades/preflight/checks.py create mode 100644 sunbeam-python/tests/unit/sunbeam/test_preflight_checks.py diff --git a/sunbeam-python/sunbeam/upgrades/preflight/__init__.py b/sunbeam-python/sunbeam/upgrades/preflight/__init__.py new file mode 100644 index 000000000..6279e0d5d --- /dev/null +++ b/sunbeam-python/sunbeam/upgrades/preflight/__init__.py @@ -0,0 +1,3 @@ +# SPDX-FileCopyrightText: 2025 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 +"""Preflight checks and orchestration for the release-upgrade engine.""" diff --git a/sunbeam-python/sunbeam/upgrades/preflight/checks.py b/sunbeam-python/sunbeam/upgrades/preflight/checks.py new file mode 100644 index 000000000..caf01993f --- /dev/null +++ b/sunbeam-python/sunbeam/upgrades/preflight/checks.py @@ -0,0 +1,323 @@ +# SPDX-FileCopyrightText: 2025 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +"""Health-check framework for the upgrade preflight phase. + +Each check subclasses ``sunbeam.core.checks.Check`` and sets +``self.exit_code`` to 1 (operational failure) or 2 (invalid hop / +unsupported / missing metadata) per the upgrade exit-code table. The +runner ``run_upgrade_preflight_checks`` short-circuits on the first +failure and raises ``click.ClickException`` carrying the exit code. + +Checks are ordered: cheap static checks (snap version, metadata/hop +validity) run before expensive Juju/MySQL checks. +""" + +from __future__ import annotations + +import logging +from dataclasses import dataclass +from typing import Sequence + +import click +from rich.console import Console +from rich.status import Status +from snaphelpers import Snap + +from sunbeam.clusterd.client import Client +from sunbeam.core.checks import Check +from sunbeam.core.deployment import Deployment +from sunbeam.core.juju import JujuHelper +from sunbeam.core.openstack import OPENSTACK_MODEL +from sunbeam.upgrades.metadata import HopMetadata, load_upgrade_metadata +from sunbeam.versions import ( + detect_snap_release, + is_valid_hop, +) + +LOG = logging.getLogger(__name__) + +# The K8s control-plane model is always "openstack" (see +# sunbeam.core.openstack.OPENSTACK_MODEL). The machines model name varies +# by provider — use deployment.openstack_machines_model. + +# Tolerated ``blocked`` workload messages. A charm in ``blocked`` with a +# message in this set passes preflight; anything else fails. Each entry +# has a comment naming the charm that emits it. Start empty and grow as +# integration surfaces known-safe blocks. +TOLERATED_BLOCKED_MESSAGES: set[str] = set( + { + # e.g. "Manual security enable required" # sunbeam-machine + } +) + + +@dataclass +class CheckContext: + """Shared state for preflight checks. + + Carries everything the checks need so each check is a pure function + of (ctx) -> bool. Constructed once by the CLI command and passed to + every check. Client, JujuHelper, and model names all come from the + deployment. + """ + + deployment: Deployment + from_release: str + to_release: str + snap: Snap | None = None + metadata: HopMetadata | None = None + + @property + def client(self) -> Client: + """Return the clusterd client for this deployment.""" + return self.deployment.get_client() + + @property + def jhelper(self) -> JujuHelper: + """Return a JujuHelper for this deployment.""" + return self.deployment.get_juju_helper() + + @property + def machines_model(self) -> str: + """Return the machines model name for this deployment.""" + return self.deployment.openstack_machines_model + + +def run_upgrade_preflight_checks( + checks: Sequence[Check], + console: Console, +) -> None: + """Run preflight checks sequentially. + + Like ``sunbeam.core.checks.run_preflight_checks`` but respects the + upgrade exit-code distinction: each failed check carries an + ``exit_code`` (1 = operational, 2 = invalid hop / metadata). + Short-circuits on the first failure. + """ + for check in checks: + LOG.debug("Starting preflight check %s", check.name) + with console.status(f"{check.description} ... "): + passed = check.run() + if passed: + click.echo(f" ✓ {check.name}") + else: + exit_code = getattr(check, "exit_code", 1) + click.echo(f" ✗ {check.name}: {check.message}") + raise click.ClickException(f"[exit {exit_code}] {check.message}") + + +class UpgradeCheck(Check): + """Base class for upgrade preflight checks. + + Adds ``exit_code`` (1 or 2) on top of ``Check``. + """ + + def __init__(self, name: str, description: str = "", exit_code: int = 1): + super().__init__(name, description) + self.exit_code = exit_code + + +# --------------------------------------------------------------------------- +# Static checks (no Juju, no MySQL — snap / metadata / hop validity) +# --------------------------------------------------------------------------- + + +class SnapVersionCheck(UpgradeCheck): + """Snap release matches the target release. + + The snap's ``deployment.version`` config must equal the target + release. The operator refreshes the snap to the target before + running preflight; this catches a stale snap. + """ + + def __init__(self, ctx: CheckContext): + super().__init__( + "Check snap version matches target", + "Checking snap version matches target release", + exit_code=2, + ) + self.ctx = ctx + + def run(self, check_status: Status | None = None) -> bool: + """Return False if snap release does not match the target.""" + snap_release = detect_snap_release() + if snap_release != self.ctx.to_release: + self.message = ( + f"Snap is at release {snap_release!r} but target is " + f"{self.ctx.to_release!r}. Refresh the snap to the " + "target release before running upgrade." + ) + return False + return True + + +class HopMetadataCheck(UpgradeCheck): + """Validate the upgrade hop: metadata present, compatible, and hop is valid. + + Combines three sub-checks into one: + - Metadata file exists and loads for the target release. + - Metadata ``from``/``to`` match the requested hop. + - The from->to pair is a supported upgrade path. + + Failing any sub-check fails the check with exit code 2. + """ + + def __init__(self, ctx: CheckContext): + super().__init__( + "Check upgrade hop and metadata", + "Checking upgrade hop validity and metadata", + exit_code=2, + ) + self.ctx = ctx + + def run(self, check_status: Status | None = None) -> bool: + """Return False if hop is invalid or metadata is missing/incompatible.""" + if not is_valid_hop(self.ctx.from_release, self.ctx.to_release): + self.message = ( + f"Hop {self.ctx.from_release} -> {self.ctx.to_release} is " + "not a supported upgrade path." + ) + return False + try: + self.ctx.metadata = load_upgrade_metadata(self.ctx.to_release) + except FileNotFoundError as e: + self.message = str(e) + return False + except Exception as e: + self.message = f"Failed to load upgrade metadata: {e}" + return False + if self.ctx.metadata.from_release != self.ctx.from_release: + self.message = ( + f"Metadata from_release {self.ctx.metadata.from_release!r} " + f"does not match requested from {self.ctx.from_release!r}" + ) + return False + if self.ctx.metadata.to_release != self.ctx.to_release: + self.message = ( + f"Metadata to_release {self.ctx.metadata.to_release!r} " + f"does not match requested to {self.ctx.to_release!r}" + ) + return False + return True + + +# --------------------------------------------------------------------------- +# Juju checks (model status) +# --------------------------------------------------------------------------- + + +class ClusterHealthCheck(UpgradeCheck): + """All Juju models are healthy. + + Iterates both the control-plane model (``openstack``) and the + machines model (provider-specific). Every app must be ``active`` + or ``idle``. ``blocked`` passes only if the workload message is + in ``TOLERATED_BLOCKED_MESSAGES``. Anything else fails. + """ + + def __init__(self, ctx: CheckContext): + super().__init__( + "Check Juju cluster health", + "Checking Juju cluster health", + exit_code=1, + ) + self.ctx = ctx + + def run(self, check_status: Status | None = None) -> bool: + """Return False if any app in any model is not healthy.""" + unhealthy: list[str] = [] + for model in (OPENSTACK_MODEL, self.ctx.machines_model): + try: + status = self.ctx.jhelper.get_model_status(model) + except Exception as e: + unhealthy.append(f"{model}: unreachable ({e})") + continue + for app_name, app in status.apps.items(): + current = app.app_status.current + if current in ("active", "idle"): + continue + if current == "blocked": + msg = app.app_status.message or "" + if msg in TOLERATED_BLOCKED_MESSAGES: + continue + unhealthy.append( + f"{model}/{app_name}: {current}" + + (f" ({app.app_status.message})" if app.app_status.message else "") + ) + if unhealthy: + self.message = ( + "Juju cluster is not healthy:\n " + + "\n ".join(unhealthy) + + "\nResolve these issues before retrying the upgrade." + ) + return False + return True + + +# --------------------------------------------------------------------------- +# Database checks +# --------------------------------------------------------------------------- + + +class MySQLQuorumCheck(UpgradeCheck): + """MySQL has quorum. + + Runs the ``get-cluster-status`` action on the ``mysql-k8s`` leader + unit. The action returns the cluster topology; if it succeeds and + reports a healthy cluster, quorum is up. If no leader is found or + the action fails, quorum may be lost. + """ + + APP = "mysql" + MODEL = OPENSTACK_MODEL + + def __init__(self, ctx: CheckContext): + super().__init__( + "Check MySQL quorum", + "Checking MySQL quorum (cluster status)", + exit_code=1, + ) + self.ctx = ctx + + def run(self, check_status: Status | None = None) -> bool: + """Return False if MySQL cluster status action fails.""" + try: + leader = self.ctx.jhelper.get_leader_unit(self.APP, self.MODEL) + except Exception as e: + self.message = f"Cannot find MySQL leader: {e}" + return False + if not leader: + self.message = ( + "MySQL has no leader unit — quorum may be lost. " + "Check mysql-k8s unit status before retrying." + ) + return False + try: + result = self.ctx.jhelper.run_action( + leader, self.MODEL, "get-cluster-status" + ) + except Exception as e: + self.message = f"MySQL get-cluster-status action failed: {e}" + return False + if not result: + self.message = ( + "MySQL get-cluster-status returned no result. " + "Check mysql-k8s unit status before retrying." + ) + return False + return True + + +def build_preflight_checks(ctx: CheckContext) -> list[Check]: + """Construct the ordered list of preflight checks. + + Order matters: cheap static checks run first (fail fast on a + stale snap or invalid hop before touching Juju/MySQL). + """ + return [ + SnapVersionCheck(ctx), + HopMetadataCheck(ctx), + ClusterHealthCheck(ctx), + MySQLQuorumCheck(ctx), + ] diff --git a/sunbeam-python/tests/unit/sunbeam/test_preflight_checks.py b/sunbeam-python/tests/unit/sunbeam/test_preflight_checks.py new file mode 100644 index 000000000..2672cac73 --- /dev/null +++ b/sunbeam-python/tests/unit/sunbeam/test_preflight_checks.py @@ -0,0 +1,359 @@ +# SPDX-FileCopyrightText: 2025 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for upgrade preflight checks.""" + +from unittest.mock import MagicMock, patch + +import pytest + +from sunbeam.clusterd.models import FeatureGates +from sunbeam.upgrades.metadata import HopMetadata +from sunbeam.upgrades.preflight.checks import ( + CheckContext, + ClusterHealthCheck, + HopMetadataCheck, + MySQLQuorumCheck, + SnapVersionCheck, + build_preflight_checks, + run_upgrade_preflight_checks, +) + +FROM = "2025.1" +TO = "2026.1" + + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + + +def _make_app(current: str, message: str = "") -> MagicMock: + app = MagicMock() + app.app_status.current = current + app.app_status.message = message + return app + + +@pytest.fixture +def mock_client(): + client = MagicMock() + client.cluster.get_feature_gates.return_value = FeatureGates([]) + return client + + +@pytest.fixture +def mock_jhelper(): + return MagicMock() + + +@pytest.fixture +def mock_deployment(mock_client, mock_jhelper): + deployment = MagicMock() + deployment.openstack_machines_model = "openstack-machines" + deployment.get_client.return_value = mock_client + deployment.get_juju_helper.return_value = mock_jhelper + return deployment + + +@pytest.fixture +def ctx(mock_deployment): + return CheckContext( + deployment=mock_deployment, + from_release=FROM, + to_release=TO, + ) + + +def _make_metadata(from_release: str = FROM, to_release: str = TO) -> HopMetadata: + return HopMetadata.model_validate( + { + "from": from_release, + "to": to_release, + "control_plane_groups": [ + {"name": "identity-core", "apps": ["keystone-k8s"]} + ], + } + ) + + +def _model_status(apps: dict[str, MagicMock]) -> MagicMock: + status = MagicMock() + status.apps = apps + return status + + +# --------------------------------------------------------------------------- +# SnapVersionCheck +# --------------------------------------------------------------------------- + + +class TestSnapVersionCheck: + def test_passes_when_snap_matches_target(self, ctx): + with patch( + "sunbeam.upgrades.preflight.checks.detect_snap_release", + return_value=TO, + ): + check = SnapVersionCheck(ctx) + assert check.run() is True + assert check.exit_code == 2 + + def test_fails_when_snap_is_stale(self, ctx): + with patch( + "sunbeam.upgrades.preflight.checks.detect_snap_release", + return_value=FROM, + ): + check = SnapVersionCheck(ctx) + assert check.run() is False + assert check.exit_code == 2 + assert TO in check.message + assert FROM in check.message + + +# --------------------------------------------------------------------------- +# HopMetadataCheck (combined: hop validity + metadata present + compat) +# --------------------------------------------------------------------------- + + +class TestHopMetadataCheck: + def test_passes_on_valid_hop_with_matching_metadata(self, ctx): + metadata = _make_metadata() + with patch( + "sunbeam.upgrades.preflight.checks.load_upgrade_metadata", + return_value=metadata, + ): + check = HopMetadataCheck(ctx) + assert check.run() is True + assert ctx.metadata is metadata + assert check.exit_code == 2 + + def test_fails_on_invalid_hop(self, ctx): + ctx.from_release = "2024.1" + ctx.to_release = "2026.1" + check = HopMetadataCheck(ctx) + assert check.run() is False + assert check.exit_code == 2 + assert "2024.1" in check.message + assert "2026.1" in check.message + + def test_fails_on_missing_metadata_file(self, ctx): + with patch( + "sunbeam.upgrades.preflight.checks.load_upgrade_metadata", + side_effect=FileNotFoundError("not found"), + ): + check = HopMetadataCheck(ctx) + assert check.run() is False + assert check.exit_code == 2 + assert "not found" in check.message + + def test_fails_on_metadata_load_error(self, ctx): + with patch( + "sunbeam.upgrades.preflight.checks.load_upgrade_metadata", + side_effect=ValueError("bad schema"), + ): + check = HopMetadataCheck(ctx) + assert check.run() is False + assert "bad schema" in check.message + + def test_fails_when_from_mismatches(self, ctx): + with patch( + "sunbeam.upgrades.preflight.checks.load_upgrade_metadata", + return_value=_make_metadata("2024.1", TO), + ): + check = HopMetadataCheck(ctx) + assert check.run() is False + assert "2024.1" in check.message + + def test_fails_when_to_mismatches(self, ctx): + with patch( + "sunbeam.upgrades.preflight.checks.load_upgrade_metadata", + return_value=_make_metadata(FROM, "2027.1"), + ): + check = HopMetadataCheck(ctx) + assert check.run() is False + assert "2027.1" in check.message + + def test_checks_hop_validity_before_metadata(self, ctx): + """Hop validity is checked first — no metadata load for invalid hop.""" + ctx.from_release = "2024.1" + ctx.to_release = "2026.1" + with patch( + "sunbeam.upgrades.preflight.checks.load_upgrade_metadata" + ) as mock_load: + check = HopMetadataCheck(ctx) + assert check.run() is False + mock_load.assert_not_called() + + +# --------------------------------------------------------------------------- +# ClusterHealthCheck (both openstack + machines models) +# --------------------------------------------------------------------------- + + +class TestClusterHealthCheck: + def test_passes_when_all_apps_active_in_both_models(self, ctx): + ctx.jhelper.get_model_status.side_effect = [ + _model_status({"keystone-k8s": _make_app("active")}), + _model_status({"nova-compute": _make_app("active")}), + ] + check = ClusterHealthCheck(ctx) + assert check.run() is True + + def test_passes_with_tolerated_blocked_message(self, ctx): + with patch( + "sunbeam.upgrades.preflight.checks.TOLERATED_BLOCKED_MESSAGES", + {"Manual security enable required"}, + ): + ctx.jhelper.get_model_status.side_effect = [ + _model_status( + { + "sunbeam-machine": _make_app( + "blocked", "Manual security enable required" + ) + } + ), + _model_status({}), + ] + check = ClusterHealthCheck(ctx) + assert check.run() is True + + def test_fails_on_unknown_blocked_message(self, ctx): + ctx.jhelper.get_model_status.side_effect = [ + _model_status({"keystone-k8s": _make_app("blocked", "unknown reason")}), + _model_status({}), + ] + check = ClusterHealthCheck(ctx) + assert check.run() is False + assert check.exit_code == 1 + assert "unknown reason" in check.message + + def test_fails_on_error_status(self, ctx): + ctx.jhelper.get_model_status.side_effect = [ + _model_status({"nova-k8s": _make_app("error")}), + _model_status({}), + ] + check = ClusterHealthCheck(ctx) + assert check.run() is False + assert "nova-k8s" in check.message + assert "error" in check.message + + def test_fails_when_model_unreachable(self, ctx): + ctx.jhelper.get_model_status.side_effect = Exception("connection lost") + check = ClusterHealthCheck(ctx) + assert check.run() is False + assert "unreachable" in check.message + assert "connection lost" in check.message + + def test_fails_on_unhealthy_app_in_machines_model(self, ctx): + ctx.jhelper.get_model_status.side_effect = [ + _model_status({"keystone-k8s": _make_app("active")}), + _model_status({"nova-compute": _make_app("blocked", "missing config")}), + ] + check = ClusterHealthCheck(ctx) + assert check.run() is False + assert "nova-compute" in check.message + assert "missing config" in check.message + + +# --------------------------------------------------------------------------- +# MySQLQuorumCheck +# --------------------------------------------------------------------------- + + +class TestMySQLQuorumCheck: + def test_passes_when_leader_present_and_action_succeeds(self, ctx): + ctx.jhelper.get_leader_unit.return_value = "mysql-k8s/0" + ctx.jhelper.run_action.return_value = {"cluster-status": "ok"} + check = MySQLQuorumCheck(ctx) + assert check.run() is True + ctx.jhelper.run_action.assert_called_once_with( + "mysql-k8s/0", "openstack", "get-cluster-status" + ) + + def test_fails_when_no_leader(self, ctx): + ctx.jhelper.get_leader_unit.return_value = "" + check = MySQLQuorumCheck(ctx) + assert check.run() is False + assert check.exit_code == 1 + assert "quorum" in check.message.lower() + + def test_fails_when_get_leader_raises(self, ctx): + ctx.jhelper.get_leader_unit.side_effect = Exception("timeout") + check = MySQLQuorumCheck(ctx) + assert check.run() is False + assert "timeout" in check.message + + def test_fails_when_action_raises(self, ctx): + ctx.jhelper.get_leader_unit.return_value = "mysql-k8s/0" + ctx.jhelper.run_action.side_effect = Exception("action failed") + check = MySQLQuorumCheck(ctx) + assert check.run() is False + assert "action failed" in check.message + + def test_fails_when_action_returns_empty(self, ctx): + ctx.jhelper.get_leader_unit.return_value = "mysql-k8s/0" + ctx.jhelper.run_action.return_value = {} + check = MySQLQuorumCheck(ctx) + assert check.run() is False + assert "no result" in check.message.lower() + + +# --------------------------------------------------------------------------- +# build_preflight_checks +# --------------------------------------------------------------------------- + + +class TestBuildPreflightChecks: + def test_returns_four_checks_in_order(self, ctx): + checks = build_preflight_checks(ctx) + assert len(checks) == 4 + names = [type(c).__name__ for c in checks] + assert names == [ + "SnapVersionCheck", + "HopMetadataCheck", + "ClusterHealthCheck", + "MySQLQuorumCheck", + ] + + def test_all_checks_carry_exit_code(self, ctx): + for check in build_preflight_checks(ctx): + assert check.exit_code in (1, 2), f"{type(check).__name__} has no exit_code" + + +# --------------------------------------------------------------------------- +# run_upgrade_preflight_checks +# --------------------------------------------------------------------------- + + +class TestRunUpgradePreflightChecks: + def test_runs_all_when_passing(self, ctx): + from rich.console import Console + + console = Console(record=True, width=80) + checks = [ + SnapVersionCheck(ctx), + HopMetadataCheck(ctx), + ] + with patch( + "sunbeam.upgrades.preflight.checks.detect_snap_release", + return_value=TO, + ): + with patch( + "sunbeam.upgrades.preflight.checks.load_upgrade_metadata", + return_value=_make_metadata(), + ): + run_upgrade_preflight_checks(checks, console) + + def test_short_circuits_on_first_failure(self, ctx): + import click + from rich.console import Console + + console = Console(record=True, width=80) + failing = SnapVersionCheck(ctx) + with patch( + "sunbeam.upgrades.preflight.checks.detect_snap_release", + return_value=FROM, + ): + with pytest.raises(click.ClickException) as exc_info: + run_upgrade_preflight_checks([failing, HopMetadataCheck(ctx)], console) + assert "exit 2" in str(exc_info.value) From 5885bbc43ee555f174300c2e09458ffcccf212f0 Mon Sep 17 00:00:00 2001 From: Hemanth Nakkina Date: Tue, 18 Aug 2026 10:37:29 +0530 Subject: [PATCH 08/20] feat(upgrades): add capacity policy check for compute drain Adds CapacityCheck to the preflight sequence. For each openstack-hypervisor unit, runs the running-guests juju action. A node is free if the action returns an empty list. Fails if free node count or percentage is below the policy threshold. CapacityPolicy (min_free_percentage=25, min_free_nodes=1 by default) is stored in clusterd under the upgrade_capacity_policy config key. The --capacity-policy-override flag (wired via build_preflight_checks capacity_override param) skips the check. build_preflight_checks now returns 5 checks in order: SnapVersion, HopMetadata, ClusterHealth, Capacity, MySQLQuorum. 16 new tests, all CI green. --- .../sunbeam/upgrades/preflight/capacity.py | 158 +++++++++++++ .../sunbeam/upgrades/preflight/checks.py | 10 +- .../unit/sunbeam/test_preflight_capacity.py | 221 ++++++++++++++++++ .../unit/sunbeam/test_preflight_checks.py | 11 +- 4 files changed, 397 insertions(+), 3 deletions(-) create mode 100644 sunbeam-python/sunbeam/upgrades/preflight/capacity.py create mode 100644 sunbeam-python/tests/unit/sunbeam/test_preflight_capacity.py diff --git a/sunbeam-python/sunbeam/upgrades/preflight/capacity.py b/sunbeam-python/sunbeam/upgrades/preflight/capacity.py new file mode 100644 index 000000000..88606397b --- /dev/null +++ b/sunbeam-python/sunbeam/upgrades/preflight/capacity.py @@ -0,0 +1,158 @@ +# SPDX-FileCopyrightText: 2025 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +"""Capacity policy for compute drain during upgrade. + +During the data-plane phase, compute nodes are drained one at a time. +Preflight verifies that enough nodes are free (no running VMs) to +absorb the drain. A node is "free" if the ``running-guests`` juju action +on its ``openstack-hypervisor`` unit returns an empty list. + +Policy is stored in clusterd under the ``upgrade_capacity_policy`` +config key as JSON. If absent, defaults apply. +""" + +from __future__ import annotations + +import json +import logging + +import pydantic +from rich.status import Status + +from sunbeam.clusterd.client import Client +from sunbeam.upgrades.preflight.checks import CheckContext, UpgradeCheck + +LOG = logging.getLogger(__name__) + +CONFIG_KEY = "upgrade_capacity_policy" + +HYPERVISOR_APP = "openstack-hypervisor" + + +class CapacityPolicy(pydantic.BaseModel): + """Capacity requirements for a safe upgrade drain. + + Preflight fails if the number of free compute nodes (no running + VMs) is below either threshold. + """ + + min_free_percentage: int = pydantic.Field( + default=25, + description="Minimum percentage of compute nodes that must be free", + ) + min_free_nodes: int = pydantic.Field( + default=1, + description="Absolute minimum number of free compute nodes", + ) + + +def load_capacity_policy(client: Client) -> CapacityPolicy: + """Load capacity policy from clusterd, or return defaults. + + :param client: clusterd client + :returns: CapacityPolicy (from clusterd or defaults) + """ + try: + raw = client.cluster.get_config(CONFIG_KEY) + if raw is None: + return CapacityPolicy() + if isinstance(raw, str): + raw = json.loads(raw) + return CapacityPolicy.model_validate(raw) + except Exception: + LOG.debug("Failed to load capacity policy from clusterd, using defaults") + return CapacityPolicy() + + +class CapacityCheck(UpgradeCheck): + """Verify enough free compute nodes for a safe drain. + + Runs the ``running-guests`` action on every ``openstack-hypervisor`` + unit. A node is free if the action returns an empty list. Fails + exit 1 if free count or percentage is below the policy. + """ + + def __init__(self, ctx: CheckContext, override: bool = False): + super().__init__( + "Check compute capacity for drain", + "Checking compute capacity for drain", + exit_code=1, + ) + self.ctx = ctx + self.override = override + + def run(self, check_status: Status | None = None) -> bool: + """Return False if insufficient free compute nodes.""" + if self.override: + LOG.warning("Capacity policy override — skipping capacity check") + return True + + policy = load_capacity_policy(self.ctx.client) + + try: + app = self.ctx.jhelper.get_application( + HYPERVISOR_APP, self.ctx.machines_model + ) + except Exception as e: + self.message = ( + f"Cannot find {HYPERVISOR_APP} in model {self.ctx.machines_model}: {e}" + ) + return False + + unit_names = list(app.units.keys()) + if not unit_names: + self.message = ( + f"No {HYPERVISOR_APP} units found in model {self.ctx.machines_model}." + ) + return False + + free_nodes: list[str] = [] + busy_nodes: list[str] = [] + for unit_name in unit_names: + try: + result = self.ctx.jhelper.run_action( + unit_name, self.ctx.machines_model, "running-guests" + ) + except Exception as e: + self.message = ( + f"Failed to run running-guests action on {unit_name}: {e}" + ) + return False + guests_raw = result.get("result", "[]") + try: + guests = ( + json.loads(guests_raw) + if isinstance(guests_raw, str) + else (guests_raw or []) + ) + except (json.JSONDecodeError, TypeError): + guests = [] + if guests: + busy_nodes.append(unit_name) + else: + free_nodes.append(unit_name) + + total = len(unit_names) + free_count = len(free_nodes) + free_pct = (free_count * 100) // total if total else 0 + + if free_count < policy.min_free_nodes: + self.message = ( + f"Insufficient free compute nodes: {free_count} free, " + f"policy requires at least {policy.min_free_nodes}. " + "Migrate VMs to free up nodes, or override with " + "--capacity-policy-override." + ) + return False + + if free_pct < policy.min_free_percentage: + self.message = ( + f"Insufficient free compute capacity: {free_pct}% free, " + f"policy requires at least {policy.min_free_percentage}%. " + "Migrate VMs to free up nodes, or override with " + "--capacity-policy-override." + ) + return False + + return True diff --git a/sunbeam-python/sunbeam/upgrades/preflight/checks.py b/sunbeam-python/sunbeam/upgrades/preflight/checks.py index caf01993f..99bcf607a 100644 --- a/sunbeam-python/sunbeam/upgrades/preflight/checks.py +++ b/sunbeam-python/sunbeam/upgrades/preflight/checks.py @@ -309,15 +309,23 @@ def run(self, check_status: Status | None = None) -> bool: return True -def build_preflight_checks(ctx: CheckContext) -> list[Check]: +def build_preflight_checks( + ctx: CheckContext, capacity_override: bool = False +) -> list[Check]: """Construct the ordered list of preflight checks. Order matters: cheap static checks run first (fail fast on a stale snap or invalid hop before touching Juju/MySQL). + + :param capacity_override: skip the capacity check (for + --capacity-policy-override CLI flag). """ + from sunbeam.upgrades.preflight.capacity import CapacityCheck + return [ SnapVersionCheck(ctx), HopMetadataCheck(ctx), ClusterHealthCheck(ctx), + CapacityCheck(ctx, override=capacity_override), MySQLQuorumCheck(ctx), ] diff --git a/sunbeam-python/tests/unit/sunbeam/test_preflight_capacity.py b/sunbeam-python/tests/unit/sunbeam/test_preflight_capacity.py new file mode 100644 index 000000000..aed667008 --- /dev/null +++ b/sunbeam-python/tests/unit/sunbeam/test_preflight_capacity.py @@ -0,0 +1,221 @@ +# SPDX-FileCopyrightText: 2025 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for capacity policy preflight check.""" + +import json +from unittest.mock import MagicMock + +import pytest + +from sunbeam.upgrades.preflight.capacity import ( + CapacityCheck, + CapacityPolicy, + load_capacity_policy, +) +from sunbeam.upgrades.preflight.checks import CheckContext + +FROM = "2025.1" +TO = "2026.1" + + +def _make_unit(name: str) -> tuple[str, MagicMock]: + return name, MagicMock() + + +def _make_app(units: dict[str, MagicMock]) -> MagicMock: + app = MagicMock() + app.units = units + return app + + +@pytest.fixture +def mock_client(): + client = MagicMock() + return client + + +@pytest.fixture +def mock_jhelper(): + return MagicMock() + + +@pytest.fixture +def mock_deployment(mock_client, mock_jhelper): + deployment = MagicMock() + deployment.openstack_machines_model = "openstack-machines" + deployment.get_client.return_value = mock_client + deployment.get_juju_helper.return_value = mock_jhelper + return deployment + + +@pytest.fixture +def ctx(mock_deployment): + return CheckContext( + deployment=mock_deployment, + from_release=FROM, + to_release=TO, + ) + + +# --------------------------------------------------------------------------- +# CapacityPolicy +# --------------------------------------------------------------------------- + + +class TestCapacityPolicy: + def test_defaults(self): + policy = CapacityPolicy() + assert policy.min_free_percentage == 25 + assert policy.min_free_nodes == 1 + + def test_custom_values(self): + policy = CapacityPolicy(min_free_percentage=50, min_free_nodes=3) + assert policy.min_free_percentage == 50 + assert policy.min_free_nodes == 3 + + +# --------------------------------------------------------------------------- +# load_capacity_policy +# --------------------------------------------------------------------------- + + +class TestLoadCapacityPolicy: + def test_returns_defaults_when_config_absent(self, mock_client): + mock_client.cluster.get_config.side_effect = Exception("not found") + policy = load_capacity_policy(mock_client) + assert policy.min_free_percentage == 25 + assert policy.min_free_nodes == 1 + + def test_loads_from_clusterd_json_string(self, mock_client): + mock_client.cluster.get_config.return_value = json.dumps( + {"min_free_percentage": 50, "min_free_nodes": 2} + ) + policy = load_capacity_policy(mock_client) + assert policy.min_free_percentage == 50 + assert policy.min_free_nodes == 2 + + def test_loads_from_clusterd_dict(self, mock_client): + mock_client.cluster.get_config.return_value = { + "min_free_percentage": 30, + "min_free_nodes": 3, + } + policy = load_capacity_policy(mock_client) + assert policy.min_free_percentage == 30 + assert policy.min_free_nodes == 3 + + +# --------------------------------------------------------------------------- +# CapacityCheck +# --------------------------------------------------------------------------- + + +class TestCapacityCheck: + def test_passes_when_enough_free_nodes(self, ctx, mock_jhelper): + ctx.client.cluster.get_config.side_effect = Exception("not found") + units = dict(_make_unit(f"openstack-hypervisor/{i}") for i in range(4)) + mock_jhelper.get_application.return_value = _make_app(units) + # 1 busy, 3 free + mock_jhelper.run_action.side_effect = [ + {"result": '["vm-1"]'}, + {"result": "[]"}, + {"result": "[]"}, + {"result": "[]"}, + ] + check = CapacityCheck(ctx) + assert check.run() is True + + def test_fails_when_no_free_nodes(self, ctx, mock_jhelper): + ctx.client.cluster.get_config.side_effect = Exception("not found") + units = dict(_make_unit(f"openstack-hypervisor/{i}") for i in range(2)) + mock_jhelper.get_application.return_value = _make_app(units) + mock_jhelper.run_action.side_effect = [ + {"result": '["vm-1"]'}, + {"result": '["vm-2"]'}, + ] + check = CapacityCheck(ctx) + assert check.run() is False + assert check.exit_code == 1 + assert "0 free" in check.message + assert "at least 1" in check.message + + def test_fails_when_free_percentage_below_policy(self, ctx, mock_jhelper): + ctx.client.cluster.get_config.return_value = json.dumps( + {"min_free_percentage": 50, "min_free_nodes": 1} + ) + units = dict(_make_unit(f"openstack-hypervisor/{i}") for i in range(4)) + mock_jhelper.get_application.return_value = _make_app(units) + # 3 busy, 1 free = 25% < 50% + mock_jhelper.run_action.side_effect = [ + {"result": '["vm-1"]'}, + {"result": '["vm-2"]'}, + {"result": '["vm-3"]'}, + {"result": "[]"}, + ] + check = CapacityCheck(ctx) + assert check.run() is False + assert "25%" in check.message + assert "50%" in check.message + + def test_override_skips_check(self, ctx, mock_jhelper): + check = CapacityCheck(ctx, override=True) + assert check.run() is True + mock_jhelper.get_application.assert_not_called() + + def test_fails_when_no_hypervisor_units(self, ctx, mock_jhelper): + ctx.client.cluster.get_config.side_effect = Exception("not found") + mock_jhelper.get_application.return_value = _make_app({}) + check = CapacityCheck(ctx) + assert check.run() is False + assert "No openstack-hypervisor units" in check.message + + def test_fails_when_app_not_found(self, ctx, mock_jhelper): + ctx.client.cluster.get_config.side_effect = Exception("not found") + mock_jhelper.get_application.side_effect = Exception("not deployed") + check = CapacityCheck(ctx) + assert check.run() is False + assert "Cannot find" in check.message + + def test_fails_when_action_raises(self, ctx, mock_jhelper): + ctx.client.cluster.get_config.side_effect = Exception("not found") + units = dict(_make_unit(f"openstack-hypervisor/{i}") for i in range(2)) + mock_jhelper.get_application.return_value = _make_app(units) + mock_jhelper.run_action.side_effect = Exception("action failed") + check = CapacityCheck(ctx) + assert check.run() is False + assert "action failed" in check.message + + def test_passes_with_all_nodes_free(self, ctx, mock_jhelper): + ctx.client.cluster.get_config.side_effect = Exception("not found") + units = dict(_make_unit(f"openstack-hypervisor/{i}") for i in range(3)) + mock_jhelper.get_application.return_value = _make_app(units) + mock_jhelper.run_action.side_effect = [ + {"result": "[]"}, + {"result": "[]"}, + {"result": "[]"}, + ] + check = CapacityCheck(ctx) + assert check.run() is True + + def test_handles_non_json_result_gracefully(self, ctx, mock_jhelper): + ctx.client.cluster.get_config.side_effect = Exception("not found") + units = dict(_make_unit(f"openstack-hypervisor/{i}") for i in range(2)) + mock_jhelper.get_application.return_value = _make_app(units) + mock_jhelper.run_action.side_effect = [ + {"result": "not-json"}, + {"result": "[]"}, + ] + check = CapacityCheck(ctx) + # Invalid JSON is treated as empty (free), so 2 free out of 2 + assert check.run() is True + + def test_handles_missing_result_key(self, ctx, mock_jhelper): + ctx.client.cluster.get_config.side_effect = Exception("not found") + units = dict(_make_unit(f"openstack-hypervisor/{i}") for i in range(2)) + mock_jhelper.get_application.return_value = _make_app(units) + mock_jhelper.run_action.side_effect = [ + {}, + {}, + ] + check = CapacityCheck(ctx) + assert check.run() is True diff --git a/sunbeam-python/tests/unit/sunbeam/test_preflight_checks.py b/sunbeam-python/tests/unit/sunbeam/test_preflight_checks.py index 2672cac73..fbc36589d 100644 --- a/sunbeam-python/tests/unit/sunbeam/test_preflight_checks.py +++ b/sunbeam-python/tests/unit/sunbeam/test_preflight_checks.py @@ -9,6 +9,7 @@ from sunbeam.clusterd.models import FeatureGates from sunbeam.upgrades.metadata import HopMetadata +from sunbeam.upgrades.preflight.capacity import CapacityCheck from sunbeam.upgrades.preflight.checks import ( CheckContext, ClusterHealthCheck, @@ -304,14 +305,15 @@ def test_fails_when_action_returns_empty(self, ctx): class TestBuildPreflightChecks: - def test_returns_four_checks_in_order(self, ctx): + def test_returns_five_checks_in_order(self, ctx): checks = build_preflight_checks(ctx) - assert len(checks) == 4 + assert len(checks) == 5 names = [type(c).__name__ for c in checks] assert names == [ "SnapVersionCheck", "HopMetadataCheck", "ClusterHealthCheck", + "CapacityCheck", "MySQLQuorumCheck", ] @@ -319,6 +321,11 @@ def test_all_checks_carry_exit_code(self, ctx): for check in build_preflight_checks(ctx): assert check.exit_code in (1, 2), f"{type(check).__name__} has no exit_code" + def test_capacity_override_flag(self, ctx): + checks = build_preflight_checks(ctx, capacity_override=True) + capacity_check = next(c for c in checks if isinstance(c, CapacityCheck)) + assert capacity_check.override is True + # --------------------------------------------------------------------------- # run_upgrade_preflight_checks From 7a4b334f2fce6e5650e9eb177d9347dcb5e3a37e Mon Sep 17 00:00:00 2001 From: Hemanth Nakkina Date: Tue, 18 Aug 2026 12:36:51 +0530 Subject: [PATCH 09/20] feat(upgrades): guard mutating commands during active upgrade hop Adds GuardedGroup (subclass of CatchGroup) to sunbeam/utils.py. Before invoking any subcommand in GUARDED_COMMANDS, checks is_upgrade_active() on clusterd. If an upgrade is in progress, raises ClickException with a clear message directing the operator to check status or abandon. Applied to cluster, enable, disable, configure, and storage command groups. Read-only commands (list, show, status) are not in the guarded set and pass through. The guard checks both ctx.info_name (for top-level groups like enable/disable) and ctx.invoked_subcommand (for nested groups like cluster refresh/join/bootstrap). 10 tests, all CI green. --- sunbeam-python/sunbeam/commands/configure.py | 3 +- sunbeam-python/sunbeam/main.py | 6 +- .../sunbeam/provider/local/commands.py | 4 +- .../sunbeam/provider/maas/commands.py | 3 +- sunbeam-python/sunbeam/storage/manager.py | 7 +- sunbeam-python/sunbeam/utils.py | 74 ++++++++++++++++ .../tests/unit/sunbeam/test_upgrade_guard.py | 86 +++++++++++++++++++ 7 files changed, 175 insertions(+), 8 deletions(-) create mode 100644 sunbeam-python/tests/unit/sunbeam/test_upgrade_guard.py diff --git a/sunbeam-python/sunbeam/commands/configure.py b/sunbeam-python/sunbeam/commands/configure.py index 9bcde1635..058cecd39 100644 --- a/sunbeam-python/sunbeam/commands/configure.py +++ b/sunbeam-python/sunbeam/commands/configure.py @@ -27,6 +27,7 @@ TerraformInitStep, ) from sunbeam.steps.configure import CLOUD_CONFIG_SECTION +from sunbeam.utils import GuardedGroup PCI_CONFIG_SECTION = "PCI" DPDK_CONFIG_SECTION = "DPDK" @@ -442,7 +443,7 @@ def _keep_cmd_params(cmd: click.Command, params: dict) -> dict: return out_params -@click.group(invoke_without_command=True) +@click.group(invoke_without_command=True, cls=GuardedGroup) @click.pass_context @click.option("-a", "--accept-defaults", help="Accept all defaults.", is_flag=True) @click.option( diff --git a/sunbeam-python/sunbeam/main.py b/sunbeam-python/sunbeam/main.py index 69efba9d3..b74918122 100644 --- a/sunbeam-python/sunbeam/main.py +++ b/sunbeam-python/sunbeam/main.py @@ -25,7 +25,7 @@ from sunbeam.feature_gates import FeatureGateError, validate_feature_gate_config from sunbeam.feature_manager import list_feature_gates, list_features from sunbeam.provider import commands as provider_cmds -from sunbeam.utils import CatchGroup, clean_env +from sunbeam.utils import CatchGroup, GuardedGroup, clean_env LOG = logging.getLogger() @@ -73,7 +73,7 @@ def proxy(ctx): """Manage proxy configuration.""" -@click.group("enable", context_settings=CONTEXT_SETTINGS, cls=CatchGroup) +@click.group("enable", context_settings=CONTEXT_SETTINGS, cls=GuardedGroup) @click.option( "-m", "--manifest", @@ -85,7 +85,7 @@ def enable(ctx, manifest: Path | None = None): """Enable features.""" -@click.group("disable", context_settings=CONTEXT_SETTINGS, cls=CatchGroup) +@click.group("disable", context_settings=CONTEXT_SETTINGS, cls=GuardedGroup) @click.pass_context def disable(ctx): """Disable features.""" diff --git a/sunbeam-python/sunbeam/provider/local/commands.py b/sunbeam-python/sunbeam/provider/local/commands.py index e3e51d8cc..1e0ff5051 100644 --- a/sunbeam-python/sunbeam/provider/local/commands.py +++ b/sunbeam-python/sunbeam/provider/local/commands.py @@ -198,7 +198,7 @@ ) from sunbeam.steps.sync_feature_gates import SyncFeatureGatesToCluster from sunbeam.utils import ( - CatchGroup, + GuardedGroup, click_option_show_hints, ) @@ -208,7 +208,7 @@ DEFAULT_LXD_CLOUD = "localhost" -@click.group("cluster", context_settings=CONTEXT_SETTINGS, cls=CatchGroup) +@click.group("cluster", context_settings=CONTEXT_SETTINGS, cls=GuardedGroup) @click.pass_context def cluster(ctx): """Manage the Sunbeam Cluster.""" diff --git a/sunbeam-python/sunbeam/provider/maas/commands.py b/sunbeam-python/sunbeam/provider/maas/commands.py index b456051fa..4c4d5462e 100644 --- a/sunbeam-python/sunbeam/provider/maas/commands.py +++ b/sunbeam-python/sunbeam/provider/maas/commands.py @@ -210,6 +210,7 @@ from sunbeam.utils import ( CatchGroup, DefaultableMappingParameter, + GuardedGroup, click_option_show_hints, ) @@ -217,7 +218,7 @@ console = Console() -@click.group("cluster", context_settings=CONTEXT_SETTINGS, cls=CatchGroup) +@click.group("cluster", context_settings=CONTEXT_SETTINGS, cls=GuardedGroup) @click.pass_context def cluster(ctx): """Manage the Sunbeam Cluster.""" diff --git a/sunbeam-python/sunbeam/storage/manager.py b/sunbeam-python/sunbeam/storage/manager.py index b6edaa715..3b057106e 100644 --- a/sunbeam-python/sunbeam/storage/manager.py +++ b/sunbeam-python/sunbeam/storage/manager.py @@ -19,6 +19,7 @@ from sunbeam.storage.base import StorageBackendBase from sunbeam.storage.models import BackendNotFoundException, StorageBackendInfo from sunbeam.storage.service import StorageBackendService +from sunbeam.utils import GuardedGroup LOG = logging.getLogger(__name__) console = Console() @@ -27,7 +28,11 @@ _STORAGE_BACKENDS: Dict[str, StorageBackendBase] = {} -@click.group("storage", context_settings={"help_option_names": ["-h", "--help"]}) +@click.group( + "storage", + context_settings={"help_option_names": ["-h", "--help"]}, + cls=GuardedGroup, +) @click.pass_context def storage(ctx): """Manage Cinder storage backends. diff --git a/sunbeam-python/sunbeam/utils.py b/sunbeam-python/sunbeam/utils.py index cbf3eb49e..3da8fc4f1 100644 --- a/sunbeam-python/sunbeam/utils.py +++ b/sunbeam-python/sunbeam/utils.py @@ -255,6 +255,80 @@ def __call__(self, *args, **kwargs): sys.exit(1) +# Command names that are blocked during an active upgrade hop. +# Read-only commands (list, show, status) are NOT here — they pass through. +GUARDED_COMMANDS: set[str] = { + "refresh", + "mysql", + "vault", + "k8s", + "bootstrap", + "add", + "join", + "remove", + "resize", + "destroy", + "configure", + "sriov", + "dpdk", + "enable", + "disable", +} + + +def check_upgrade_active(deployment: typing.Any) -> None: + """Raise ClickException if an upgrade hop is active. + + :param deployment: the deployment object (from click context) + :raises click.ClickException: if an upgrade is in progress + """ + try: + client = deployment.get_client() + except (SunbeamException, ValueError): + LOG.debug("Cannot get clusterd client — skipping upgrade guard") + return + + try: + if client.cluster.is_upgrade_active(): + raise click.ClickException( + "An upgrade is in progress. Complete or abandon the upgrade " + "before running this command. Use " + "'sunbeam cluster upgrade status' to check progress, or " + "'sunbeam cluster upgrade abandon' to abandon." + ) + except click.ClickException: + raise + except Exception: + LOG.debug("Cannot check upgrade state — skipping guard", exc_info=True) + + +class GuardedGroup(CatchGroup): + """A click.Group that guards mutating commands during active upgrades. + + Before invoking any subcommand whose name is in + ``GUARDED_COMMANDS``, checks if an upgrade hop is active. If so, + raises ``click.ClickException`` with a clear message. + + Read-only commands pass through unchanged. + """ + + def invoke(self, ctx: click.Context) -> None: + """Guard mutating subcommands before invoking.""" + # ctx.info_name is the group name (e.g. "enable", "disable") — + # checked for top-level guarded groups. + # ctx.invoked_subcommand is the subcommand name (e.g. "refresh", + # "join") — checked for nested groups like "cluster". + names = {ctx.info_name, ctx.invoked_subcommand} + names.discard(None) + + if names & GUARDED_COMMANDS: + deployment = ctx.obj + if deployment is not None and hasattr(deployment, "get_client"): + check_upgrade_active(deployment) + + return super().invoke(ctx) + + K = typing.TypeVar("K") V = typing.TypeVar("V") diff --git a/sunbeam-python/tests/unit/sunbeam/test_upgrade_guard.py b/sunbeam-python/tests/unit/sunbeam/test_upgrade_guard.py new file mode 100644 index 000000000..7c005f19d --- /dev/null +++ b/sunbeam-python/tests/unit/sunbeam/test_upgrade_guard.py @@ -0,0 +1,86 @@ +# SPDX-FileCopyrightText: 2025 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for upgrade guard framework.""" + +from unittest.mock import MagicMock + +import click +import pytest + +from sunbeam.utils import ( + GUARDED_COMMANDS, + GuardedGroup, + check_upgrade_active, +) + +# --------------------------------------------------------------------------- +# check_upgrade_active +# --------------------------------------------------------------------------- + + +class TestCheckUpgradeActive: + def _make_deployment(self, is_active=False, client_error=False): + deployment = MagicMock() + client = MagicMock() + client.cluster.is_upgrade_active.return_value = is_active + if client_error: + deployment.get_client.side_effect = ValueError("no client") + else: + deployment.get_client.return_value = client + return deployment + + def test_raises_when_upgrade_active(self): + deployment = self._make_deployment(is_active=True) + with pytest.raises(click.ClickException) as exc_info: + check_upgrade_active(deployment) + assert "upgrade is in progress" in str(exc_info.value.message).lower() + + def test_passes_when_no_upgrade(self): + deployment = self._make_deployment(is_active=False) + check_upgrade_active(deployment) + + def test_passes_when_client_unavailable(self): + deployment = self._make_deployment(client_error=True) + check_upgrade_active(deployment) + + def test_passes_when_clusterd_unreachable(self): + deployment = MagicMock() + client = MagicMock() + client.cluster.is_upgrade_active.side_effect = Exception("conn refused") + deployment.get_client.return_value = client + check_upgrade_active(deployment) + + +# --------------------------------------------------------------------------- +# GUARDED_COMMANDS +# --------------------------------------------------------------------------- + + +class TestGuardedCommands: + def test_refresh_is_guarded(self): + assert "refresh" in GUARDED_COMMANDS + + def test_bootstrap_is_guarded(self): + assert "bootstrap" in GUARDED_COMMANDS + + def test_list_is_not_guarded(self): + assert "list" not in GUARDED_COMMANDS + + def test_show_is_not_guarded(self): + assert "show" not in GUARDED_COMMANDS + + def test_status_is_not_guarded(self): + assert "status" not in GUARDED_COMMANDS + + +# --------------------------------------------------------------------------- +# GuardedGroup +# --------------------------------------------------------------------------- + + +class TestGuardedGroup: + def test_is_subclass_of_catch_group(self): + from sunbeam.utils import CatchGroup + + assert issubclass(GuardedGroup, CatchGroup) From 55080f608a588a6431014780dc3c4d162a540293 Mon Sep 17 00:00:00 2001 From: Hemanth Nakkina Date: Tue, 18 Aug 2026 12:42:55 +0530 Subject: [PATCH 10/20] feat(upgrades): add cluster upgrade abandon subcommand MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds the 'sunbeam cluster upgrade' command group with the 'abandon' subcommand. Abandon marks the active hop as abandoned (terminal state), releases the upgrade lock, and prints recovery guidance pointing at 'sunbeam restore' for operator-driven recovery. Confirmation prompt by default showing the hop's from->to release pair. --yes flag skips the prompt for automation. Fails with a clear message if no active hop exists or if the upgrade lock is held by another process. The upgrade command group is registered under 'cluster' in both local and MaaS providers. It does not use GuardedGroup — abandon must be runnable during an active upgrade. 8 tests, all CI green. --- sunbeam-python/sunbeam/commands/upgrade.py | 87 +++++++++++ .../sunbeam/provider/local/commands.py | 2 + .../sunbeam/provider/maas/commands.py | 2 + .../sunbeam/upgrades/coordinator.py | 1 + .../tests/unit/sunbeam/test_abandon.py | 142 ++++++++++++++++++ .../tests/unit/sunbeam/test_coordinator.py | 3 +- 6 files changed, 236 insertions(+), 1 deletion(-) create mode 100644 sunbeam-python/sunbeam/commands/upgrade.py create mode 100644 sunbeam-python/tests/unit/sunbeam/test_abandon.py diff --git a/sunbeam-python/sunbeam/commands/upgrade.py b/sunbeam-python/sunbeam/commands/upgrade.py new file mode 100644 index 000000000..f871da95f --- /dev/null +++ b/sunbeam-python/sunbeam/commands/upgrade.py @@ -0,0 +1,87 @@ +# SPDX-FileCopyrightText: 2025 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +"""Upgrade CLI commands. + +Provides the ``sunbeam cluster upgrade`` command group with +subcommands for managing the upgrade lifecycle. + +Current subcommands: +- abandon: mark the active hop as abandoned, release the lock, and + print recovery guidance. +""" + +from __future__ import annotations + +import click +from rich.console import Console + +from sunbeam.clusterd.service import UpgradeLockHeldException +from sunbeam.core.deployment import Deployment +from sunbeam.upgrades.coordinator import ReleaseUpgradeCoordinator +from sunbeam.upgrades.observability import UpgradeLogger + +console = Console() + + +@click.group("upgrade", context_settings={"help_option_names": ["-h", "--help"]}) +@click.pass_context +def upgrade(ctx: click.Context) -> None: + """Manage cluster upgrades.""" + + +@upgrade.command("abandon") +@click.option( + "--yes", + is_flag=True, + default=False, + help="Skip confirmation prompt.", +) +@click.pass_context +def abandon(ctx: click.Context, yes: bool) -> None: + """Abandon the active upgrade hop. + + Marks the active hop as abandoned, releases the upgrade lock, and + prints recovery guidance. The hop enters a terminal state — it + cannot be resumed. Recovery is operator-driven via + ``sunbeam restore`` and component-specific commands. + """ + deployment: Deployment = ctx.obj + client = deployment.get_client() + + coordinator = ReleaseUpgradeCoordinator(client, UpgradeLogger()) + coordinator.load_state() + hop = coordinator.get_current_hop() + if hop is None: + raise click.ClickException("No active upgrade hop to abandon.") + + if not yes: + click.echo( + f"This will abandon the active upgrade hop " + f"{hop.from_release} -> {hop.to_release}.\n" + "The hop will enter a terminal state and cannot be resumed.\n" + "Recovery is operator-driven via 'sunbeam restore'.\n" + ) + click.confirm("Are you sure?", abort=True) + + try: + coordinator.acquire_lock() + coordinator.load_state() + coordinator.abandon() + except UpgradeLockHeldException: + raise click.ClickException( + "Cannot acquire the upgrade lock — it is held by another process. " + "Wait for the lock to expire and retry, or identify and stop " + "the process holding the lock." + ) + finally: + coordinator.release_lock() + + click.echo( + f"\nUpgrade hop {hop.from_release} -> {hop.to_release} abandoned.\n\n" + "Recovery steps:\n" + " 1. Recover from failure using 'sunbeam restore' and\n" + " component-specific commands.\n" + " 2. Once recovered, start a new upgrade with\n" + " 'sunbeam cluster upgrade preflight'.\n" + ) diff --git a/sunbeam-python/sunbeam/provider/local/commands.py b/sunbeam-python/sunbeam/provider/local/commands.py index 1e0ff5051..12f445609 100644 --- a/sunbeam-python/sunbeam/provider/local/commands.py +++ b/sunbeam-python/sunbeam/provider/local/commands.py @@ -20,6 +20,7 @@ ) from sunbeam.commands import refresh as refresh_cmds from sunbeam.commands import resize as resize_cmds +from sunbeam.commands import upgrade as upgrade_cmds from sunbeam.commands.configure import ( DemoSetup, TerraformDemoInitStep, @@ -246,6 +247,7 @@ def register_cli( cluster.add_command(remove) cluster.add_command(resize_cmds.resize) cluster.add_command(refresh_cmds.refresh) + cluster.add_command(upgrade_cmds.upgrade) def deployment_type(self) -> Tuple[str, Type[Deployment]]: """Retrieve the deployment type and class.""" diff --git a/sunbeam-python/sunbeam/provider/maas/commands.py b/sunbeam-python/sunbeam/provider/maas/commands.py index 4c4d5462e..75009ef78 100644 --- a/sunbeam-python/sunbeam/provider/maas/commands.py +++ b/sunbeam-python/sunbeam/provider/maas/commands.py @@ -19,6 +19,7 @@ from sunbeam.clusterd.service import ConfigItemNotFoundException from sunbeam.commands import refresh as refresh_cmds from sunbeam.commands import resize as resize_cmds +from sunbeam.commands import upgrade as upgrade_cmds from sunbeam.commands.configure import ( DemoSetup, TerraformDemoInitStep, @@ -270,6 +271,7 @@ def register_cli( cluster.add_command(list_nodes) cluster.add_command(resize_cmds.resize) cluster.add_command(refresh_cmds.refresh) + cluster.add_command(upgrade_cmds.upgrade) cluster.add_command(remove_node) cluster.add_command(destroy_deployment_cmd) configure.add_command(configure_cmd) diff --git a/sunbeam-python/sunbeam/upgrades/coordinator.py b/sunbeam-python/sunbeam/upgrades/coordinator.py index 4351638bc..08fa7b822 100644 --- a/sunbeam-python/sunbeam/upgrades/coordinator.py +++ b/sunbeam-python/sunbeam/upgrades/coordinator.py @@ -520,6 +520,7 @@ def abandon(self) -> None: raise RuntimeError("no active hop to abandon") self._transition_hop(hop, HopStatus.ABANDONED) + self._state.active_hop.hop_history_index = None self.persist_state() self.logger.log_state_change("hop", "active_hop", "hop_abandoned", "abandoned") self.release_lock() diff --git a/sunbeam-python/tests/unit/sunbeam/test_abandon.py b/sunbeam-python/tests/unit/sunbeam/test_abandon.py new file mode 100644 index 000000000..e38f47791 --- /dev/null +++ b/sunbeam-python/tests/unit/sunbeam/test_abandon.py @@ -0,0 +1,142 @@ +# SPDX-FileCopyrightText: 2025 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for the abandon CLI command.""" + +import json +from unittest.mock import MagicMock + +import pytest +from click.testing import CliRunner + +from sunbeam.clusterd.models import AcquireUpgradeLockResponse +from sunbeam.clusterd.service import UpgradeLockHeldException +from sunbeam.commands.upgrade import upgrade + + +def _state_json( + from_release: str = "2025.1", + to_release: str = "2026.1", + status: str = "in_progress", +) -> str: + return json.dumps( + { + "active_hop": {"hop_history_index": 0}, + "hop_history": [ + { + "from": from_release, + "to": to_release, + "metadata_version": 1, + "metadata_build_id": "123", + "status": status, + } + ], + } + ) + + +@pytest.fixture +def mock_deployment_no_hop(): + deployment = MagicMock() + client = MagicMock() + client.cluster.acquire_upgrade_lock.return_value = AcquireUpgradeLockResponse( + token=1 + ) + client.cluster.get_upgrade_state.return_value = None + client.cluster.is_upgrade_active.return_value = False + deployment.get_client.return_value = client + return deployment + + +@pytest.fixture +def mock_deployment_with_hop(): + deployment = MagicMock() + client = MagicMock() + client.cluster.acquire_upgrade_lock.return_value = AcquireUpgradeLockResponse( + token=1 + ) + client.cluster.get_upgrade_state.return_value = _state_json() + client.cluster.is_upgrade_active.return_value = True + client.cluster.update_upgrade_state.return_value = None + client.cluster.release_upgrade_lock.return_value = None + deployment.get_client.return_value = client + return deployment + + +@pytest.fixture +def mock_deployment_lock_held(): + deployment = MagicMock() + client = MagicMock() + client.cluster.acquire_upgrade_lock.side_effect = UpgradeLockHeldException("held") + client.cluster.get_upgrade_state.return_value = _state_json() + deployment.get_client.return_value = client + return deployment + + +class TestAbandonCommand: + def test_abandon_with_yes_flag(self, mock_deployment_with_hop): + runner = CliRunner() + result = runner.invoke( + upgrade, ["abandon", "--yes"], obj=mock_deployment_with_hop + ) + assert result.exit_code == 0 + assert "abandoned" in result.output.lower() + client = mock_deployment_with_hop.get_client.return_value + client.cluster.update_upgrade_state.assert_called_once() + + def test_abandon_aborts_without_confirmation(self, mock_deployment_with_hop): + runner = CliRunner() + result = runner.invoke( + upgrade, ["abandon"], input="n\n", obj=mock_deployment_with_hop + ) + assert result.exit_code != 0 + + def test_abandon_with_confirmation(self, mock_deployment_with_hop): + runner = CliRunner() + result = runner.invoke( + upgrade, ["abandon"], input="y\n", obj=mock_deployment_with_hop + ) + assert result.exit_code == 0 + assert "abandoned" in result.output.lower() + + def test_abandon_no_active_hop(self, mock_deployment_no_hop): + runner = CliRunner() + result = runner.invoke( + upgrade, ["abandon", "--yes"], obj=mock_deployment_no_hop + ) + assert result.exit_code != 0 + assert "no active upgrade hop" in result.output.lower() + + def test_abandon_prints_recovery_guidance(self, mock_deployment_with_hop): + runner = CliRunner() + result = runner.invoke( + upgrade, ["abandon", "--yes"], obj=mock_deployment_with_hop + ) + assert result.exit_code == 0 + assert "sunbeam restore" in result.output + assert "sunbeam cluster upgrade preflight" in result.output + + def test_abandon_lock_held(self, mock_deployment_lock_held): + runner = CliRunner() + result = runner.invoke( + upgrade, ["abandon", "--yes"], obj=mock_deployment_lock_held + ) + assert result.exit_code != 0 + assert "lock" in result.output.lower() + + def test_abandon_releases_lock_after_success(self, mock_deployment_with_hop): + runner = CliRunner() + result = runner.invoke( + upgrade, ["abandon", "--yes"], obj=mock_deployment_with_hop + ) + assert result.exit_code == 0 + client = mock_deployment_with_hop.get_client.return_value + client.cluster.release_upgrade_lock.assert_called_once_with(1) + + def test_abandon_shows_hop_details_in_prompt(self, mock_deployment_with_hop): + runner = CliRunner() + result = runner.invoke( + upgrade, ["abandon"], input="n\n", obj=mock_deployment_with_hop + ) + assert "2025.1" in result.output + assert "2026.1" in result.output diff --git a/sunbeam-python/tests/unit/sunbeam/test_coordinator.py b/sunbeam-python/tests/unit/sunbeam/test_coordinator.py index c4081ad97..66663e73d 100644 --- a/sunbeam-python/tests/unit/sunbeam/test_coordinator.py +++ b/sunbeam-python/tests/unit/sunbeam/test_coordinator.py @@ -346,8 +346,9 @@ def test_abandon_marks_hop_and_releases_lock(self, coordinator, mock_client): coordinator.create_hop("2025.1", "2026.1", "rev-100") coordinator.abandon() - hop = coordinator.get_current_hop() + hop = coordinator.state.hop_history[0] assert hop.status == HopStatus.ABANDONED + assert coordinator.state.active_hop.hop_history_index is None assert coordinator.token is None def test_abandon_without_hop_raises(self, coordinator, mock_client): From 3ab1e7d9b9c070272239edc45a024f318e9ad77e Mon Sep 17 00:00:00 2001 From: Hemanth Nakkina Date: Tue, 18 Aug 2026 14:00:37 +0530 Subject: [PATCH 11/20] feat(upgrades): add active hop creation after preflight checks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds create_hop_after_preflight() which creates the active upgrade hop atomically after all preflight checks pass. The hop is created with status pending — the first mutating command transitions it to in_progress. Steps: acquire lock, load state, verify no active hop exists, get snap revision as metadata_build_id, create hop via coordinator, copy orchestration metadata to clusterd config key upgrade_metadata (so all nodes can read it), release lock. Releases the lock on any failure. Raises if the lock is held by another process or if an active hop already exists. 8 tests, all CI green. --- .../sunbeam/upgrades/preflight/hop.py | 89 ++++++++++++ .../tests/unit/sunbeam/test_preflight_hop.py | 134 ++++++++++++++++++ 2 files changed, 223 insertions(+) create mode 100644 sunbeam-python/sunbeam/upgrades/preflight/hop.py create mode 100644 sunbeam-python/tests/unit/sunbeam/test_preflight_hop.py diff --git a/sunbeam-python/sunbeam/upgrades/preflight/hop.py b/sunbeam-python/sunbeam/upgrades/preflight/hop.py new file mode 100644 index 000000000..29c284c67 --- /dev/null +++ b/sunbeam-python/sunbeam/upgrades/preflight/hop.py @@ -0,0 +1,89 @@ +# SPDX-FileCopyrightText: 2025 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +"""Active hop creation after preflight checks pass. + +This module bridges the preflight checks and the coordinator: after +all checks pass, it creates the active hop atomically and copies the +orchestration metadata to clusterd so it's available to all nodes. + +The hop is created with status ``pending``. The first mutating +command (control-plane, dataplane, storage) transitions it to +``in_progress``. +""" + +from __future__ import annotations + +import logging + +from snaphelpers import Snap + +from sunbeam.clusterd.client import Client +from sunbeam.upgrades.coordinator import ReleaseUpgradeCoordinator +from sunbeam.upgrades.metadata import HopMetadata +from sunbeam.upgrades.observability import UpgradeLogger +from sunbeam.upgrades.state import Hop + +LOG = logging.getLogger(__name__) + +# clusterd config key where the orchestration metadata is copied so +# all nodes can read it during the upgrade. +UPGRADE_METADATA_KEY = "upgrade_metadata" + + +def _get_snap_revision() -> str: + """Return the current snap revision.""" + return Snap().revision + + +def create_hop_after_preflight( + client: Client, + from_release: str, + to_release: str, + metadata: HopMetadata, +) -> Hop: + """Create the active hop after all preflight checks pass. + + Acquires the upgrade lock, creates the hop in persisted state with + status ``pending``, copies the orchestration metadata to clusterd, + and releases the lock. The hop is ready for the first mutating + command to transition it to ``in_progress``. + + :param client: clusterd client + :param from_release: source release (e.g. "2025.1") + :param to_release: target release (e.g. "2026.1") + :param metadata: the loaded orchestration metadata for this hop + :returns: the newly created Hop + :raises UpgradeLockHeldException: if the lock is held by another process + """ + coordinator = ReleaseUpgradeCoordinator(client, UpgradeLogger()) + + try: + coordinator.acquire_lock() + coordinator.load_state() + + if coordinator.get_current_hop() is not None: + raise RuntimeError( + "An active hop already exists. Abandon it before " + "starting a new upgrade." + ) + + metadata_build_id = _get_snap_revision() + hop = coordinator.create_hop(from_release, to_release, metadata_build_id) + + # Copy metadata to clusterd so all nodes can read it + client.cluster.update_config( + UPGRADE_METADATA_KEY, + metadata.model_dump(by_alias=True), + ) + + coordinator.persist_state() + LOG.info( + "created hop %s -> %s (build_id=%s)", + from_release, + to_release, + metadata_build_id, + ) + return hop + finally: + coordinator.release_lock() diff --git a/sunbeam-python/tests/unit/sunbeam/test_preflight_hop.py b/sunbeam-python/tests/unit/sunbeam/test_preflight_hop.py new file mode 100644 index 000000000..9b72e56fd --- /dev/null +++ b/sunbeam-python/tests/unit/sunbeam/test_preflight_hop.py @@ -0,0 +1,134 @@ +# SPDX-FileCopyrightText: 2025 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for active hop creation after preflight.""" + +import json +from unittest.mock import MagicMock, patch + +import pytest + +from sunbeam.clusterd.models import AcquireUpgradeLockResponse +from sunbeam.clusterd.service import UpgradeLockHeldException +from sunbeam.upgrades.metadata import HopMetadata +from sunbeam.upgrades.preflight.hop import ( + UPGRADE_METADATA_KEY, + create_hop_after_preflight, +) +from sunbeam.upgrades.state import HopStatus + +FROM = "2025.1" +TO = "2026.1" + + +def _make_metadata(from_release: str = FROM, to_release: str = TO) -> HopMetadata: + return HopMetadata.model_validate( + { + "from": from_release, + "to": to_release, + "control_plane_groups": [ + {"name": "identity-core", "apps": ["keystone-k8s"]} + ], + } + ) + + +@pytest.fixture +def mock_client(): + client = MagicMock() + client.cluster.acquire_upgrade_lock.return_value = AcquireUpgradeLockResponse( + token=1 + ) + client.cluster.get_upgrade_state.return_value = None + client.cluster.update_upgrade_state.return_value = None + client.cluster.release_upgrade_lock.return_value = None + client.cluster.update_config.return_value = None + return client + + +class TestCreateHopAfterPreflight: + def test_creates_hop_with_pending_status(self, mock_client): + with patch( + "sunbeam.upgrades.preflight.hop._get_snap_revision", + return_value="123", + ): + hop = create_hop_after_preflight(mock_client, FROM, TO, _make_metadata()) + assert hop.from_release == FROM + assert hop.to_release == TO + assert hop.status == HopStatus.PENDING + assert hop.metadata_build_id == "123" + + def test_writes_metadata_to_clusterd(self, mock_client): + metadata = _make_metadata() + with patch( + "sunbeam.upgrades.preflight.hop._get_snap_revision", + return_value="123", + ): + create_hop_after_preflight(mock_client, FROM, TO, metadata) + mock_client.cluster.update_config.assert_called_once() + args = mock_client.cluster.update_config.call_args + assert args[0][0] == UPGRADE_METADATA_KEY + assert args[0][1]["from"] == FROM + assert args[0][1]["to"] == TO + + def test_persists_state(self, mock_client): + with patch( + "sunbeam.upgrades.preflight.hop._get_snap_revision", + return_value="123", + ): + create_hop_after_preflight(mock_client, FROM, TO, _make_metadata()) + mock_client.cluster.update_upgrade_state.assert_called() + + def test_releases_lock_after_success(self, mock_client): + with patch( + "sunbeam.upgrades.preflight.hop._get_snap_revision", + return_value="123", + ): + create_hop_after_preflight(mock_client, FROM, TO, _make_metadata()) + mock_client.cluster.release_upgrade_lock.assert_called_once_with(1) + + def test_releases_lock_on_failure(self, mock_client): + mock_client.cluster.update_config.side_effect = Exception("config write failed") + with patch( + "sunbeam.upgrades.preflight.hop._get_snap_revision", + return_value="123", + ): + with pytest.raises(Exception): + create_hop_after_preflight(mock_client, FROM, TO, _make_metadata()) + mock_client.cluster.release_upgrade_lock.assert_called_once_with(1) + + def test_raises_if_lock_held(self, mock_client): + mock_client.cluster.acquire_upgrade_lock.side_effect = UpgradeLockHeldException( + "held" + ) + with pytest.raises(UpgradeLockHeldException): + create_hop_after_preflight(mock_client, FROM, TO, _make_metadata()) + + def test_raises_if_active_hop_already_exists(self, mock_client): + state = { + "active_hop": {"hop_history_index": 0}, + "hop_history": [ + { + "from": FROM, + "to": TO, + "metadata_version": 1, + "metadata_build_id": "999", + "status": "in_progress", + } + ], + } + mock_client.cluster.get_upgrade_state.return_value = json.dumps(state) + with patch( + "sunbeam.upgrades.preflight.hop._get_snap_revision", + return_value="123", + ): + with pytest.raises(RuntimeError, match="already exists"): + create_hop_after_preflight(mock_client, FROM, TO, _make_metadata()) + + def test_sets_metadata_build_id_from_snap_revision(self, mock_client): + with patch( + "sunbeam.upgrades.preflight.hop._get_snap_revision", + return_value="456", + ): + hop = create_hop_after_preflight(mock_client, FROM, TO, _make_metadata()) + assert hop.metadata_build_id == "456" From 59a4a8ee13653add6377a1b5c5c1a9a6b43783ad Mon Sep 17 00:00:00 2001 From: Hemanth Nakkina Date: Tue, 18 Aug 2026 14:27:13 +0530 Subject: [PATCH 12/20] feat(upgrades): add preflight CLI command Adds 'sunbeam cluster upgrade preflight' subcommand. Runs all preflight checks (snap version, hop/metadata, cluster health, capacity, MySQL quorum) and creates the active hop if all pass. --from auto-detected from deployed charm channels via detect_deployed_release() if omitted. --to auto-detected from detect_snap_release(). --capacity-policy-override skips the capacity check. On success: prints backup note, creates active hop with status pending, prints next step (control-plane --auto). On failure: exits with the check's exit code (1 = operational, 2 = invalid hop/metadata). 8 tests, all CI green. --- sunbeam-python/sunbeam/commands/upgrade.py | 106 ++++++++ .../sunbeam/upgrades/preflight/checks.py | 8 + .../tests/unit/sunbeam/test_preflight_cli.py | 229 ++++++++++++++++++ 3 files changed, 343 insertions(+) create mode 100644 sunbeam-python/tests/unit/sunbeam/test_preflight_cli.py diff --git a/sunbeam-python/sunbeam/commands/upgrade.py b/sunbeam-python/sunbeam/commands/upgrade.py index f871da95f..f33337e97 100644 --- a/sunbeam-python/sunbeam/commands/upgrade.py +++ b/sunbeam-python/sunbeam/commands/upgrade.py @@ -7,6 +7,7 @@ subcommands for managing the upgrade lifecycle. Current subcommands: +- preflight: run pre-flight checks and create the active hop. - abandon: mark the active hop as abandoned, release the lock, and print recovery guidance. """ @@ -18,18 +19,123 @@ from sunbeam.clusterd.service import UpgradeLockHeldException from sunbeam.core.deployment import Deployment +from sunbeam.core.openstack import OPENSTACK_MODEL from sunbeam.upgrades.coordinator import ReleaseUpgradeCoordinator from sunbeam.upgrades.observability import UpgradeLogger +from sunbeam.upgrades.preflight.checks import ( + CheckContext, + build_preflight_checks, + run_upgrade_preflight_checks, +) +from sunbeam.upgrades.preflight.hop import create_hop_after_preflight +from sunbeam.versions import detect_deployed_release, detect_snap_release console = Console() +def _detect_from_release(deployment: Deployment) -> str: + """Auto-detect the source release from deployed charm channels. + + Reads charm channels from the openstack model and matches against + RELEASE_TRACKS. Falls back to prompting the operator if detection + fails. + """ + jhelper = deployment.get_juju_helper() + status = jhelper.get_model_status(OPENSTACK_MODEL) + charm_channels: dict[str, str] = {} + for app_name, app in status.apps.items(): + if app.charm_channel: + charm_channels[app_name] = app.charm_channel + release = detect_deployed_release(charm_channels) + if release is None: + raise click.ClickException( + "Cannot auto-detect the deployed OpenStack release from charm " + "channels. Specify --from explicitly." + ) + return release + + @click.group("upgrade", context_settings={"help_option_names": ["-h", "--help"]}) @click.pass_context def upgrade(ctx: click.Context) -> None: """Manage cluster upgrades.""" +@upgrade.command("preflight") +@click.option( + "--from", + "from_release", + help="Source release (auto-detected if omitted).", + default=None, +) +@click.option( + "--capacity-policy-override", + is_flag=True, + default=False, + help="Skip the capacity policy check.", +) +@click.pass_context +def preflight( + ctx: click.Context, + from_release: str | None, + capacity_policy_override: bool, +) -> None: + """Run pre-flight checks and create the active upgrade hop. + + Validates cluster health, capacity, and metadata before starting + an upgrade. If all checks pass, creates the active hop with status + pending. + + Exit codes: + 0 — all checks passed, active hop created + 1 — operational failure (unhealthy cluster, capacity, MySQL) + 2 — invalid hop or missing metadata + """ + deployment: Deployment = ctx.obj + client = deployment.get_client() + + to_release = detect_snap_release() + if from_release is None: + from_release = _detect_from_release(deployment) + + click.echo(f"\nRunning pre-flight checks for {from_release} -> {to_release}...\n") + + ctx_obj = CheckContext( + deployment=deployment, + from_release=from_release, + to_release=to_release, + ) + checks = build_preflight_checks(ctx_obj, capacity_override=capacity_policy_override) + + try: + run_upgrade_preflight_checks(checks, console) + except click.ClickException: + raise + + click.echo(" All checks passed.\n") + click.echo( + " NOTE: Ensure backups are taken before proceeding (sunbeam backup).\n" + " The upgrade engine does not create or verify backups.\n" + ) + + metadata = ctx_obj.metadata + if metadata is None: + raise click.ClickException("Metadata not loaded after preflight checks.") + + try: + hop = create_hop_after_preflight(client, from_release, to_release, metadata) + except UpgradeLockHeldException: + raise click.ClickException( + "Cannot acquire the upgrade lock — it is held by another process. " + "Wait for the lock to expire and retry." + ) + except RuntimeError as e: + raise click.ClickException(str(e)) + + click.echo(f"Active hop created: {hop.from_release} -> {hop.to_release}") + click.echo("Next: sunbeam cluster upgrade control-plane --auto") + + @upgrade.command("abandon") @click.option( "--yes", diff --git a/sunbeam-python/sunbeam/upgrades/preflight/checks.py b/sunbeam-python/sunbeam/upgrades/preflight/checks.py index 99bcf607a..41c550b3e 100644 --- a/sunbeam-python/sunbeam/upgrades/preflight/checks.py +++ b/sunbeam-python/sunbeam/upgrades/preflight/checks.py @@ -173,6 +173,14 @@ def __init__(self, ctx: CheckContext): def run(self, check_status: Status | None = None) -> bool: """Return False if hop is invalid or metadata is missing/incompatible.""" + if self.ctx.from_release == self.ctx.to_release: + self.message = ( + f"Source and target releases are both " + f"{self.ctx.to_release!r}. The snap must be refreshed to the " + "target release before running the upgrade: " + "'sudo snap refresh openstack --channel=/stable'." + ) + return False if not is_valid_hop(self.ctx.from_release, self.ctx.to_release): self.message = ( f"Hop {self.ctx.from_release} -> {self.ctx.to_release} is " diff --git a/sunbeam-python/tests/unit/sunbeam/test_preflight_cli.py b/sunbeam-python/tests/unit/sunbeam/test_preflight_cli.py new file mode 100644 index 000000000..72decfd52 --- /dev/null +++ b/sunbeam-python/tests/unit/sunbeam/test_preflight_cli.py @@ -0,0 +1,229 @@ +# SPDX-FileCopyrightText: 2025 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for the preflight CLI command.""" + +from unittest.mock import MagicMock, patch + +import pytest +from click.testing import CliRunner + +from sunbeam.clusterd.models import AcquireUpgradeLockResponse +from sunbeam.clusterd.service import UpgradeLockHeldException +from sunbeam.commands.upgrade import upgrade +from sunbeam.upgrades.metadata import HopMetadata +from sunbeam.upgrades.preflight.capacity import CapacityPolicy + +FROM = "2025.1" +TO = "2026.1" + + +def _make_app(current="active", charm_channel="2025.1/stable"): + app = MagicMock() + app.app_status.current = current + app.app_status.message = "" + app.charm_channel = charm_channel + return app + + +def _model_status(apps): + status = MagicMock() + status.apps = apps + return status + + +def _make_metadata(): + return HopMetadata.model_validate( + { + "from": FROM, + "to": TO, + "control_plane_groups": [ + {"name": "identity-core", "apps": ["keystone-k8s"]} + ], + } + ) + + +@pytest.fixture +def mock_deployment(): + deployment = MagicMock() + client = MagicMock() + client.cluster.acquire_upgrade_lock.return_value = AcquireUpgradeLockResponse( + token=1 + ) + client.cluster.get_upgrade_state.return_value = None + client.cluster.update_upgrade_state.return_value = None + client.cluster.release_upgrade_lock.return_value = None + client.cluster.update_config.return_value = None + deployment.get_client.return_value = client + + jhelper = MagicMock() + jhelper.get_model_status.side_effect = [ + _model_status( + { + "keystone-k8s": _make_app("active", "2025.1/stable"), + "nova-k8s": _make_app("active", "2025.1/stable"), + "mysql-k8s": _make_app("active", "8.0/stable"), + } + ), + _model_status( + { + "nova-compute": _make_app("active"), + "openstack-network-agents": _make_app("active"), + } + ), + ] + jhelper.get_leader_unit.return_value = "mysql-k8s/0" + jhelper.run_action.return_value = {"cluster-status": "ok"} + jhelper.get_application.return_value = MagicMock( + units={"openstack-hypervisor/0": MagicMock()} + ) + deployment.get_juju_helper.return_value = jhelper + return deployment + + +def _preflight_patches(**overrides): + """Common patches for preflight tests. All checks pass by default.""" + defaults = { + "snap_release": TO, + "metadata": _make_metadata(), + "snap_revision": "123", + "capacity_policy": CapacityPolicy(), + } + defaults.update(overrides) + return [ + patch( + "sunbeam.commands.upgrade.detect_snap_release", + return_value=defaults["snap_release"], + ), + patch( + "sunbeam.upgrades.preflight.checks.detect_snap_release", + return_value=defaults["snap_release"], + ), + patch( + "sunbeam.upgrades.preflight.checks.load_upgrade_metadata", + return_value=defaults["metadata"], + ), + patch( + "sunbeam.upgrades.preflight.hop._get_snap_revision", + return_value=defaults["snap_revision"], + ), + patch( + "sunbeam.upgrades.preflight.capacity.load_capacity_policy", + return_value=defaults["capacity_policy"], + ), + ] + + +class TestPreflightCommand: + def test_preflight_success_creates_hop(self, mock_deployment): + runner = CliRunner() + for p in _preflight_patches(): + p.start() + try: + result = runner.invoke( + upgrade, ["preflight", "--from", FROM], obj=mock_deployment + ) + finally: + patch.stopall() + assert result.exit_code == 0 + assert "All checks passed" in result.output + assert "Active hop created" in result.output + + def test_preflight_auto_detects_from(self, mock_deployment): + runner = CliRunner() + patches = _preflight_patches() + patches.append( + patch("sunbeam.commands.upgrade._detect_from_release", return_value=FROM) + ) + for p in patches: + p.start() + try: + result = runner.invoke(upgrade, ["preflight"], obj=mock_deployment) + finally: + patch.stopall() + assert result.exit_code == 0 + assert f"{FROM} -> {TO}" in result.output + + def test_preflight_fails_on_stale_snap(self, mock_deployment): + runner = CliRunner() + for p in _preflight_patches(snap_release=FROM): + p.start() + try: + result = runner.invoke( + upgrade, ["preflight", "--from", FROM], obj=mock_deployment + ) + finally: + patch.stopall() + assert result.exit_code != 0 + assert "exit 2" in result.output + + def test_preflight_capacity_override_flag(self, mock_deployment): + runner = CliRunner() + for p in _preflight_patches(): + p.start() + try: + result = runner.invoke( + upgrade, + ["preflight", "--from", FROM, "--capacity-policy-override"], + obj=mock_deployment, + ) + finally: + patch.stopall() + assert result.exit_code == 0 + + def test_preflight_lock_held(self, mock_deployment): + client = mock_deployment.get_client.return_value + client.cluster.acquire_upgrade_lock.side_effect = UpgradeLockHeldException( + "held" + ) + runner = CliRunner() + for p in _preflight_patches(): + p.start() + try: + result = runner.invoke( + upgrade, ["preflight", "--from", FROM], obj=mock_deployment + ) + finally: + patch.stopall() + assert result.exit_code != 0 + assert "lock" in result.output.lower() + + def test_preflight_prints_backup_note(self, mock_deployment): + runner = CliRunner() + for p in _preflight_patches(): + p.start() + try: + result = runner.invoke( + upgrade, ["preflight", "--from", FROM], obj=mock_deployment + ) + finally: + patch.stopall() + assert result.exit_code == 0 + assert "sunbeam backup" in result.output + + def test_preflight_prints_next_step(self, mock_deployment): + runner = CliRunner() + for p in _preflight_patches(): + p.start() + try: + result = runner.invoke( + upgrade, ["preflight", "--from", FROM], obj=mock_deployment + ) + finally: + patch.stopall() + assert result.exit_code == 0 + assert "control-plane" in result.output + + +class TestDetectFromRelease: + def test_auto_detect_fails_without_charm_channels(self, mock_deployment): + # Override jhelper to return empty model status so + # detect_deployed_release returns None + jhelper = mock_deployment.get_juju_helper.return_value + jhelper.get_model_status.side_effect = None + jhelper.get_model_status.return_value = _model_status({}) + runner = CliRunner() + result = runner.invoke(upgrade, ["preflight"], obj=mock_deployment) + assert result.exit_code != 0 + assert "auto-detect" in result.output.lower() From f4556df910cb452d4d5dfc377f8427fc35df3d94 Mon Sep 17 00:00:00 2001 From: Hemanth Nakkina Date: Tue, 18 Aug 2026 16:45:37 +0530 Subject: [PATCH 13/20] feat(upgrades): add control-plane group handler with scoped terraform apply Adds ControlPlaneHandler implementing the PhaseHandler protocol. Reads upgrade groups from orchestration metadata, upgrades each group via scoped terraform apply (update_partial_tfvars_and_apply_tf), waits for convergence via wait_until_desired_status, and persists per-group state transitions. Groups are upgraded in metadata-defined order. Completed groups are skipped on resume. Failed groups are re-attempted on retry (terraform apply is idempotent). On terraform failure: CONTROL_PLANE_APPLY_FAILED. On convergence timeout: CONTROL_PLANE_CONVERGENCE_TIMEOUT. The handler lazily loads tfhelper, manifest, and jhelper from the Deployment object. 10 tests, all CI green. --- .../upgrades/control_plane/__init__.py | 3 + .../sunbeam/upgrades/control_plane/groups.py | 573 ++++++++++++++++++ .../unit/sunbeam/test_control_plane_groups.py | 348 +++++++++++ 3 files changed, 924 insertions(+) create mode 100644 sunbeam-python/sunbeam/upgrades/control_plane/__init__.py create mode 100644 sunbeam-python/sunbeam/upgrades/control_plane/groups.py create mode 100644 sunbeam-python/tests/unit/sunbeam/test_control_plane_groups.py diff --git a/sunbeam-python/sunbeam/upgrades/control_plane/__init__.py b/sunbeam-python/sunbeam/upgrades/control_plane/__init__.py new file mode 100644 index 000000000..70e4efd93 --- /dev/null +++ b/sunbeam-python/sunbeam/upgrades/control_plane/__init__.py @@ -0,0 +1,3 @@ +# SPDX-FileCopyrightText: 2025 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 +"""Control-plane upgrade phase handler.""" diff --git a/sunbeam-python/sunbeam/upgrades/control_plane/groups.py b/sunbeam-python/sunbeam/upgrades/control_plane/groups.py new file mode 100644 index 000000000..ad3e06145 --- /dev/null +++ b/sunbeam-python/sunbeam/upgrades/control_plane/groups.py @@ -0,0 +1,573 @@ +# SPDX-FileCopyrightText: 2025 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +"""Group resolution and scoped terraform apply for control-plane upgrades. + +The control-plane handler reads upgrade groups from the orchestration +metadata, upgrades each group via scoped terraform apply, waits for +convergence, and persists per-group state. + +Each group is upgraded independently: if one group fails, the handler +returns failure and the operator can retry that group. Completed +groups are skipped on resume. +""" + +from __future__ import annotations + +import datetime +import logging +import typing + +import click +import yaml +from rich.console import Console + +from sunbeam.clusterd.client import Client +from sunbeam.core.common import RiskLevel, infer_risk +from sunbeam.core.deployment import Deployment +from sunbeam.core.juju import JujuHelper, JujuWaitException +from sunbeam.core.manifest import Manifest, embedded_manifest_path +from sunbeam.core.openstack import OPENSTACK_MODEL +from sunbeam.core.terraform import TerraformException, TerraformHelper +from sunbeam.steps.openstack import CONFIG_KEY as OPENSTACK_CONFIG_KEY +from sunbeam.upgrades.control_plane.actions import ( + run_post_actions, + run_pre_actions, +) +from sunbeam.upgrades.coordinator import ( + PhaseResult, + ReleaseUpgradeCoordinator, +) +from sunbeam.upgrades.errors import UpgradeErrorCode +from sunbeam.upgrades.metadata import HopMetadata +from sunbeam.upgrades.state import Group, LastError, PhaseStatus, UpgradeState + +LOG = logging.getLogger(__name__) +console = Console() + + +def _now_iso() -> str: + return datetime.datetime.now(tz=datetime.timezone.utc).isoformat() + + +def _load_target_charm_manifests(to_release: str) -> dict[str, dict]: + """Load per-charm manifest entries from the snap's embedded target manifest. + + Reads ``etc/manifests//.yml`` from the snap and + returns a mapping of charm name to its raw manifest dict (channel, + revision, config, resources — whatever the embedded manifest carries). + + :param to_release: target release, e.g. "2026.1" + :returns: charm name -> raw manifest dict, or empty dict if not found + """ + from snaphelpers import Snap + + try: + snap = Snap() + risk = infer_risk(snap) + if risk == RiskLevel.STABLE: + risk_str = "stable" + else: + risk_str = str(risk) + path = embedded_manifest_path(snap, to_release, risk_str) + data = yaml.safe_load(path.read_text()) + charms = data.get("core", {}).get("software", {}).get("charms", {}) + return {name: cfg for name, cfg in charms.items()} + except Exception as e: + LOG.warning("Could not load target manifest for %s: %s", to_release, e) + return {} + + +def _terraform_targets_for_charms( + charms: list[str], terraform_targets: dict[str, list[str]] +) -> list[str]: + """Build terraform ``-target`` CLI args from the group's metadata. + + Reads the ``terraform_targets`` mapping from the upgrade metadata + (``upgrade.yml``). Each charm maps to one or more terraform resource + addresses. Integrations are NOT targeted — they are applied in the + reapply-terraform finalize step (full apply without -target) after + both ends are upgraded. + + :param charms: charm names being upgraded + :param terraform_targets: per-app target addresses from group metadata + :returns: list of ``-target=
`` strings for terraform CLI + :raises KeyError: if any charm is not in the mapping + """ + targets: list[str] = [] + for charm in charms: + for addr in terraform_targets[charm]: + targets.append(f"-target={addr}") + return targets + + +class ControlPlaneHandler: + """Phase handler for the control-plane upgrade. + + Reads groups from metadata, upgrades each via scoped terraform + apply, waits for convergence, and persists per-group state. + + Implements the PhaseHandler protocol. + """ + + def __init__(self, deployment: Deployment, to_release: str = ""): + self.deployment = deployment + self.to_release = to_release + self._tfhelper: TerraformHelper | None = None + self._manifest: Manifest | None = None + self._jhelper: JujuHelper | None = None + self._target_channels: dict[str, dict] | None = None + + @property + def tfhelper(self) -> TerraformHelper: + """Return the TerraformHelper for the openstack plan.""" + if self._tfhelper is None: + self._tfhelper = self.deployment.get_tfhelper("openstack-plan") + return self._tfhelper + + @property + def manifest(self) -> Manifest: + """Return the deployment manifest.""" + if self._manifest is None: + self._manifest = self.deployment.get_manifest() + return self._manifest + + @property + def jhelper(self) -> JujuHelper: + """Return a JujuHelper for this deployment.""" + if self._jhelper is None: + self._jhelper = self.deployment.get_juju_helper() + return self._jhelper + + @property + def target_charms(self) -> dict[str, dict]: + """Per-charm manifest entries from the snap's embedded target manifest.""" + if self._target_channels is None: + self._target_channels = _load_target_charm_manifests(self.to_release) + return self._target_channels + + def _override_charm_manifests(self, charms: list[str]) -> None: + """Override manifest entries for the group's charms from the target release. + + Replaces the full CharmManifest (channel, revision, config, resources) + for the group's charms with values from the snap's embedded target + manifest. Other charms keep their current (clusterd) values. + """ + if not self.to_release: + return + from sunbeam.core.manifest import CharmManifest + + for charm in charms: + target_cfg = self.target_charms.get(charm) + if target_cfg and charm in self.manifest.core.software.charms: + old = self.manifest.core.software.charms[charm].channel + self.manifest.core.software.charms[charm] = CharmManifest( + **target_cfg + ) + LOG.info( + "overrode %s channel: %s -> %s", + charm, + old, + target_cfg.get("channel"), + ) + elif target_cfg: + LOG.warning( + "charm %s not in deployment manifest, skipping override", + charm, + ) + + def run( + self, + coordinator: ReleaseUpgradeCoordinator, + metadata: HopMetadata | None, + state: UpgradeState, + ) -> PhaseResult: + """Execute the control-plane upgrade. + + Upgrades each metadata group in order. Skips completed groups + (resume). Returns failure on the first failed group. + """ + if metadata is None: + return PhaseResult( + success=False, + error_code=UpgradeErrorCode.METADATA_MISSING, + error_message="No metadata loaded for control-plane phase", + ) + + hop = coordinator.get_current_hop() + if hop is None: + return PhaseResult( + success=False, + error_code=UpgradeErrorCode.HOP_INVALID_TRANSITION, + error_message="No active hop", + ) + + control_plane = hop.phases.control_plane + + for group_meta in metadata.control_plane_groups: + group_name = group_meta.name + + group_state = control_plane.groups.get(group_name) + if group_state is None: + group_state = Group() + control_plane.groups[group_name] = group_state + + if group_state.status == PhaseStatus.COMPLETED: + LOG.info("skipping completed group %s", group_name) + continue + + group_state.status = PhaseStatus.IN_PROGRESS + group_state.started_at = _now_iso() + coordinator.persist_state() + + result = self._upgrade_group(group_meta, group_name) + + if result.success: + group_state.status = PhaseStatus.COMPLETED + group_state.completed_at = _now_iso() + coordinator.persist_state() + LOG.info("group %s completed", group_name) + else: + group_state.status = PhaseStatus.FAILED + group_state.last_error = LastError( + code=result.error_code.value if result.error_code else "", + message=result.error_message or "", + ) + coordinator.persist_state() + LOG.warning("group %s failed: %s", group_name, result.error_message) + return result + + return PhaseResult(success=True) + + def plan_group( + self, + group_meta: typing.Any, + ) -> list[dict]: + """Run terraform plan for a single group and return change events. + + Updates tfvars for the group's charms and runs ``terraform plan`` + without applying. Does not modify clusterd state. + + Runs both JSON and text plan: JSON events are returned for + user-facing display; text output is logged at debug level for + human-readable ``+``/``-`` diff in the sunbeam log file. + + :param group_meta: ControlPlaneGroup metadata for the group + :returns: list of terraform plan JSON events + :raises TerraformException: if the plan command fails + """ + client = self.deployment.get_client() + charms = group_meta.apps + target_args = _terraform_targets_for_charms( + charms, group_meta.terraform_targets + ) + + self._override_charm_manifests(charms) + self.tfhelper.init() + events = self.tfhelper.update_partial_tfvars_and_plan_tf( + client, + self.manifest, + charms, + OPENSTACK_CONFIG_KEY, + tf_plan_extra_args=target_args, + ) + + # Run text plan for debug logging (tfvars already written above) + try: + plan_text = self.tfhelper.terraform_plan_text( + extra_args=target_args + ) + LOG.debug( + "Terraform plan (text) for group %s:\n%s", + group_meta.name, + plan_text, + ) + except TerraformException as e: + LOG.warning( + "Terraform text plan failed for group %s: %s", + group_meta.name, + e, + ) + + return events + + def run_group( + self, + coordinator: ReleaseUpgradeCoordinator, + metadata: HopMetadata | None, + group_name: str, + ) -> PhaseResult: + """Upgrade a single group by name. + + Does not skip completed groups — use run() for resume behavior. + """ + if metadata is None: + return PhaseResult( + success=False, + error_code=UpgradeErrorCode.METADATA_MISSING, + error_message="No metadata loaded", + ) + + hop = coordinator.get_current_hop() + if hop is None: + return PhaseResult( + success=False, + error_code=UpgradeErrorCode.HOP_INVALID_TRANSITION, + error_message="No active hop", + ) + + group_meta = None + for g in metadata.control_plane_groups: + if g.name == group_name: + group_meta = g + break + if group_meta is None: + return PhaseResult( + success=False, + error_code=UpgradeErrorCode.METADATA_INVALID, + error_message=f"Group {group_name} not found in metadata", + ) + + control_plane = hop.phases.control_plane + group_state = control_plane.groups.get(group_name) + if group_state is None: + group_state = Group() + control_plane.groups[group_name] = group_state + + group_state.status = PhaseStatus.IN_PROGRESS + group_state.started_at = _now_iso() + coordinator.persist_state() + + result = self._upgrade_group(group_meta, group_name) + if result.success: + group_state.status = PhaseStatus.COMPLETED + group_state.completed_at = _now_iso() + else: + group_state.status = PhaseStatus.FAILED + group_state.last_error = LastError( + code=result.error_code.value if result.error_code else "", + message=result.error_message or "", + ) + coordinator.persist_state() + return result + + def run_application( + self, + coordinator: ReleaseUpgradeCoordinator, + metadata: HopMetadata | None, + charm_name: str, + ) -> PhaseResult: + """Upgrade a single application via scoped terraform apply. + + Does not run pre/post actions (those are group-level). + + :param charm_name: charm name (e.g. 'placement-k8s'), resolved + from the juju app name by the CLI. + """ + if metadata is None: + return PhaseResult( + success=False, + error_code=UpgradeErrorCode.METADATA_MISSING, + error_message="No metadata loaded", + ) + + hop = coordinator.get_current_hop() + if hop is None: + return PhaseResult( + success=False, + error_code=UpgradeErrorCode.HOP_INVALID_TRANSITION, + error_message="No active hop", + ) + + # Verify the charm exists in some group + group_meta = None + for g in metadata.control_plane_groups: + if charm_name in g.apps: + group_meta = g + break + if group_meta is None: + return PhaseResult( + success=False, + error_code=UpgradeErrorCode.METADATA_INVALID, + error_message=f"Charm {charm_name} not found in any group", + ) + + # Mark group as in_progress (single-app upgrade doesn't complete the group) + control_plane = hop.phases.control_plane + group_state = control_plane.groups.get(group_meta.name) + if group_state is None: + group_state = Group() + control_plane.groups[group_meta.name] = group_state + if group_state.status != PhaseStatus.COMPLETED: + group_state.status = PhaseStatus.IN_PROGRESS + group_state.started_at = _now_iso() + coordinator.persist_state() + + client = self.deployment.get_client() + target_args = _terraform_targets_for_charms( + [charm_name], group_meta.terraform_targets + ) + self._override_charm_manifests([charm_name]) + + # Resolve charm name to deployed juju app name for Juju operations + status = self.jhelper.get_model_status(OPENSTACK_MODEL) + juju_app_names = [ + name for name, app in status.apps.items() if app.charm == charm_name + ] + if not juju_app_names: + return PhaseResult( + success=False, + error_code=UpgradeErrorCode.METADATA_INVALID, + error_message=f"No deployed application found for charm {charm_name}", + ) + + try: + with console.status(f" {charm_name}: applying terraform plan..."): + self.tfhelper.init() + self.tfhelper.update_partial_tfvars_and_apply_tf( + client, + self.manifest, + [charm_name], + OPENSTACK_CONFIG_KEY, + tf_apply_extra_args=target_args, + ) + except TerraformException as e: + click.echo(f" {charm_name}: terraform apply FAILED") + return PhaseResult( + success=False, + error_code=UpgradeErrorCode.CONTROL_PLANE_APPLY_FAILED, + error_message=f"Terraform init/apply failed for {charm_name}: {e}", + ) + + try: + with console.status(f" {charm_name}: waiting for convergence..."): + self.jhelper.wait_until_desired_status( + OPENSTACK_MODEL, + juju_app_names, + status=["active"], + timeout=600, + ) + except (JujuWaitException, TimeoutError) as e: + click.echo(f" {charm_name}: convergence timeout") + return PhaseResult( + success=False, + error_code=UpgradeErrorCode.CONTROL_PLANE_CONVERGENCE_TIMEOUT, + error_message=f"{charm_name} did not converge: {e}", + ) + + # If this was the only app in the group, mark group as completed + if len(group_meta.apps) == 1: + group_state.status = PhaseStatus.COMPLETED + group_state.completed_at = _now_iso() + coordinator.persist_state() + click.echo(f" {charm_name}: completed") + + return PhaseResult(success=True) + + def _upgrade_group( + self, + group_meta: typing.Any, + group_name: str, + ) -> PhaseResult: + """Upgrade a single group: pre-actions, apply, converge, post-actions.""" + charms = group_meta.apps + LOG.info( + "upgrading group %s: apps=%s timeout=%ds", + group_name, + charms, + group_meta.ready_timeout_sec, + ) + + # Resolve charm names to deployed app names, skipping undeployed charms + status = self.jhelper.get_model_status(OPENSTACK_MODEL) + deployed_charms = {app.charm for app in status.apps.values()} + charms = [c for c in charms if c in deployed_charms] + app_names = [ + name for name, app in status.apps.items() if app.charm in charms + ] + if not charms: + LOG.info("group %s has no deployed charms — skipping", group_name) + click.echo(f" {group_name}: no deployed charms — skipping") + return PhaseResult(success=True) + LOG.info("resolved charms %s to apps %s", charms, app_names) + + # Pre-upgrade actions (use app names, not charm names) + with console.status(f" {group_name}: running pre-upgrade actions..."): + pre_result = run_pre_actions(self.jhelper, group_meta.pre_actions) + if not pre_result.success: + click.echo(f" {group_name}: pre-upgrade actions FAILED") + # Still run post-actions as cleanup attempt + run_post_actions(self.jhelper, group_meta.post_actions) + return pre_result + + client = self.deployment.get_client() + target_args = _terraform_targets_for_charms( + charms, group_meta.terraform_targets + ) + + self._override_charm_manifests(charms) + try: + # ponytail: init re-resolves providers from the snap mirror; a snap + # refresh can bump the juju provider version, leaving the .terraform + # dir stale and apply failing with "unavailable provider". + with console.status(f" {group_name}: applying terraform plan..."): + self.tfhelper.init() + self.tfhelper.update_partial_tfvars_and_apply_tf( + client, + self.manifest, + charms, + OPENSTACK_CONFIG_KEY, + tf_apply_extra_args=target_args, + ) + except TerraformException as e: + click.echo(f" {group_name}: terraform apply FAILED") + # Run post-actions as cleanup even on failure + run_post_actions(self.jhelper, group_meta.post_actions) + return PhaseResult( + success=False, + error_code=UpgradeErrorCode.CONTROL_PLANE_APPLY_FAILED, + error_message=f"Terraform init/apply failed for {group_name}: {e}", + ) + + try: + with console.status( + f" {group_name}: waiting for convergence " + f"({group_meta.ready_timeout_sec}s)..." + ): + self.jhelper.wait_until_desired_status( + OPENSTACK_MODEL, + app_names, + status=["active"], + timeout=group_meta.ready_timeout_sec, + ) + except (JujuWaitException, TimeoutError) as e: + click.echo(f" {group_name}: convergence timeout") + # Run post-actions as cleanup even on failure + run_post_actions(self.jhelper, group_meta.post_actions) + return PhaseResult( + success=False, + error_code=UpgradeErrorCode.CONTROL_PLANE_CONVERGENCE_TIMEOUT, + error_message=( + f"Group {group_name} did not converge within " + f"{group_meta.ready_timeout_sec}s: {e}" + ), + ) + + # Post-upgrade actions + with console.status(f" {group_name}: running post-upgrade actions..."): + post_result = run_post_actions(self.jhelper, group_meta.post_actions) + if not post_result.success: + click.echo(f" {group_name}: post-upgrade actions FAILED") + return PhaseResult( + success=False, + error_code=UpgradeErrorCode.CONTROL_PLANE_ACTION_FAILED, + error_message=( + f"Post-upgrade action failed for group {group_name}: " + f"{post_result.error_message}. The upgrade applied " + "successfully but post-upgrade actions may not have " + "completed — manual intervention may be required." + ), + ) + + click.echo(f" {group_name}: completed") + return PhaseResult(success=True) diff --git a/sunbeam-python/tests/unit/sunbeam/test_control_plane_groups.py b/sunbeam-python/tests/unit/sunbeam/test_control_plane_groups.py new file mode 100644 index 000000000..7a0e3f651 --- /dev/null +++ b/sunbeam-python/tests/unit/sunbeam/test_control_plane_groups.py @@ -0,0 +1,348 @@ +# SPDX-FileCopyrightText: 2025 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for the control-plane upgrade handler.""" + +from unittest.mock import MagicMock + +import pytest + +from sunbeam.clusterd.models import AcquireUpgradeLockResponse +from sunbeam.core.terraform import TerraformException +from sunbeam.upgrades.control_plane.groups import ( + ControlPlaneHandler, + _terraform_targets_for_charms, +) +from sunbeam.upgrades.coordinator import ReleaseUpgradeCoordinator +from sunbeam.upgrades.errors import UpgradeErrorCode +from sunbeam.upgrades.metadata import HopMetadata +from sunbeam.upgrades.state import Group, HopStatus, PhaseStatus, UpgradeState + +FROM = "2025.1" +TO = "2026.1" + +_TF_TARGETS = { + "keystone-k8s": ["module.keystone"], + "glance-k8s": ["module.glance"], + "ceilometer-k8s": ["juju_application.ceilometer"], + "heat-k8s": ["module.heat"], +} + + +class TestTerraformTargetsForCharms: + """Tests for _terraform_targets_for_charms.""" + + def test_single_charm(self): + targets = _terraform_targets_for_charms(["keystone-k8s"], _TF_TARGETS) + assert targets == ["-target=module.keystone"] + + def test_ceilometer_maps_to_juju_application(self): + targets = _terraform_targets_for_charms(["ceilometer-k8s"], _TF_TARGETS) + assert targets == ["-target=juju_application.ceilometer"] + + def test_multiple_charms(self): + targets = _terraform_targets_for_charms( + ["keystone-k8s", "glance-k8s", "ceilometer-k8s"], _TF_TARGETS + ) + assert targets == [ + "-target=module.keystone", + "-target=module.glance", + "-target=juju_application.ceilometer", + ] + + def test_empty_list(self): + targets = _terraform_targets_for_charms([], _TF_TARGETS) + assert targets == [] + + def test_unknown_charm_raises_keyerror(self): + with pytest.raises(KeyError): + _terraform_targets_for_charms(["unknown-k8s"], _TF_TARGETS) + + +def _make_metadata(groups=None): + if groups is None: + groups = [ + { + "name": "identity-core", + "apps": ["keystone-k8s"], + "ready_timeout_sec": 600, + "terraform_targets": {"keystone-k8s": ["module.keystone"]}, + }, + { + "name": "image", + "apps": ["glance-k8s"], + "ready_timeout_sec": 600, + "terraform_targets": {"glance-k8s": ["module.glance"]}, + }, + ] + return HopMetadata.model_validate( + { + "from": FROM, + "to": TO, + "control_plane_groups": groups, + } + ) + + +def _make_state(): + state = UpgradeState() + state.hop_history.append( + type( + "Hop", + (), + { + "from": FROM, + "to": TO, + "status": HopStatus.IN_PROGRESS, + "phase": "control_plane", + "phases": state.__class__().phases, + }, + )() + ) + return state + + +@pytest.fixture +def mock_deployment(): + deployment = MagicMock() + tfhelper = MagicMock() + manifest = MagicMock() + jhelper = MagicMock() + client = MagicMock() + client.cluster.acquire_upgrade_lock.return_value = AcquireUpgradeLockResponse( + token=1 + ) + deployment.get_tfhelper.return_value = tfhelper + deployment.get_manifest.return_value = manifest + deployment.get_juju_helper.return_value = jhelper + deployment.get_client.return_value = client + + # Set up model status with apps matching metadata charm names + status = MagicMock() + keystone_app = MagicMock() + keystone_app.charm = "keystone-k8s" + glance_app = MagicMock() + glance_app.charm = "glance-k8s" + status.apps = {"keystone": keystone_app, "glance": glance_app} + jhelper.get_model_status.return_value = status + + return deployment + + +@pytest.fixture +def coordinator(mock_deployment): + client = mock_deployment.get_client.return_value + client.cluster.get_upgrade_state.return_value = None + coord = ReleaseUpgradeCoordinator(client) + coord.acquire_lock() + coord.load_state() + # Create a hop manually + hop = coord.create_hop(FROM, TO, "123") + hop.status = HopStatus.IN_PROGRESS + hop.phase = "control_plane" + coord.persist_state() + return coord + + +class TestControlPlaneHandler: + def test_all_groups_succeed(self, mock_deployment, coordinator): + metadata = _make_metadata() + state = coordinator.state + handler = ControlPlaneHandler(mock_deployment) + + result = handler.run(coordinator, metadata, state) + + assert result.success is True + cp = state.current_hop.phases.control_plane + assert cp.groups["identity-core"].status == PhaseStatus.COMPLETED + assert cp.groups["image"].status == PhaseStatus.COMPLETED + + def test_terraform_apply_called_per_group(self, mock_deployment, coordinator): + metadata = _make_metadata() + state = coordinator.state + handler = ControlPlaneHandler(mock_deployment) + + handler.run(coordinator, metadata, state) + + tfhelper = mock_deployment.get_tfhelper.return_value + assert tfhelper.update_partial_tfvars_and_apply_tf.call_count == 2 + + def test_terraform_init_runs_before_apply_per_group( + self, mock_deployment, coordinator + ): + metadata = _make_metadata() + state = coordinator.state + handler = ControlPlaneHandler(mock_deployment) + + handler.run(coordinator, metadata, state) + + tfhelper = mock_deployment.get_tfhelper.return_value + assert tfhelper.init.call_count == 2 + names = [c[0] for c in tfhelper.method_calls] + assert names.index("init") < names.index("update_partial_tfvars_and_apply_tf") + + def test_convergence_wait_called_per_group(self, mock_deployment, coordinator): + metadata = _make_metadata() + state = coordinator.state + handler = ControlPlaneHandler(mock_deployment) + + handler.run(coordinator, metadata, state) + + jhelper = mock_deployment.get_juju_helper.return_value + assert jhelper.wait_until_desired_status.call_count == 2 + + def test_terraform_failure_fails_group(self, mock_deployment, coordinator): + tfhelper = mock_deployment.get_tfhelper.return_value + tfhelper.update_partial_tfvars_and_apply_tf.side_effect = TerraformException( + "apply failed" + ) + metadata = _make_metadata() + state = coordinator.state + handler = ControlPlaneHandler(mock_deployment) + + result = handler.run(coordinator, metadata, state) + + assert result.success is False + assert result.error_code == UpgradeErrorCode.CONTROL_PLANE_APPLY_FAILED + cp = state.current_hop.phases.control_plane + assert cp.groups["identity-core"].status == PhaseStatus.FAILED + # Second group not attempted + assert "image" not in cp.groups + + def test_convergence_timeout_fails_group(self, mock_deployment, coordinator): + jhelper = mock_deployment.get_juju_helper.return_value + jhelper.wait_until_desired_status.side_effect = TimeoutError("timed out") + metadata = _make_metadata() + state = coordinator.state + handler = ControlPlaneHandler(mock_deployment) + + result = handler.run(coordinator, metadata, state) + + assert result.success is False + assert result.error_code == UpgradeErrorCode.CONTROL_PLANE_CONVERGENCE_TIMEOUT + cp = state.current_hop.phases.control_plane + assert cp.groups["identity-core"].status == PhaseStatus.FAILED + + def test_skips_completed_groups_on_resume(self, mock_deployment, coordinator): + metadata = _make_metadata() + state = coordinator.state + + # Mark first group as completed + handler = ControlPlaneHandler(mock_deployment) + cp = state.current_hop.phases.control_plane + cp.groups["identity-core"] = Group( + status=PhaseStatus.COMPLETED, + started_at="2025-01-01T00:00:00Z", + completed_at="2025-01-01T00:01:00Z", + ) + + result = handler.run(coordinator, metadata, state) + + assert result.success is True + tfhelper = mock_deployment.get_tfhelper.return_value + # Only image group should be upgraded + assert tfhelper.update_partial_tfvars_and_apply_tf.call_count == 1 + + def test_no_metadata_returns_failure(self, mock_deployment, coordinator): + state = coordinator.state + handler = ControlPlaneHandler(mock_deployment) + + result = handler.run(coordinator, None, state) + + assert result.success is False + assert result.error_code == UpgradeErrorCode.METADATA_MISSING + + def test_persists_state_after_each_group(self, mock_deployment, coordinator): + metadata = _make_metadata() + state = coordinator.state + handler = ControlPlaneHandler(mock_deployment) + + handler.run(coordinator, metadata, state) + + client = mock_deployment.get_client.return_value + # 2 groups × (start persist + complete persist) = 4 calls minimum + # plus initial state creation + assert client.cluster.update_upgrade_state.call_count >= 4 + + def test_group_state_has_timestamps(self, mock_deployment, coordinator): + metadata = _make_metadata() + state = coordinator.state + handler = ControlPlaneHandler(mock_deployment) + + handler.run(coordinator, metadata, state) + + cp = state.current_hop.phases.control_plane + group = cp.groups["identity-core"] + assert group.started_at is not None + assert group.completed_at is not None + + def test_failed_group_has_last_error(self, mock_deployment, coordinator): + tfhelper = mock_deployment.get_tfhelper.return_value + tfhelper.update_partial_tfvars_and_apply_tf.side_effect = TerraformException( + "apply failed" + ) + metadata = _make_metadata() + state = coordinator.state + handler = ControlPlaneHandler(mock_deployment) + + handler.run(coordinator, metadata, state) + + cp = state.current_hop.phases.control_plane + group = cp.groups["identity-core"] + assert group.last_error is not None + assert ( + group.last_error.code == UpgradeErrorCode.CONTROL_PLANE_APPLY_FAILED.value + ) + + def test_apply_passes_target_args(self, mock_deployment, coordinator): + """Verify -target args are passed to terraform apply per group.""" + metadata = _make_metadata() + state = coordinator.state + handler = ControlPlaneHandler(mock_deployment) + + handler.run(coordinator, metadata, state) + + tfhelper = mock_deployment.get_tfhelper.return_value + assert tfhelper.update_partial_tfvars_and_apply_tf.call_count == 2 + # First call: identity-core group with keystone-k8s + first_call = tfhelper.update_partial_tfvars_and_apply_tf.call_args_list[0] + assert first_call.kwargs["tf_apply_extra_args"] == ["-target=module.keystone"] + # Second call: image group with glance-k8s + second_call = tfhelper.update_partial_tfvars_and_apply_tf.call_args_list[1] + assert second_call.kwargs["tf_apply_extra_args"] == ["-target=module.glance"] + + def test_run_application_passes_target_args(self, mock_deployment, coordinator): + """Verify -target args are passed when upgrading a single app.""" + metadata = _make_metadata() + handler = ControlPlaneHandler(mock_deployment) + + # Mock juju status: app "keystone" has charm "keystone-k8s" + jhelper = mock_deployment.get_juju_helper.return_value + status = MagicMock() + keystone_app = MagicMock() + keystone_app.charm = "keystone-k8s" + status.apps = {"keystone": keystone_app} + jhelper.get_model_status.return_value = status + + handler.run_application(coordinator, metadata, "keystone-k8s") + + tfhelper = mock_deployment.get_tfhelper.return_value + call = tfhelper.update_partial_tfvars_and_apply_tf.call_args + assert call.kwargs["tf_apply_extra_args"] == ["-target=module.keystone"] + # Verify juju wait uses the app name, not the charm name + wait_call = jhelper.wait_until_desired_status.call_args + assert wait_call.args[1] == ["keystone"] + + def test_plan_group_passes_target_args(self, mock_deployment): + """Verify -target args are passed to terraform plan in dry-run.""" + metadata = _make_metadata() + handler = ControlPlaneHandler(mock_deployment) + group_meta = metadata.control_plane_groups[0] + + handler.plan_group(group_meta) + + tfhelper = mock_deployment.get_tfhelper.return_value + plan_call = tfhelper.update_partial_tfvars_and_plan_tf.call_args + assert plan_call.kwargs["tf_plan_extra_args"] == ["-target=module.keystone"] + text_call = tfhelper.terraform_plan_text.call_args + assert text_call.kwargs["extra_args"] == ["-target=module.keystone"] From 5316c0e49e6a7ac48eb2d459eebca4174405d681 Mon Sep 17 00:00:00 2001 From: Hemanth Nakkina Date: Tue, 18 Aug 2026 17:38:20 +0530 Subject: [PATCH 14/20] feat(upgrades): add pre/post-upgrade action orchestration for control-plane groups Adds run_pre_actions() and run_post_actions() to dispatch juju actions declared in group metadata. Pre-actions run before terraform apply with a short propagation delay. Post-actions run after convergence (or after failure, as cleanup). Actions support leader-only and all-units scope. On pre-action failure, post-actions still run as cleanup before returning the error. On post-action failure, the group is marked failed with CONTROL_PLANE_ACTION_FAILED. Wired into ControlPlaneHandler._upgrade_group flow: pre-actions -> terraform apply -> convergence -> post-actions. 15 new tests, all CI green. --- .../sunbeam/upgrades/control_plane/actions.py | 159 ++++++++ .../sunbeam/upgrades/control_plane/groups.py | 366 ++---------------- .../sunbeam/test_control_plane_actions.py | 169 ++++++++ 3 files changed, 350 insertions(+), 344 deletions(-) create mode 100644 sunbeam-python/sunbeam/upgrades/control_plane/actions.py create mode 100644 sunbeam-python/tests/unit/sunbeam/test_control_plane_actions.py diff --git a/sunbeam-python/sunbeam/upgrades/control_plane/actions.py b/sunbeam-python/sunbeam/upgrades/control_plane/actions.py new file mode 100644 index 000000000..21b7f6d03 --- /dev/null +++ b/sunbeam-python/sunbeam/upgrades/control_plane/actions.py @@ -0,0 +1,159 @@ +# SPDX-FileCopyrightText: 2025 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +"""Pre/post-upgrade action orchestration for control-plane groups. + +Runs juju actions declared in the group's ``pre_actions`` and +``post_actions`` metadata. Pre-actions run before terraform apply; +post-actions run after convergence (or after failure, as cleanup). +""" + +from __future__ import annotations + +import logging +import time + +from sunbeam.core.juju import ( + ActionFailedException, + JujuHelper, + LeaderNotFoundException, +) +from sunbeam.core.openstack import OPENSTACK_MODEL +from sunbeam.upgrades.coordinator import PhaseResult +from sunbeam.upgrades.errors import UpgradeErrorCode +from sunbeam.upgrades.metadata import ActionScope, ActionSpec + +LOG = logging.getLogger(__name__) + +# Seconds to wait after pre-upgrade actions before proceeding with +# terraform apply. Gives the charms time to apply any configuration +# changes declared in the action. +PRE_ACTION_PROPAGATION_DELAY_SEC = 5 + + +def _run_action_on_unit(jhelper: JujuHelper, unit: str, action: str) -> dict | None: + """Run a juju action on a unit, returning the result or None on failure.""" + try: + return jhelper.run_action(unit, OPENSTACK_MODEL, action) + except (ActionFailedException, Exception) as e: + LOG.warning("action %s on %s failed: %s", action, unit, e) + return None + + +def run_actions( + jhelper: JujuHelper, + actions: list[ActionSpec], +) -> PhaseResult: + """Run a list of juju actions on the specified apps. + + For each action spec, resolves charm names to deployed app names + (e.g. ``keystone-k8s`` → ``keystone``), then runs the action on + the leader unit (or all units if scope=all-units). Returns failure + on the first action that fails. + + :param jhelper: JujuHelper for the deployment + :param actions: list of ActionSpec from metadata + :returns: PhaseResult (success or failure with error code) + """ + for spec in actions: + # Resolve charm names to deployed app names + status = jhelper.get_model_status(OPENSTACK_MODEL) + apps = [name for name, app in status.apps.items() if app.charm in spec.apps] + if not apps: + LOG.info( + "No deployed apps for charms %s — skipping action %s " + "(feature not enabled)", + spec.apps, + spec.action, + ) + continue + + for app in apps: + if spec.scope == ActionScope.LEADER: + try: + unit = jhelper.get_leader_unit(app, OPENSTACK_MODEL) + except (LeaderNotFoundException, Exception) as e: + return PhaseResult( + success=False, + error_code=UpgradeErrorCode.CONTROL_PLANE_ACTION_FAILED, + error_message=( + f"Cannot find leader for {app} to run " + f"action {spec.action}: {e}" + ), + ) + if not unit: + return PhaseResult( + success=False, + error_code=UpgradeErrorCode.CONTROL_PLANE_ACTION_FAILED, + error_message=( + f"No leader unit for {app} to run action {spec.action}" + ), + ) + result = _run_action_on_unit(jhelper, unit, spec.action) + if result is None: + return PhaseResult( + success=False, + error_code=UpgradeErrorCode.CONTROL_PLANE_ACTION_FAILED, + error_message=(f"Action {spec.action} failed on {unit}"), + ) + else: + try: + app_status = jhelper.get_application(app, OPENSTACK_MODEL) + except Exception as e: + return PhaseResult( + success=False, + error_code=UpgradeErrorCode.CONTROL_PLANE_ACTION_FAILED, + error_message=( + f"Cannot get units for {app} to run " + f"action {spec.action}: {e}" + ), + ) + for unit_name in app_status.units: + result = _run_action_on_unit(jhelper, unit_name, spec.action) + if result is None: + return PhaseResult( + success=False, + error_code=UpgradeErrorCode.CONTROL_PLANE_ACTION_FAILED, + error_message=( + f"Action {spec.action} failed on {unit_name}" + ), + ) + return PhaseResult(success=True) + + +def run_pre_actions(jhelper: JujuHelper, actions: list[ActionSpec]) -> PhaseResult: + """Run pre-upgrade actions and wait for traefik propagation. + + :param jhelper: JujuHelper for the deployment + :param actions: list of ActionSpec from group.pre_actions + :returns: PhaseResult + """ + if not actions: + return PhaseResult(success=True) + + result = run_actions(jhelper, actions) + if not result.success: + return result + + LOG.info( + "pre-upgrade actions complete, waiting %ds before proceeding", + PRE_ACTION_PROPAGATION_DELAY_SEC, + ) + time.sleep(PRE_ACTION_PROPAGATION_DELAY_SEC) + return result + + +def run_post_actions(jhelper: JujuHelper, actions: list[ActionSpec]) -> PhaseResult: + """Run post-upgrade actions (always runs, even after failure). + + :param jhelper: JujuHelper for the deployment + :param actions: list of ActionSpec from group.post_actions + :returns: PhaseResult + """ + if not actions: + return PhaseResult(success=True) + + result = run_actions(jhelper, actions) + if not result.success: + LOG.warning("post-upgrade action failed — manual intervention may be required.") + return result diff --git a/sunbeam-python/sunbeam/upgrades/control_plane/groups.py b/sunbeam-python/sunbeam/upgrades/control_plane/groups.py index ad3e06145..48171fc38 100644 --- a/sunbeam-python/sunbeam/upgrades/control_plane/groups.py +++ b/sunbeam-python/sunbeam/upgrades/control_plane/groups.py @@ -18,15 +18,9 @@ import logging import typing -import click -import yaml -from rich.console import Console - -from sunbeam.clusterd.client import Client -from sunbeam.core.common import RiskLevel, infer_risk from sunbeam.core.deployment import Deployment from sunbeam.core.juju import JujuHelper, JujuWaitException -from sunbeam.core.manifest import Manifest, embedded_manifest_path +from sunbeam.core.manifest import Manifest from sunbeam.core.openstack import OPENSTACK_MODEL from sunbeam.core.terraform import TerraformException, TerraformHelper from sunbeam.steps.openstack import CONFIG_KEY as OPENSTACK_CONFIG_KEY @@ -43,41 +37,12 @@ from sunbeam.upgrades.state import Group, LastError, PhaseStatus, UpgradeState LOG = logging.getLogger(__name__) -console = Console() def _now_iso() -> str: return datetime.datetime.now(tz=datetime.timezone.utc).isoformat() -def _load_target_charm_manifests(to_release: str) -> dict[str, dict]: - """Load per-charm manifest entries from the snap's embedded target manifest. - - Reads ``etc/manifests//.yml`` from the snap and - returns a mapping of charm name to its raw manifest dict (channel, - revision, config, resources — whatever the embedded manifest carries). - - :param to_release: target release, e.g. "2026.1" - :returns: charm name -> raw manifest dict, or empty dict if not found - """ - from snaphelpers import Snap - - try: - snap = Snap() - risk = infer_risk(snap) - if risk == RiskLevel.STABLE: - risk_str = "stable" - else: - risk_str = str(risk) - path = embedded_manifest_path(snap, to_release, risk_str) - data = yaml.safe_load(path.read_text()) - charms = data.get("core", {}).get("software", {}).get("charms", {}) - return {name: cfg for name, cfg in charms.items()} - except Exception as e: - LOG.warning("Could not load target manifest for %s: %s", to_release, e) - return {} - - def _terraform_targets_for_charms( charms: list[str], terraform_targets: dict[str, list[str]] ) -> list[str]: @@ -110,13 +75,11 @@ class ControlPlaneHandler: Implements the PhaseHandler protocol. """ - def __init__(self, deployment: Deployment, to_release: str = ""): + def __init__(self, deployment: Deployment): self.deployment = deployment - self.to_release = to_release self._tfhelper: TerraformHelper | None = None self._manifest: Manifest | None = None self._jhelper: JujuHelper | None = None - self._target_channels: dict[str, dict] | None = None @property def tfhelper(self) -> TerraformHelper: @@ -139,43 +102,6 @@ def jhelper(self) -> JujuHelper: self._jhelper = self.deployment.get_juju_helper() return self._jhelper - @property - def target_charms(self) -> dict[str, dict]: - """Per-charm manifest entries from the snap's embedded target manifest.""" - if self._target_channels is None: - self._target_channels = _load_target_charm_manifests(self.to_release) - return self._target_channels - - def _override_charm_manifests(self, charms: list[str]) -> None: - """Override manifest entries for the group's charms from the target release. - - Replaces the full CharmManifest (channel, revision, config, resources) - for the group's charms with values from the snap's embedded target - manifest. Other charms keep their current (clusterd) values. - """ - if not self.to_release: - return - from sunbeam.core.manifest import CharmManifest - - for charm in charms: - target_cfg = self.target_charms.get(charm) - if target_cfg and charm in self.manifest.core.software.charms: - old = self.manifest.core.software.charms[charm].channel - self.manifest.core.software.charms[charm] = CharmManifest( - **target_cfg - ) - LOG.info( - "overrode %s channel: %s -> %s", - charm, - old, - target_cfg.get("channel"), - ) - elif target_cfg: - LOG.warning( - "charm %s not in deployment manifest, skipping override", - charm, - ) - def run( self, coordinator: ReleaseUpgradeCoordinator, @@ -239,231 +165,6 @@ def run( return PhaseResult(success=True) - def plan_group( - self, - group_meta: typing.Any, - ) -> list[dict]: - """Run terraform plan for a single group and return change events. - - Updates tfvars for the group's charms and runs ``terraform plan`` - without applying. Does not modify clusterd state. - - Runs both JSON and text plan: JSON events are returned for - user-facing display; text output is logged at debug level for - human-readable ``+``/``-`` diff in the sunbeam log file. - - :param group_meta: ControlPlaneGroup metadata for the group - :returns: list of terraform plan JSON events - :raises TerraformException: if the plan command fails - """ - client = self.deployment.get_client() - charms = group_meta.apps - target_args = _terraform_targets_for_charms( - charms, group_meta.terraform_targets - ) - - self._override_charm_manifests(charms) - self.tfhelper.init() - events = self.tfhelper.update_partial_tfvars_and_plan_tf( - client, - self.manifest, - charms, - OPENSTACK_CONFIG_KEY, - tf_plan_extra_args=target_args, - ) - - # Run text plan for debug logging (tfvars already written above) - try: - plan_text = self.tfhelper.terraform_plan_text( - extra_args=target_args - ) - LOG.debug( - "Terraform plan (text) for group %s:\n%s", - group_meta.name, - plan_text, - ) - except TerraformException as e: - LOG.warning( - "Terraform text plan failed for group %s: %s", - group_meta.name, - e, - ) - - return events - - def run_group( - self, - coordinator: ReleaseUpgradeCoordinator, - metadata: HopMetadata | None, - group_name: str, - ) -> PhaseResult: - """Upgrade a single group by name. - - Does not skip completed groups — use run() for resume behavior. - """ - if metadata is None: - return PhaseResult( - success=False, - error_code=UpgradeErrorCode.METADATA_MISSING, - error_message="No metadata loaded", - ) - - hop = coordinator.get_current_hop() - if hop is None: - return PhaseResult( - success=False, - error_code=UpgradeErrorCode.HOP_INVALID_TRANSITION, - error_message="No active hop", - ) - - group_meta = None - for g in metadata.control_plane_groups: - if g.name == group_name: - group_meta = g - break - if group_meta is None: - return PhaseResult( - success=False, - error_code=UpgradeErrorCode.METADATA_INVALID, - error_message=f"Group {group_name} not found in metadata", - ) - - control_plane = hop.phases.control_plane - group_state = control_plane.groups.get(group_name) - if group_state is None: - group_state = Group() - control_plane.groups[group_name] = group_state - - group_state.status = PhaseStatus.IN_PROGRESS - group_state.started_at = _now_iso() - coordinator.persist_state() - - result = self._upgrade_group(group_meta, group_name) - if result.success: - group_state.status = PhaseStatus.COMPLETED - group_state.completed_at = _now_iso() - else: - group_state.status = PhaseStatus.FAILED - group_state.last_error = LastError( - code=result.error_code.value if result.error_code else "", - message=result.error_message or "", - ) - coordinator.persist_state() - return result - - def run_application( - self, - coordinator: ReleaseUpgradeCoordinator, - metadata: HopMetadata | None, - charm_name: str, - ) -> PhaseResult: - """Upgrade a single application via scoped terraform apply. - - Does not run pre/post actions (those are group-level). - - :param charm_name: charm name (e.g. 'placement-k8s'), resolved - from the juju app name by the CLI. - """ - if metadata is None: - return PhaseResult( - success=False, - error_code=UpgradeErrorCode.METADATA_MISSING, - error_message="No metadata loaded", - ) - - hop = coordinator.get_current_hop() - if hop is None: - return PhaseResult( - success=False, - error_code=UpgradeErrorCode.HOP_INVALID_TRANSITION, - error_message="No active hop", - ) - - # Verify the charm exists in some group - group_meta = None - for g in metadata.control_plane_groups: - if charm_name in g.apps: - group_meta = g - break - if group_meta is None: - return PhaseResult( - success=False, - error_code=UpgradeErrorCode.METADATA_INVALID, - error_message=f"Charm {charm_name} not found in any group", - ) - - # Mark group as in_progress (single-app upgrade doesn't complete the group) - control_plane = hop.phases.control_plane - group_state = control_plane.groups.get(group_meta.name) - if group_state is None: - group_state = Group() - control_plane.groups[group_meta.name] = group_state - if group_state.status != PhaseStatus.COMPLETED: - group_state.status = PhaseStatus.IN_PROGRESS - group_state.started_at = _now_iso() - coordinator.persist_state() - - client = self.deployment.get_client() - target_args = _terraform_targets_for_charms( - [charm_name], group_meta.terraform_targets - ) - self._override_charm_manifests([charm_name]) - - # Resolve charm name to deployed juju app name for Juju operations - status = self.jhelper.get_model_status(OPENSTACK_MODEL) - juju_app_names = [ - name for name, app in status.apps.items() if app.charm == charm_name - ] - if not juju_app_names: - return PhaseResult( - success=False, - error_code=UpgradeErrorCode.METADATA_INVALID, - error_message=f"No deployed application found for charm {charm_name}", - ) - - try: - with console.status(f" {charm_name}: applying terraform plan..."): - self.tfhelper.init() - self.tfhelper.update_partial_tfvars_and_apply_tf( - client, - self.manifest, - [charm_name], - OPENSTACK_CONFIG_KEY, - tf_apply_extra_args=target_args, - ) - except TerraformException as e: - click.echo(f" {charm_name}: terraform apply FAILED") - return PhaseResult( - success=False, - error_code=UpgradeErrorCode.CONTROL_PLANE_APPLY_FAILED, - error_message=f"Terraform init/apply failed for {charm_name}: {e}", - ) - - try: - with console.status(f" {charm_name}: waiting for convergence..."): - self.jhelper.wait_until_desired_status( - OPENSTACK_MODEL, - juju_app_names, - status=["active"], - timeout=600, - ) - except (JujuWaitException, TimeoutError) as e: - click.echo(f" {charm_name}: convergence timeout") - return PhaseResult( - success=False, - error_code=UpgradeErrorCode.CONTROL_PLANE_CONVERGENCE_TIMEOUT, - error_message=f"{charm_name} did not converge: {e}", - ) - - # If this was the only app in the group, mark group as completed - if len(group_meta.apps) == 1: - group_state.status = PhaseStatus.COMPLETED - group_state.completed_at = _now_iso() - coordinator.persist_state() - click.echo(f" {charm_name}: completed") - - return PhaseResult(success=True) - def _upgrade_group( self, group_meta: typing.Any, @@ -478,24 +179,14 @@ def _upgrade_group( group_meta.ready_timeout_sec, ) - # Resolve charm names to deployed app names, skipping undeployed charms + # Resolve charm names to deployed app names for Juju operations status = self.jhelper.get_model_status(OPENSTACK_MODEL) - deployed_charms = {app.charm for app in status.apps.values()} - charms = [c for c in charms if c in deployed_charms] - app_names = [ - name for name, app in status.apps.items() if app.charm in charms - ] - if not charms: - LOG.info("group %s has no deployed charms — skipping", group_name) - click.echo(f" {group_name}: no deployed charms — skipping") - return PhaseResult(success=True) + app_names = [name for name, app in status.apps.items() if app.charm in charms] LOG.info("resolved charms %s to apps %s", charms, app_names) # Pre-upgrade actions (use app names, not charm names) - with console.status(f" {group_name}: running pre-upgrade actions..."): - pre_result = run_pre_actions(self.jhelper, group_meta.pre_actions) + pre_result = run_pre_actions(self.jhelper, group_meta.pre_actions) if not pre_result.success: - click.echo(f" {group_name}: pre-upgrade actions FAILED") # Still run post-actions as cleanup attempt run_post_actions(self.jhelper, group_meta.post_actions) return pre_result @@ -505,43 +196,31 @@ def _upgrade_group( charms, group_meta.terraform_targets ) - self._override_charm_manifests(charms) try: - # ponytail: init re-resolves providers from the snap mirror; a snap - # refresh can bump the juju provider version, leaving the .terraform - # dir stale and apply failing with "unavailable provider". - with console.status(f" {group_name}: applying terraform plan..."): - self.tfhelper.init() - self.tfhelper.update_partial_tfvars_and_apply_tf( - client, - self.manifest, - charms, - OPENSTACK_CONFIG_KEY, - tf_apply_extra_args=target_args, - ) + self.tfhelper.update_partial_tfvars_and_apply_tf( + client, + self.manifest, + charms, + OPENSTACK_CONFIG_KEY, + tf_apply_extra_args=target_args, + ) except TerraformException as e: - click.echo(f" {group_name}: terraform apply FAILED") # Run post-actions as cleanup even on failure run_post_actions(self.jhelper, group_meta.post_actions) return PhaseResult( success=False, error_code=UpgradeErrorCode.CONTROL_PLANE_APPLY_FAILED, - error_message=f"Terraform init/apply failed for {group_name}: {e}", + error_message=f"Terraform apply failed for group {group_name}: {e}", ) try: - with console.status( - f" {group_name}: waiting for convergence " - f"({group_meta.ready_timeout_sec}s)..." - ): - self.jhelper.wait_until_desired_status( - OPENSTACK_MODEL, - app_names, - status=["active"], - timeout=group_meta.ready_timeout_sec, - ) + self.jhelper.wait_until_desired_status( + OPENSTACK_MODEL, + app_names, + status=["active"], + timeout=group_meta.ready_timeout_sec, + ) except (JujuWaitException, TimeoutError) as e: - click.echo(f" {group_name}: convergence timeout") # Run post-actions as cleanup even on failure run_post_actions(self.jhelper, group_meta.post_actions) return PhaseResult( @@ -554,10 +233,10 @@ def _upgrade_group( ) # Post-upgrade actions - with console.status(f" {group_name}: running post-upgrade actions..."): - post_result = run_post_actions(self.jhelper, group_meta.post_actions) + post_result = run_post_actions(self.jhelper, group_meta.post_actions) if not post_result.success: - click.echo(f" {group_name}: post-upgrade actions FAILED") + # Group upgraded successfully but post-action failed — + # healthcheck may not be restored. Mark blocked. return PhaseResult( success=False, error_code=UpgradeErrorCode.CONTROL_PLANE_ACTION_FAILED, @@ -569,5 +248,4 @@ def _upgrade_group( ), ) - click.echo(f" {group_name}: completed") return PhaseResult(success=True) diff --git a/sunbeam-python/tests/unit/sunbeam/test_control_plane_actions.py b/sunbeam-python/tests/unit/sunbeam/test_control_plane_actions.py new file mode 100644 index 000000000..562c83889 --- /dev/null +++ b/sunbeam-python/tests/unit/sunbeam/test_control_plane_actions.py @@ -0,0 +1,169 @@ +# SPDX-FileCopyrightText: 2025 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for control-plane pre/post-upgrade action orchestration.""" + +from unittest.mock import MagicMock, patch + +import pytest + +from sunbeam.upgrades.control_plane.actions import ( + run_actions, + run_post_actions, + run_pre_actions, +) +from sunbeam.upgrades.errors import UpgradeErrorCode +from sunbeam.upgrades.metadata import ActionScope, ActionSpec + + +def _make_app(charm="keystone-k8s"): + app = MagicMock() + app.charm = charm + return app + + +def _make_status(apps=None): + if apps is None: + apps = {"keystone": _make_app("keystone-k8s")} + status = MagicMock() + status.apps = apps + return status + + +def _make_action(action="pre-upgrade", apps=None, scope=ActionScope.LEADER): + if apps is None: + apps = ["keystone-k8s"] + return ActionSpec(action=action, apps=apps, scope=scope) + + +@pytest.fixture +def mock_jhelper(): + jhelper = MagicMock() + jhelper.get_model_status.return_value = _make_status() + jhelper.get_leader_unit.return_value = "keystone/0" + jhelper.run_action.return_value = {"result": "ok"} + return jhelper + + +class TestRunActions: + def test_leader_action_succeeds(self, mock_jhelper): + actions = [_make_action("pre-upgrade", ["keystone-k8s"])] + result = run_actions(mock_jhelper, actions) + assert result.success is True + mock_jhelper.run_action.assert_called_once_with( + "keystone/0", "openstack", "pre-upgrade" + ) + + def test_all_units_action_succeeds(self, mock_jhelper): + app_mock = MagicMock() + app_mock.charm = "keystone-k8s" + app_mock.units = {"keystone/0": MagicMock(), "keystone/1": MagicMock()} + mock_jhelper.get_model_status.return_value = _make_status( + {"keystone": app_mock} + ) + mock_jhelper.get_application.return_value = app_mock + actions = [_make_action("pre-upgrade", ["keystone-k8s"], ActionScope.ALL_UNITS)] + result = run_actions(mock_jhelper, actions) + assert result.success is True + assert mock_jhelper.run_action.call_count == 2 + + def test_action_failure_returns_error(self, mock_jhelper): + mock_jhelper.run_action.side_effect = Exception("action failed") + actions = [_make_action("pre-upgrade", ["keystone-k8s"])] + result = run_actions(mock_jhelper, actions) + assert result.success is False + assert result.error_code == UpgradeErrorCode.CONTROL_PLANE_ACTION_FAILED + + def test_no_leader_returns_error(self, mock_jhelper): + mock_jhelper.get_leader_unit.return_value = "" + actions = [_make_action("pre-upgrade", ["keystone-k8s"])] + result = run_actions(mock_jhelper, actions) + assert result.success is False + assert result.error_code == UpgradeErrorCode.CONTROL_PLANE_ACTION_FAILED + + def test_leader_not_found_returns_error(self, mock_jhelper): + mock_jhelper.get_leader_unit.side_effect = Exception("not found") + actions = [_make_action("pre-upgrade", ["keystone-k8s"])] + result = run_actions(mock_jhelper, actions) + assert result.success is False + assert result.error_code == UpgradeErrorCode.CONTROL_PLANE_ACTION_FAILED + + def test_multiple_apps_in_one_action(self, mock_jhelper): + mock_jhelper.get_leader_unit.side_effect = ["keystone/0", "nova/0"] + mock_jhelper.get_model_status.return_value = _make_status( + { + "keystone": _make_app("keystone-k8s"), + "nova": _make_app("nova-k8s"), + } + ) + actions = [_make_action("pre-upgrade", ["keystone-k8s", "nova-k8s"])] + result = run_actions(mock_jhelper, actions) + assert result.success is True + assert mock_jhelper.run_action.call_count == 2 + + def test_multiple_actions(self, mock_jhelper): + mock_jhelper.get_leader_unit.side_effect = ["keystone/0", "keystone/0"] + actions = [ + _make_action("pre-upgrade", ["keystone-k8s"]), + _make_action("custom-action", ["keystone-k8s"]), + ] + result = run_actions(mock_jhelper, actions) + assert result.success is True + assert mock_jhelper.run_action.call_count == 2 + + def test_empty_actions_list_succeeds(self, mock_jhelper): + result = run_actions(mock_jhelper, []) + assert result.success is True + mock_jhelper.run_action.assert_not_called() + + def test_first_action_failure_stops(self, mock_jhelper): + mock_jhelper.run_action.side_effect = Exception("failed") + actions = [ + _make_action("pre-upgrade", ["keystone-k8s"]), + _make_action("custom-action", ["keystone-k8s"]), + ] + result = run_actions(mock_jhelper, actions) + assert result.success is False + # Only first action attempted + assert mock_jhelper.run_action.call_count == 1 + + +class TestRunPreActions: + def test_runs_actions_and_waits(self, mock_jhelper): + actions = [_make_action("pre-upgrade", ["keystone-k8s"])] + with patch("sunbeam.upgrades.control_plane.actions.time.sleep") as mock_sleep: + result = run_pre_actions(mock_jhelper, actions) + assert result.success is True + mock_sleep.assert_called_once() + + def test_empty_actions_skips_wait(self, mock_jhelper): + with patch("sunbeam.upgrades.control_plane.actions.time.sleep") as mock_sleep: + result = run_pre_actions(mock_jhelper, []) + assert result.success is True + mock_sleep.assert_not_called() + + def test_action_failure_skips_wait(self, mock_jhelper): + mock_jhelper.run_action.side_effect = Exception("failed") + actions = [_make_action("pre-upgrade", ["keystone-k8s"])] + with patch("sunbeam.upgrades.control_plane.actions.time.sleep") as mock_sleep: + result = run_pre_actions(mock_jhelper, actions) + assert result.success is False + mock_sleep.assert_not_called() + + +class TestRunPostActions: + def test_runs_actions(self, mock_jhelper): + actions = [_make_action("post-upgrade", ["keystone-k8s"])] + result = run_post_actions(mock_jhelper, actions) + assert result.success is True + + def test_empty_actions_succeeds(self, mock_jhelper): + result = run_post_actions(mock_jhelper, []) + assert result.success is True + + def test_failure_returns_result_but_logs_warning(self, mock_jhelper): + mock_jhelper.run_action.side_effect = Exception("failed") + actions = [_make_action("post-upgrade", ["keystone-k8s"])] + result = run_post_actions(mock_jhelper, actions) + assert result.success is False + assert result.error_code == UpgradeErrorCode.CONTROL_PLANE_ACTION_FAILED From dc14fd4eeaf39581397b8e4ea85383023e2770e8 Mon Sep 17 00:00:00 2001 From: Hemanth Nakkina Date: Tue, 18 Aug 2026 18:35:48 +0530 Subject: [PATCH 15/20] feat(upgrades): add control-plane CLI with terraform plan dry-run MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds 'sunbeam cluster upgrade control-plane' subcommand with six flags: --auto, --group, --application, --status, --retry-group, --dry-run. --auto upgrades all remaining groups in order. --group and --application upgrade a single group or app. --application is rejected on failed groups (must use --retry-group). --status shows per-group state. --retry-group resets a failed/blocked group and re-runs it. --dry-run runs terraform plan per group and shows the actual changes that would be applied — not just a metadata listing. Adds terraform_plan() and update_partial_tfvars_and_plan_tf() to TerraformHelper for this purpose. Also adds run_group() and run_application() methods to ControlPlaneHandler for single-group and single-app execution. plan_group() runs terraform plan for a single group. 16 tests, all CI green. --- manifests/2026.1/upgrade.yml | 4 +- sunbeam-python/sunbeam/commands/upgrade.py | 306 +++++++++++++++ sunbeam-python/sunbeam/core/terraform.py | 138 +++++++ .../sunbeam/upgrades/control_plane/groups.py | 181 +++++++++ .../tests/unit/sunbeam/core/test_terraform.py | 60 +++ .../unit/sunbeam/test_control_plane_cli.py | 371 ++++++++++++++++++ .../unit/sunbeam/test_control_plane_groups.py | 35 -- 7 files changed, 1059 insertions(+), 36 deletions(-) create mode 100644 sunbeam-python/tests/unit/sunbeam/test_control_plane_cli.py diff --git a/manifests/2026.1/upgrade.yml b/manifests/2026.1/upgrade.yml index d4cf4ef2f..0e8816053 100644 --- a/manifests/2026.1/upgrade.yml +++ b/manifests/2026.1/upgrade.yml @@ -97,7 +97,9 @@ control_plane_groups: apps: [nova-k8s] scope: leader terraform_targets: - nova-k8s: [module.nova] + nova-k8s: + - module.nova + - juju_integration.nova-to-ingress-metadata post_actions: - action: post-upgrade apps: [nova-k8s] diff --git a/sunbeam-python/sunbeam/commands/upgrade.py b/sunbeam-python/sunbeam/commands/upgrade.py index f33337e97..7cdfdb8f7 100644 --- a/sunbeam-python/sunbeam/commands/upgrade.py +++ b/sunbeam-python/sunbeam/commands/upgrade.py @@ -10,6 +10,7 @@ - preflight: run pre-flight checks and create the active hop. - abandon: mark the active hop as abandoned, release the lock, and print recovery guidance. +- control-plane: upgrade control-plane charm groups. """ from __future__ import annotations @@ -20,7 +21,9 @@ from sunbeam.clusterd.service import UpgradeLockHeldException from sunbeam.core.deployment import Deployment from sunbeam.core.openstack import OPENSTACK_MODEL +from sunbeam.upgrades.control_plane.groups import ControlPlaneHandler from sunbeam.upgrades.coordinator import ReleaseUpgradeCoordinator +from sunbeam.upgrades.metadata import load_upgrade_metadata from sunbeam.upgrades.observability import UpgradeLogger from sunbeam.upgrades.preflight.checks import ( CheckContext, @@ -28,6 +31,7 @@ run_upgrade_preflight_checks, ) from sunbeam.upgrades.preflight.hop import create_hop_after_preflight +from sunbeam.upgrades.state import PhaseStatus from sunbeam.versions import detect_deployed_release, detect_snap_release console = Console() @@ -191,3 +195,305 @@ def abandon(ctx: click.Context, yes: bool) -> None: " 2. Once recovered, start a new upgrade with\n" " 'sunbeam cluster upgrade preflight'.\n" ) + + +def _print_control_plane_status(hop, metadata, control_plane_state) -> None: + """Print per-group control-plane upgrade status.""" + click.echo(f"\nActive hop: {hop.from_release} -> {hop.to_release}") + click.echo("Control-plane groups:") + for g in metadata.control_plane_groups: + gs = control_plane_state.groups.get(g.name) + if gs is None: + status_str = "pending" + elif gs.status == PhaseStatus.COMPLETED: + status_str = f"completed ({gs.completed_at or ''})" + elif gs.status == PhaseStatus.FAILED: + status_str = f"FAILED ({gs.last_error.message if gs.last_error else ''})" + elif gs.status == PhaseStatus.IN_PROGRESS: + status_str = "in progress" + else: + status_str = gs.status.value + mark = "✓" if gs and gs.status == PhaseStatus.COMPLETED else " " + click.echo(f" {g.name:<25} {mark} {status_str}") + + +def _print_dry_run( + hop, metadata, control_plane_state, handler, group_name=None +) -> None: + """Print what would execute, including terraform plan output.""" + click.echo(f"\nActive hop: {hop.from_release} -> {hop.to_release}") + click.echo("DRY RUN — no changes will be made.\n") + click.echo("Control-plane groups (will execute in order):") + groups = metadata.control_plane_groups + if group_name: + groups = [g for g in groups if g.name == group_name] + if not groups: + click.echo(f" Group {group_name} not found in metadata.") + return + total = len(groups) + for i, g in enumerate(groups, 1): + gs = control_plane_state.groups.get(g.name) + if gs and gs.status == PhaseStatus.COMPLETED: + click.echo(f" [{i}/{total}] {g.name} (completed — will skip)") + continue + + click.echo(f" [{i}/{total}] {g.name}") + for app in g.apps: + click.echo(f" └── {app}") + for action in g.pre_actions: + click.echo(f" └── Step 1 - Pre: {action.action}") + click.echo(" └── Step 2 - Terraform apply") + # Run terraform plan for this group and show changes under apply + try: + events = handler.plan_group(g) + changes = [ + e + for e in events + if e.get("@level") == "warning" + or e.get("type") == "change" + or "change" in e + ] + if changes: + click.echo(" └── Plan changes:") + for change in changes: + msg = change.get("@message", "") + if msg: + click.echo(f" {msg}") + else: + click.echo(" └── Plan: no changes") + except Exception as e: + click.echo(f" └── Plan failed: {e}") + click.echo(f" └── Step 3 - Convergence wait ({g.ready_timeout_sec}s)") + for action in g.post_actions: + click.echo(f" └── Step 4 - Post: {action.action}") + + +def _validate_flags( + auto: bool, + group_name: str | None, + app_name: str | None, + retry_group: str | None, + show_status: bool, + dry_run: bool, +) -> None: + """Validate flag combinations.""" + exclusive = [bool(group_name), bool(app_name), auto] + if sum(exclusive) > 1: + raise click.ClickException( + "--group, --application, and --auto are mutually exclusive." + ) + if retry_group and (group_name or app_name or auto): + raise click.ClickException( + "--retry-group cannot be combined with --group, --application, or --auto." + ) + if not any(exclusive) and not retry_group and not show_status and not dry_run: + raise click.ClickException( + "One of --auto, --group, --application, --retry-group, " + "--status, or --dry-run is required." + ) + + +def _resolve_charm_name(deployment: Deployment, app_name: str, metadata) -> str: + """Resolve a juju app name to its charm name. + + Looks up the app in juju status and matches its charm against the + control-plane groups in metadata. + + :raises click.ClickException: if the app is not found or its charm + is not in any upgrade group. + """ + jhelper = deployment.get_juju_helper() + status = jhelper.get_model_status(OPENSTACK_MODEL) + app = status.apps.get(app_name) + if app is None: + raise click.ClickException( + f"Application {app_name!r} not found in model {OPENSTACK_MODEL}. " + "Use 'juju status' to list applications." + ) + charm = app.charm + for g in metadata.control_plane_groups: + if charm in g.apps: + return charm + raise click.ClickException( + f"Application {app_name!r} (charm {charm!r}) is not part of any " + "control-plane upgrade group." + ) + + +def _check_application_group(app_name: str, metadata, control_plane_state) -> None: + """Reject --application on failed groups.""" + for g in metadata.control_plane_groups: + if app_name in g.apps: + gs = control_plane_state.groups.get(g.name) + if gs and gs.status == PhaseStatus.FAILED: + raise click.ClickException( + f"Application {app_name} belongs to group {g.name} " + "which is in failed state. Use --retry-group to retry " + "the group." + ) + break + + +def _execute_control_plane( + coordinator: ReleaseUpgradeCoordinator, + handler: ControlPlaneHandler, + metadata, + auto: bool, + group_name: str | None, + app_name: str | None, + retry_group: str | None, + control_plane_state, +) -> None: + """Acquire lock and execute the control-plane upgrade.""" + try: + coordinator.acquire_lock() + coordinator.load_state() + + if retry_group: + gs = control_plane_state.groups.get(retry_group) + if gs is None: + raise click.ClickException(f"Group {retry_group} not found in state.") + if gs.status not in (PhaseStatus.FAILED, PhaseStatus.BLOCKED): + raise click.ClickException( + f"Group {retry_group} is not failed or blocked " + f"(status: {gs.status.value}). Use --group instead." + ) + gs.status = PhaseStatus.PENDING + coordinator.persist_state() + result = handler.run_group(coordinator, metadata, retry_group) + elif group_name: + result = handler.run_group(coordinator, metadata, group_name) + elif app_name: + result = handler.run_application(coordinator, metadata, app_name) + elif auto: + state = coordinator.state + if state is None: + raise click.ClickException("No state loaded.") + result = handler.run(coordinator, metadata, state) + else: + return + + if not result.success: + raise click.ClickException( + result.error_message or "Control-plane upgrade failed." + ) + + except UpgradeLockHeldException: + raise click.ClickException( + "Cannot acquire the upgrade lock — it is held by another process." + ) + finally: + coordinator.release_lock() + + click.echo("Control-plane upgrade completed.") + + +@upgrade.command("control-plane") +@click.option( + "--auto", + is_flag=True, + default=False, + help="Upgrade all remaining groups in order.", +) +@click.option( + "--group", + "group_name", + default=None, + help="Upgrade a single group by name.", +) +@click.option( + "--application", + "app_name", + default=None, + help="Upgrade a single application by name (as shown in 'juju status').", +) +@click.option( + "--status", + "show_status", + is_flag=True, + default=False, + help="Show per-group upgrade status.", +) +@click.option( + "--retry-group", + "retry_group", + default=None, + help="Retry a failed or blocked group.", +) +@click.option( + "--dry-run", + is_flag=True, + default=False, + help="Show what would execute without making changes.", +) +@click.pass_context +def control_plane( + ctx: click.Context, + auto: bool, + group_name: str | None, + app_name: str | None, + show_status: bool, + retry_group: str | None, + dry_run: bool, +) -> None: + r"""Upgrade control-plane charm groups. + + Upgrades K8s control-plane charms in dependency groups defined by + release metadata. Groups are upgraded in order with pre/post-upgrade + actions per group. + + \b + Flags: + --auto Upgrade all remaining groups in order + --group Upgrade a single group + --application Upgrade a single application + --status Show per-group status + --retry-group Retry a failed/blocked group + --dry-run Show plan without executing + + --application and --group are mutually exclusive. --auto overrides + both. When using --application, dependency ordering is the + operator's responsibility — upgrading an app whose predecessor + groups are not complete may leave the cluster in an inconsistent + state. Use --status to check group completion before proceeding. + """ + deployment: Deployment = ctx.obj + client = deployment.get_client() + + coordinator = ReleaseUpgradeCoordinator(client, UpgradeLogger()) + coordinator.load_state() + hop = coordinator.get_current_hop() + if hop is None: + raise click.ClickException("No active upgrade hop. Run preflight first.") + + metadata = load_upgrade_metadata(hop.to_release) + handler = ControlPlaneHandler(deployment, to_release=hop.to_release) + control_plane_state = hop.phases.control_plane + + if show_status: + _print_control_plane_status(hop, metadata, control_plane_state) + return + + if dry_run: + _print_dry_run( + hop, metadata, control_plane_state, handler, group_name=group_name + ) + return + + _validate_flags(auto, group_name, app_name, retry_group, show_status, dry_run) + + charm_name = None + if app_name: + charm_name = _resolve_charm_name(deployment, app_name, metadata) + _check_application_group(charm_name, metadata, control_plane_state) + + _execute_control_plane( + coordinator, + handler, + metadata, + auto, + group_name, + charm_name, + retry_group, + control_plane_state, + ) diff --git a/sunbeam-python/sunbeam/core/terraform.py b/sunbeam-python/sunbeam/core/terraform.py index 23a077780..b83a8c6f6 100644 --- a/sunbeam-python/sunbeam/core/terraform.py +++ b/sunbeam-python/sunbeam/core/terraform.py @@ -223,6 +223,117 @@ def apply( cmd.append(f"-parallelism={self.parallelism}") self._run_terraform_command(cmd, os_env, reporter=reporter) + def terraform_plan(self, extra_args: list | None = None) -> list[dict]: + """Run terraform plan and return the change events. + + Executes ``terraform plan`` in JSON mode and returns the + parsed change events. Each event is a dict with ``@level``, + ``@message``, and optionally ``change`` fields. + + :param extra_args: Extra args (e.g. ``-target=module.keystone``) + to scope the plan to specific resources. + :returns: list of terraform plan JSON events + :raises TerraformException: if the plan command fails + """ + os_env = os.environ.copy() + timestamp = datetime.now().strftime("%Y%m%d%H%M%S") + tf_log = str(self.path / f"terraform-plan-{timestamp}.log") + os_env.update({"TF_LOG_PATH": tf_log}) + os_env.setdefault("TF_LOG", "INFO") + if self.env: + os_env.update(self.env) + + cmd = [ + self.terraform, + "plan", + "-input=false", + "-no-color", + "-json", + ] + if extra_args: + cmd.extend(extra_args) + if self.parallelism is not None: + cmd.append(f"-parallelism={self.parallelism}") + + LOG.debug("Running command %s with cwd: %s", " ".join(cmd), self.path) + + process = subprocess.Popen( + cmd, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True, + cwd=self.path, + env=os_env, + ) + + events: list[dict] = [] + if process.stdout is not None: + for line in process.stdout: + line = line.rstrip("\n") + if not line: + continue + try: + event = json.loads(line) + events.append(event) + except json.JSONDecodeError: + LOG.debug("Non-JSON plan line: %s", line) + + process.wait() + if process.returncode != 0: + stderr_output = "" + if process.stderr is not None: + stderr_output = process.stderr.read() + raise TerraformException( + f"terraform plan failed: {' '.join(cmd)}\nstderr: {stderr_output}" + ) + + return events + + def terraform_plan_text(self, extra_args: list | None = None) -> str: + """Run terraform plan in text mode and return human-readable output. + + Unlike ``terraform_plan`` which returns JSON events, this returns + the standard ``terraform plan`` text output with ``+``/``-``/``~`` + field-level diffs. Used for debug logging during dry-run. + + :param extra_args: Extra args (e.g. ``-target=module.keystone``) + to scope the plan to specific resources. + :returns: terraform plan text output + :raises TerraformException: if the plan command fails + """ + os_env = os.environ.copy() + timestamp = datetime.now().strftime("%Y%m%d%H%M%S") + tf_log = str(self.path / f"terraform-plan-{timestamp}.log") + os_env.update({"TF_LOG_PATH": tf_log}) + os_env.setdefault("TF_LOG", "INFO") + if self.env: + os_env.update(self.env) + + cmd = [self.terraform, "plan", "-input=false", "-no-color"] + if extra_args: + cmd.extend(extra_args) + if self.parallelism is not None: + cmd.append(f"-parallelism={self.parallelism}") + + LOG.debug("Running command %s with cwd: %s", " ".join(cmd), self.path) + + try: + process = subprocess.run( + cmd, + capture_output=True, + text=True, + check=True, + cwd=self.path, + env=os_env, + ) + LOG.debug("Plan text output:\n%s", process.stdout) + return process.stdout + except subprocess.CalledProcessError as e: + LOG.exception("Terraform plan (text) failed: %s", e.stderr) + raise TerraformException( + f"terraform plan failed: {' '.join(cmd)}\nstderr: {e.stderr}" + ) + def destroy(self, reporter: ProgressReporter | None = None): """Terraform destroy.""" os_env = os.environ.copy() @@ -411,6 +522,33 @@ def update_partial_tfvars_and_apply_tf( LOG.debug("Applying plan %s with tfvars %s", self.plan, updated_tfvars) self.apply(tf_apply_extra_args, reporter=reporter) + def update_partial_tfvars_and_plan_tf( + self, + client: Client, + manifest: Manifest, + charms: list[str], + tfvar_config: str | None = None, + tf_plan_extra_args: list | None = None, + ) -> list[dict]: + """Update tfvars for specific charms and run terraform plan. + + Same as ``update_partial_tfvars_and_apply_tf`` but runs + ``terraform plan`` instead of ``terraform apply``. Does not + save tfvars to the database — dry-run only. + + :param tf_plan_extra_args: Extra args (e.g. ``-target=...``) + to scope the plan to specific resources. + :returns: list of terraform plan JSON events + """ + computed_keys, updated_tfvars = self._load_and_filter_db_tfvars_for_charms( + client, tfvar_config, charms + ) + tfvars_from_manifest = self._get_tfvars(manifest, charms) + self._apply_tfvars(updated_tfvars, tfvars_from_manifest) + self.write_tfvars(updated_tfvars) + LOG.debug("Planning %s with tfvars %s", self.plan, updated_tfvars) + return self.terraform_plan(extra_args=tf_plan_extra_args) + def update_tfvars_and_apply_tf( self, client: Client, diff --git a/sunbeam-python/sunbeam/upgrades/control_plane/groups.py b/sunbeam-python/sunbeam/upgrades/control_plane/groups.py index 48171fc38..cef4b9472 100644 --- a/sunbeam-python/sunbeam/upgrades/control_plane/groups.py +++ b/sunbeam-python/sunbeam/upgrades/control_plane/groups.py @@ -165,6 +165,187 @@ def run( return PhaseResult(success=True) + def plan_group( + self, + group_meta: typing.Any, + ) -> list[dict]: + """Run terraform plan for a single group and return change events. + + Updates tfvars for the group's charms and runs ``terraform plan`` + without applying. Does not modify clusterd state. + + Runs both JSON and text plan: JSON events are returned for + user-facing display; text output is logged at debug level for + human-readable ``+``/``-`` diff in the sunbeam log file. + + :param group_meta: ControlPlaneGroup metadata for the group + :returns: list of terraform plan JSON events + :raises TerraformException: if the plan command fails + """ + client = self.deployment.get_client() + charms = group_meta.apps + target_args = _terraform_targets_for_charms( + charms, group_meta.terraform_targets + ) + + events = self.tfhelper.update_partial_tfvars_and_plan_tf( + client, + self.manifest, + charms, + OPENSTACK_CONFIG_KEY, + tf_plan_extra_args=target_args, + ) + + # Run text plan for debug logging (tfvars already written above) + try: + plan_text = self.tfhelper.terraform_plan_text(extra_args=target_args) + LOG.debug( + "Terraform plan (text) for group %s:\n%s", + group_meta.name, + plan_text, + ) + except TerraformException as e: + LOG.warning( + "Terraform text plan failed for group %s: %s", + group_meta.name, + e, + ) + + return events + + def run_group( + self, + coordinator: ReleaseUpgradeCoordinator, + metadata: HopMetadata | None, + group_name: str, + ) -> PhaseResult: + """Upgrade a single group by name. + + Does not skip completed groups — use run() for resume behavior. + """ + if metadata is None: + return PhaseResult( + success=False, + error_code=UpgradeErrorCode.METADATA_MISSING, + error_message="No metadata loaded", + ) + + hop = coordinator.get_current_hop() + if hop is None: + return PhaseResult( + success=False, + error_code=UpgradeErrorCode.HOP_INVALID_TRANSITION, + error_message="No active hop", + ) + + group_meta = None + for g in metadata.control_plane_groups: + if g.name == group_name: + group_meta = g + break + if group_meta is None: + return PhaseResult( + success=False, + error_code=UpgradeErrorCode.METADATA_INVALID, + error_message=f"Group {group_name} not found in metadata", + ) + + control_plane = hop.phases.control_plane + group_state = control_plane.groups.get(group_name) + if group_state is None: + group_state = Group() + control_plane.groups[group_name] = group_state + + group_state.status = PhaseStatus.IN_PROGRESS + group_state.started_at = _now_iso() + coordinator.persist_state() + + result = self._upgrade_group(group_meta, group_name) + if result.success: + group_state.status = PhaseStatus.COMPLETED + group_state.completed_at = _now_iso() + else: + group_state.status = PhaseStatus.FAILED + group_state.last_error = LastError( + code=result.error_code.value if result.error_code else "", + message=result.error_message or "", + ) + coordinator.persist_state() + return result + + def run_application( + self, + coordinator: ReleaseUpgradeCoordinator, + metadata: HopMetadata | None, + app_name: str, + ) -> PhaseResult: + """Upgrade a single application via scoped terraform apply. + + Does not run pre/post actions (those are group-level). + """ + if metadata is None: + return PhaseResult( + success=False, + error_code=UpgradeErrorCode.METADATA_MISSING, + error_message="No metadata loaded", + ) + + hop = coordinator.get_current_hop() + if hop is None: + return PhaseResult( + success=False, + error_code=UpgradeErrorCode.HOP_INVALID_TRANSITION, + error_message="No active hop", + ) + + # Verify the app exists in some group + group_meta = None + for g in metadata.control_plane_groups: + if app_name in g.apps: + group_meta = g + break + if group_meta is None: + return PhaseResult( + success=False, + error_code=UpgradeErrorCode.METADATA_INVALID, + error_message=f"Application {app_name} not found in any group", + ) + + client = self.deployment.get_client() + target_args = _terraform_targets_for_charms( + [app_name], group_meta.terraform_targets + ) + try: + self.tfhelper.update_partial_tfvars_and_apply_tf( + client, + self.manifest, + [app_name], + OPENSTACK_CONFIG_KEY, + tf_apply_extra_args=target_args, + ) + except TerraformException as e: + return PhaseResult( + success=False, + error_code=UpgradeErrorCode.CONTROL_PLANE_APPLY_FAILED, + error_message=f"Terraform apply failed for {app_name}: {e}", + ) + + try: + self.jhelper.wait_until_desired_status( + OPENSTACK_MODEL, + [app_name], + status=["active"], + timeout=600, + ) + except (JujuWaitException, TimeoutError) as e: + return PhaseResult( + success=False, + error_code=UpgradeErrorCode.CONTROL_PLANE_CONVERGENCE_TIMEOUT, + error_message=f"{app_name} did not converge: {e}", + ) + + return PhaseResult(success=True) + def _upgrade_group( self, group_meta: typing.Any, diff --git a/sunbeam-python/tests/unit/sunbeam/core/test_terraform.py b/sunbeam-python/tests/unit/sunbeam/core/test_terraform.py index 9421028b7..bfd08b55a 100644 --- a/sunbeam-python/tests/unit/sunbeam/core/test_terraform.py +++ b/sunbeam-python/tests/unit/sunbeam/core/test_terraform.py @@ -1265,3 +1265,63 @@ def test_none_reporter_still_works(self, mocker, snap, tmp_path): env={}, reporter=None, ) + + +class TestTerraformPlanExtraArgs: + """Tests for extra_args support in terraform_plan and terraform_plan_text.""" + + def _make_helper(self, mocker, snap, tmp_path): + mocker.patch.object(terraform_mod, "Snap", return_value=snap) + return TerraformHelper( + path=tmp_path, + plan="test-plan", + tfvar_map={}, + ) + + def test_plan_includes_extra_args(self, mocker, snap, tmp_path): + helper = self._make_helper(mocker, snap, tmp_path) + mock_process = MagicMock() + mock_process.stdout = iter([]) + mock_process.stderr.read.return_value = "" + mock_process.wait.return_value = 0 + mock_process.returncode = 0 + + with patch("subprocess.Popen", return_value=mock_process) as mock_popen: + helper.terraform_plan(extra_args=["-target=module.keystone"]) + cmd = mock_popen.call_args.args[0] + assert "-target=module.keystone" in cmd + + def test_plan_without_extra_args(self, mocker, snap, tmp_path): + helper = self._make_helper(mocker, snap, tmp_path) + mock_process = MagicMock() + mock_process.stdout = iter([]) + mock_process.stderr.read.return_value = "" + mock_process.wait.return_value = 0 + mock_process.returncode = 0 + + with patch("subprocess.Popen", return_value=mock_process) as mock_popen: + helper.terraform_plan() + cmd = mock_popen.call_args.args[0] + assert not any(arg.startswith("-target") for arg in cmd) + + def test_plan_text_includes_extra_args(self, mocker, snap, tmp_path): + helper = self._make_helper(mocker, snap, tmp_path) + mock_result = MagicMock() + mock_result.stdout = "Plan: 0 to add" + mock_result.stderr = "" + + with patch("subprocess.run", return_value=mock_result) as mock_run: + helper.terraform_plan_text(extra_args=["-target=module.keystone"]) + cmd = mock_run.call_args.args[0] + assert "-target=module.keystone" in cmd + + def test_plan_text_without_extra_args(self, mocker, snap, tmp_path): + helper = self._make_helper(mocker, snap, tmp_path) + mock_result = MagicMock() + mock_result.stdout = "Plan: 0 to add" + mock_result.stderr = "" + + with patch("subprocess.run", return_value=mock_result) as mock_run: + helper.terraform_plan_text() + cmd = mock_run.call_args.args[0] + assert not any(arg.startswith("-target") for arg in cmd) diff --git a/sunbeam-python/tests/unit/sunbeam/test_control_plane_cli.py b/sunbeam-python/tests/unit/sunbeam/test_control_plane_cli.py new file mode 100644 index 000000000..2f70d2471 --- /dev/null +++ b/sunbeam-python/tests/unit/sunbeam/test_control_plane_cli.py @@ -0,0 +1,371 @@ +# SPDX-FileCopyrightText: 2025 - Canonical Ltd +# SPDX-License-Identifier: Apache-2.0 + +"""Tests for the control-plane CLI command.""" + +import json +from unittest.mock import MagicMock, patch + +import pytest +from click.testing import CliRunner + +from sunbeam.clusterd.models import AcquireUpgradeLockResponse +from sunbeam.clusterd.service import UpgradeLockHeldException +from sunbeam.commands.upgrade import upgrade +from sunbeam.upgrades.metadata import HopMetadata + +FROM = "2025.1" +TO = "2026.1" + + +def _metadata(): + return HopMetadata.model_validate( + { + "from": FROM, + "to": TO, + "control_plane_groups": [ + { + "name": "identity-core", + "apps": ["keystone-k8s"], + "ready_timeout_sec": 600, + "terraform_targets": {"keystone-k8s": ["module.keystone"]}, + "pre_actions": [ + { + "action": "pre-upgrade", + "apps": ["keystone-k8s"], + "scope": "leader", + } + ], + "post_actions": [ + { + "action": "post-upgrade", + "apps": ["keystone-k8s"], + "scope": "leader", + } + ], + }, + { + "name": "image", + "apps": ["glance-k8s"], + "ready_timeout_sec": 600, + "terraform_targets": {"glance-k8s": ["module.glance"]}, + }, + ], + } + ) + + +def _patch_meta(): + return patch( + "sunbeam.commands.upgrade.load_upgrade_metadata", + return_value=_metadata(), + ) + + +def _state_json(groups=None): + if groups is None: + groups = {} + return json.dumps( + { + "active_hop": {"hop_history_index": 0}, + "hop_history": [ + { + "from": FROM, + "to": TO, + "metadata_version": 1, + "metadata_build_id": "123", + "status": "in_progress", + "phase": "control_plane", + "phases": { + "preflight": {"status": "completed"}, + "control_plane": { + "status": "in_progress", + "groups": groups, + }, + "dataplane": {"status": "pending", "nodes": {}}, + "storage": {"status": "pending", "nodes": {}}, + "finalize": {"status": "pending"}, + }, + } + ], + } + ) + + +@pytest.fixture +def mock_deployment(): + deployment = MagicMock() + client = MagicMock() + client.cluster.acquire_upgrade_lock.return_value = AcquireUpgradeLockResponse( + token=1 + ) + client.cluster.get_upgrade_state.return_value = _state_json() + client.cluster.update_upgrade_state.return_value = None + client.cluster.release_upgrade_lock.return_value = None + deployment.get_client.return_value = client + + tfhelper = MagicMock() + manifest = MagicMock() + jhelper = MagicMock() + + # Set up model status with apps matching metadata charm names + status = MagicMock() + keystone_app = MagicMock() + keystone_app.charm = "keystone-k8s" + glance_app = MagicMock() + glance_app.charm = "glance-k8s" + status.apps = {"keystone": keystone_app, "glance": glance_app} + jhelper.get_model_status.return_value = status + + jhelper.get_leader_unit.return_value = "keystone/0" + jhelper.run_action.return_value = {"result": "ok"} + deployment.get_tfhelper.return_value = tfhelper + deployment.get_manifest.return_value = manifest + deployment.get_juju_helper.return_value = jhelper + return deployment + + +class TestControlPlaneStatus: + def test_status_shows_groups(self, mock_deployment): + runner = CliRunner() + with _patch_meta(): + result = runner.invoke( + upgrade, ["control-plane", "--status"], obj=mock_deployment + ) + assert result.exit_code == 0 + assert "identity-core" in result.output + assert "image" in result.output + + def test_status_shows_completed(self, mock_deployment): + client = mock_deployment.get_client.return_value + client.cluster.get_upgrade_state.return_value = _state_json( + groups={ + "identity-core": { + "status": "completed", + "completed_at": "2025-01-01T00:00:00Z", + } + } + ) + runner = CliRunner() + with _patch_meta(): + result = runner.invoke( + upgrade, ["control-plane", "--status"], obj=mock_deployment + ) + assert result.exit_code == 0 + assert "completed" in result.output + + def test_status_no_active_hop(self, mock_deployment): + client = mock_deployment.get_client.return_value + client.cluster.get_upgrade_state.return_value = None + runner = CliRunner() + result = runner.invoke( + upgrade, ["control-plane", "--status"], obj=mock_deployment + ) + assert result.exit_code != 0 + assert "no active" in result.output.lower() + + +class TestControlPlaneDryRun: + def test_dry_run_shows_plan(self, mock_deployment): + tfhelper = mock_deployment.get_tfhelper.return_value + tfhelper.update_partial_tfvars_and_plan_tf.return_value = [ + {"@level": "warning", "@message": "keystone-k8s: channel will change"}, + ] + tfhelper.terraform_plan_text.return_value = ( + "# keystone will be updated in-place\n" + ' ~ channel = "2025.1/edge" -> "2026.1/edge/upgrade"' + ) + runner = CliRunner() + with _patch_meta(): + result = runner.invoke( + upgrade, ["control-plane", "--dry-run"], obj=mock_deployment + ) + assert result.exit_code == 0 + assert "DRY RUN" in result.output + assert "identity-core" in result.output + assert "keystone-k8s" in result.output + assert "Plan changes:" in result.output + + def test_dry_run_with_group_filter(self, mock_deployment): + tfhelper = mock_deployment.get_tfhelper.return_value + tfhelper.update_partial_tfvars_and_plan_tf.return_value = [] + tfhelper.terraform_plan_text.return_value = "" + runner = CliRunner() + with _patch_meta(): + result = runner.invoke( + upgrade, + ["control-plane", "--dry-run", "--group", "identity-core"], + obj=mock_deployment, + ) + assert result.exit_code == 0 + assert "identity-core" in result.output + assert "image" not in result.output + + def test_dry_run_shows_actions(self, mock_deployment): + tfhelper = mock_deployment.get_tfhelper.return_value + tfhelper.update_partial_tfvars_and_plan_tf.return_value = [] + tfhelper.terraform_plan_text.return_value = "" + runner = CliRunner() + with _patch_meta(): + result = runner.invoke( + upgrade, ["control-plane", "--dry-run"], obj=mock_deployment + ) + assert result.exit_code == 0 + assert "Pre: pre-upgrade" in result.output + assert "Post: post-upgrade" in result.output + + +class TestControlPlaneAuto: + def test_auto_upgrades_all(self, mock_deployment): + runner = CliRunner() + with _patch_meta(): + result = runner.invoke( + upgrade, ["control-plane", "--auto"], obj=mock_deployment + ) + assert result.exit_code == 0 + assert "completed" in result.output.lower() + + def test_auto_lock_held(self, mock_deployment): + client = mock_deployment.get_client.return_value + client.cluster.acquire_upgrade_lock.side_effect = UpgradeLockHeldException( + "held" + ) + runner = CliRunner() + with _patch_meta(): + result = runner.invoke( + upgrade, ["control-plane", "--auto"], obj=mock_deployment + ) + assert result.exit_code != 0 + assert "lock" in result.output.lower() + + +class TestControlPlaneGroup: + def test_group_upgrades_single(self, mock_deployment): + runner = CliRunner() + with _patch_meta(): + result = runner.invoke( + upgrade, + ["control-plane", "--group", "identity-core"], + obj=mock_deployment, + ) + assert result.exit_code == 0 + + def test_group_not_found(self, mock_deployment): + runner = CliRunner() + with _patch_meta(): + result = runner.invoke( + upgrade, + ["control-plane", "--group", "nonexistent"], + obj=mock_deployment, + ) + assert result.exit_code != 0 + + +class TestControlPlaneApplication: + def test_application_upgrades_single(self, mock_deployment): + runner = CliRunner() + with _patch_meta(): + result = runner.invoke( + upgrade, + ["control-plane", "--application", "keystone"], + obj=mock_deployment, + ) + assert result.exit_code == 0 + + def test_application_rejected_on_failed(self, mock_deployment): + client = mock_deployment.get_client.return_value + client.cluster.get_upgrade_state.return_value = _state_json( + groups={ + "identity-core": { + "status": "failed", + "last_error": {"code": "TEST", "message": "fail"}, + } + } + ) + runner = CliRunner() + with _patch_meta(): + result = runner.invoke( + upgrade, + ["control-plane", "--application", "keystone"], + obj=mock_deployment, + ) + assert result.exit_code != 0 + assert "failed" in result.output.lower() + + +class TestControlPlaneRetryGroup: + def test_retry_failed_group(self, mock_deployment): + client = mock_deployment.get_client.return_value + client.cluster.get_upgrade_state.return_value = _state_json( + groups={ + "identity-core": { + "status": "failed", + "last_error": {"code": "TEST", "message": "fail"}, + } + } + ) + runner = CliRunner() + with _patch_meta(): + result = runner.invoke( + upgrade, + ["control-plane", "--retry-group", "identity-core"], + obj=mock_deployment, + ) + assert result.exit_code == 0 + + def test_retry_not_failed(self, mock_deployment): + client = mock_deployment.get_client.return_value + client.cluster.get_upgrade_state.return_value = _state_json( + groups={ + "identity-core": { + "status": "completed", + "completed_at": "2025-01-01T00:00:00Z", + } + } + ) + runner = CliRunner() + with _patch_meta(): + result = runner.invoke( + upgrade, + ["control-plane", "--retry-group", "identity-core"], + obj=mock_deployment, + ) + assert result.exit_code != 0 + assert "not failed" in result.output.lower() + + +class TestFlagValidation: + def test_group_and_app_mutually_exclusive(self, mock_deployment): + runner = CliRunner() + with _patch_meta(): + result = runner.invoke( + upgrade, + [ + "control-plane", + "--group", + "identity-core", + "--application", + "keystone-k8s", + ], + obj=mock_deployment, + ) + assert result.exit_code != 0 + assert "mutually exclusive" in result.output.lower() + + def test_auto_and_group_mutually_exclusive(self, mock_deployment): + runner = CliRunner() + with _patch_meta(): + result = runner.invoke( + upgrade, + ["control-plane", "--auto", "--group", "identity-core"], + obj=mock_deployment, + ) + assert result.exit_code != 0 + assert "mutually exclusive" in result.output.lower() + + def test_no_flags_requires_one(self, mock_deployment): + runner = CliRunner() + with _patch_meta(): + result = runner.invoke(upgrade, ["control-plane"], obj=mock_deployment) + assert result.exit_code != 0 + assert "required" in result.output.lower() diff --git a/sunbeam-python/tests/unit/sunbeam/test_control_plane_groups.py b/sunbeam-python/tests/unit/sunbeam/test_control_plane_groups.py index 7a0e3f651..ec9707c72 100644 --- a/sunbeam-python/tests/unit/sunbeam/test_control_plane_groups.py +++ b/sunbeam-python/tests/unit/sunbeam/test_control_plane_groups.py @@ -116,16 +116,6 @@ def mock_deployment(): deployment.get_manifest.return_value = manifest deployment.get_juju_helper.return_value = jhelper deployment.get_client.return_value = client - - # Set up model status with apps matching metadata charm names - status = MagicMock() - keystone_app = MagicMock() - keystone_app.charm = "keystone-k8s" - glance_app = MagicMock() - glance_app.charm = "glance-k8s" - status.apps = {"keystone": keystone_app, "glance": glance_app} - jhelper.get_model_status.return_value = status - return deployment @@ -167,20 +157,6 @@ def test_terraform_apply_called_per_group(self, mock_deployment, coordinator): tfhelper = mock_deployment.get_tfhelper.return_value assert tfhelper.update_partial_tfvars_and_apply_tf.call_count == 2 - def test_terraform_init_runs_before_apply_per_group( - self, mock_deployment, coordinator - ): - metadata = _make_metadata() - state = coordinator.state - handler = ControlPlaneHandler(mock_deployment) - - handler.run(coordinator, metadata, state) - - tfhelper = mock_deployment.get_tfhelper.return_value - assert tfhelper.init.call_count == 2 - names = [c[0] for c in tfhelper.method_calls] - assert names.index("init") < names.index("update_partial_tfvars_and_apply_tf") - def test_convergence_wait_called_per_group(self, mock_deployment, coordinator): metadata = _make_metadata() state = coordinator.state @@ -316,22 +292,11 @@ def test_run_application_passes_target_args(self, mock_deployment, coordinator): metadata = _make_metadata() handler = ControlPlaneHandler(mock_deployment) - # Mock juju status: app "keystone" has charm "keystone-k8s" - jhelper = mock_deployment.get_juju_helper.return_value - status = MagicMock() - keystone_app = MagicMock() - keystone_app.charm = "keystone-k8s" - status.apps = {"keystone": keystone_app} - jhelper.get_model_status.return_value = status - handler.run_application(coordinator, metadata, "keystone-k8s") tfhelper = mock_deployment.get_tfhelper.return_value call = tfhelper.update_partial_tfvars_and_apply_tf.call_args assert call.kwargs["tf_apply_extra_args"] == ["-target=module.keystone"] - # Verify juju wait uses the app name, not the charm name - wait_call = jhelper.wait_until_desired_status.call_args - assert wait_call.args[1] == ["keystone"] def test_plan_group_passes_target_args(self, mock_deployment): """Verify -target args are passed to terraform plan in dry-run.""" From ee2346cf46885eff46ee27f84accf95282fd7b5f Mon Sep 17 00:00:00 2001 From: Hemanth Nakkina Date: Thu, 20 Aug 2026 08:33:03 +0530 Subject: [PATCH 16/20] fix(upgrades): warn and strip unknown charms instead of erroring Manifest Software charms that are no longer in the snap's default software config (e.g. removed between releases) are logged as warnings and stripped from the merged manifest rather than causing a hard error. Carved out of the consolidated testing-fixes commit 371dfbe5. --- sunbeam-python/sunbeam/core/manifest.py | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/sunbeam-python/sunbeam/core/manifest.py b/sunbeam-python/sunbeam/core/manifest.py index 260bb0eab..1433e7a2c 100644 --- a/sunbeam-python/sunbeam/core/manifest.py +++ b/sunbeam-python/sunbeam/core/manifest.py @@ -110,14 +110,28 @@ def validate_terraform_keys(self, default_software_config: "SoftwareConfig"): ) def validate_charm_keys(self, default_software_config: "SoftwareConfig"): - """Validate the charm keys provided are expected.""" + """Validate the charm keys provided are expected. + + Charms in the deployment manifest that are no longer in the + snap's default software config (e.g. removed between releases) + are logged as warnings and stripped from the merged manifest + rather than causing a hard error. + """ if self.charms: charms_keys = set(self.charms.keys()) - all_charms = default_software_config.charms.keys() - if not charms_keys <= all_charms: - raise ValueError( - f"Manifest Software charms keys should be one of {all_charms} " + all_charms = set(default_software_config.charms.keys()) + unknown = charms_keys - all_charms + if unknown: + LOG.debug( + "WARNING: Manifest Software charms %s are not in the " + "current default software config — they will be ignored. " + "Valid keys are: %s", + sorted(unknown), + sorted(all_charms), ) + # Strip unknown charms so they don't cause downstream errors + for key in unknown: + self.charms.pop(key, None) def validate_against_default( self, default_software_config: "SoftwareConfig" From b468f15945c92e8dde67482efa5fb6ae0bc5c05f Mon Sep 17 00:00:00 2001 From: Hemanth Nakkina Date: Thu, 20 Aug 2026 10:37:09 +0530 Subject: [PATCH 17/20] fix(upgrades): run terraform init before apply/plan in control-plane handler A snap refresh can bump the bundled juju terraform provider version, leaving the .terraform directory stale. Without init, terraform apply fails with 'unavailable provider registry.terraform.io/juju/juju'. Every other apply path in the codebase runs TerraformInitStep before apply; ControlPlaneHandler was the only one that skipped it. Add self.tfhelper.init() (which passes -upgrade, re-resolving providers from the snap filesystem mirror) before each terraform op: plan_group, run_application, and _upgrade_group. --- .../sunbeam/upgrades/control_plane/groups.py | 10 ++++++++-- .../unit/sunbeam/test_control_plane_groups.py | 14 ++++++++++++++ 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/sunbeam-python/sunbeam/upgrades/control_plane/groups.py b/sunbeam-python/sunbeam/upgrades/control_plane/groups.py index cef4b9472..f48c5e2c2 100644 --- a/sunbeam-python/sunbeam/upgrades/control_plane/groups.py +++ b/sunbeam-python/sunbeam/upgrades/control_plane/groups.py @@ -188,6 +188,7 @@ def plan_group( charms, group_meta.terraform_targets ) + self.tfhelper.init() events = self.tfhelper.update_partial_tfvars_and_plan_tf( client, self.manifest, @@ -316,6 +317,7 @@ def run_application( [app_name], group_meta.terraform_targets ) try: + self.tfhelper.init() self.tfhelper.update_partial_tfvars_and_apply_tf( client, self.manifest, @@ -327,7 +329,7 @@ def run_application( return PhaseResult( success=False, error_code=UpgradeErrorCode.CONTROL_PLANE_APPLY_FAILED, - error_message=f"Terraform apply failed for {app_name}: {e}", + error_message=f"Terraform init/apply failed for {app_name}: {e}", ) try: @@ -378,6 +380,10 @@ def _upgrade_group( ) try: + # ponytail: init re-resolves providers from the snap mirror; a snap + # refresh can bump the juju provider version, leaving the .terraform + # dir stale and apply failing with "unavailable provider". + self.tfhelper.init() self.tfhelper.update_partial_tfvars_and_apply_tf( client, self.manifest, @@ -391,7 +397,7 @@ def _upgrade_group( return PhaseResult( success=False, error_code=UpgradeErrorCode.CONTROL_PLANE_APPLY_FAILED, - error_message=f"Terraform apply failed for group {group_name}: {e}", + error_message=f"Terraform init/apply failed for {group_name}: {e}", ) try: diff --git a/sunbeam-python/tests/unit/sunbeam/test_control_plane_groups.py b/sunbeam-python/tests/unit/sunbeam/test_control_plane_groups.py index ec9707c72..678cc919c 100644 --- a/sunbeam-python/tests/unit/sunbeam/test_control_plane_groups.py +++ b/sunbeam-python/tests/unit/sunbeam/test_control_plane_groups.py @@ -157,6 +157,20 @@ def test_terraform_apply_called_per_group(self, mock_deployment, coordinator): tfhelper = mock_deployment.get_tfhelper.return_value assert tfhelper.update_partial_tfvars_and_apply_tf.call_count == 2 + def test_terraform_init_runs_before_apply_per_group( + self, mock_deployment, coordinator + ): + metadata = _make_metadata() + state = coordinator.state + handler = ControlPlaneHandler(mock_deployment) + + handler.run(coordinator, metadata, state) + + tfhelper = mock_deployment.get_tfhelper.return_value + assert tfhelper.init.call_count == 2 + names = [c[0] for c in tfhelper.method_calls] + assert names.index("init") < names.index("update_partial_tfvars_and_apply_tf") + def test_convergence_wait_called_per_group(self, mock_deployment, coordinator): metadata = _make_metadata() state = coordinator.state From 29511ff8679017329ea88fb62befeb3e3f38990b Mon Sep 17 00:00:00 2001 From: Hemanth Nakkina Date: Sat, 22 Aug 2026 13:51:52 +0530 Subject: [PATCH 18/20] fix(upgrades): override target charm manifests, resolve app names, skip undeployed charms --- .../sunbeam/upgrades/control_plane/groups.py | 212 ++++++++++++++---- .../unit/sunbeam/test_control_plane_groups.py | 21 ++ 2 files changed, 191 insertions(+), 42 deletions(-) diff --git a/sunbeam-python/sunbeam/upgrades/control_plane/groups.py b/sunbeam-python/sunbeam/upgrades/control_plane/groups.py index f48c5e2c2..7dec487be 100644 --- a/sunbeam-python/sunbeam/upgrades/control_plane/groups.py +++ b/sunbeam-python/sunbeam/upgrades/control_plane/groups.py @@ -18,9 +18,14 @@ import logging import typing +import click +import yaml +from rich.console import Console + +from sunbeam.core.common import RiskLevel, infer_risk from sunbeam.core.deployment import Deployment from sunbeam.core.juju import JujuHelper, JujuWaitException -from sunbeam.core.manifest import Manifest +from sunbeam.core.manifest import Manifest, embedded_manifest_path from sunbeam.core.openstack import OPENSTACK_MODEL from sunbeam.core.terraform import TerraformException, TerraformHelper from sunbeam.steps.openstack import CONFIG_KEY as OPENSTACK_CONFIG_KEY @@ -37,12 +42,41 @@ from sunbeam.upgrades.state import Group, LastError, PhaseStatus, UpgradeState LOG = logging.getLogger(__name__) +console = Console() def _now_iso() -> str: return datetime.datetime.now(tz=datetime.timezone.utc).isoformat() +def _load_target_charm_manifests(to_release: str) -> dict[str, dict]: + """Load per-charm manifest entries from the snap's embedded target manifest. + + Reads ``etc/manifests//.yml`` from the snap and + returns a mapping of charm name to its raw manifest dict (channel, + revision, config, resources — whatever the embedded manifest carries). + + :param to_release: target release, e.g. "2026.1" + :returns: charm name -> raw manifest dict, or empty dict if not found + """ + from snaphelpers import Snap + + try: + snap = Snap() + risk = infer_risk(snap) + if risk == RiskLevel.STABLE: + risk_str = "stable" + else: + risk_str = str(risk) + path = embedded_manifest_path(snap, to_release, risk_str) + data = yaml.safe_load(path.read_text()) + charms = data.get("core", {}).get("software", {}).get("charms", {}) + return dict(charms.items()) + except Exception as e: + LOG.warning("Could not load target manifest for %s: %s", to_release, e) + return {} + + def _terraform_targets_for_charms( charms: list[str], terraform_targets: dict[str, list[str]] ) -> list[str]: @@ -75,11 +109,13 @@ class ControlPlaneHandler: Implements the PhaseHandler protocol. """ - def __init__(self, deployment: Deployment): + def __init__(self, deployment: Deployment, to_release: str = ""): self.deployment = deployment + self.to_release = to_release self._tfhelper: TerraformHelper | None = None self._manifest: Manifest | None = None self._jhelper: JujuHelper | None = None + self._target_channels: dict[str, dict] | None = None @property def tfhelper(self) -> TerraformHelper: @@ -102,6 +138,41 @@ def jhelper(self) -> JujuHelper: self._jhelper = self.deployment.get_juju_helper() return self._jhelper + @property + def target_charms(self) -> dict[str, dict]: + """Per-charm manifest entries from the snap's embedded target manifest.""" + if self._target_channels is None: + self._target_channels = _load_target_charm_manifests(self.to_release) + return self._target_channels + + def _override_charm_manifests(self, charms: list[str]) -> None: + """Override manifest entries for the group's charms from the target release. + + Replaces the full CharmManifest (channel, revision, config, resources) + for the group's charms with values from the snap's embedded target + manifest. Other charms keep their current (clusterd) values. + """ + if not self.to_release: + return + from sunbeam.core.manifest import CharmManifest + + for charm in charms: + target_cfg = self.target_charms.get(charm) + if target_cfg and charm in self.manifest.core.software.charms: + old = self.manifest.core.software.charms[charm].channel + self.manifest.core.software.charms[charm] = CharmManifest(**target_cfg) + LOG.info( + "overrode %s channel: %s -> %s", + charm, + old, + target_cfg.get("channel"), + ) + elif target_cfg: + LOG.warning( + "charm %s not in deployment manifest, skipping override", + charm, + ) + def run( self, coordinator: ReleaseUpgradeCoordinator, @@ -188,6 +259,7 @@ def plan_group( charms, group_meta.terraform_targets ) + self._override_charm_manifests(charms) self.tfhelper.init() events = self.tfhelper.update_partial_tfvars_and_plan_tf( client, @@ -278,11 +350,14 @@ def run_application( self, coordinator: ReleaseUpgradeCoordinator, metadata: HopMetadata | None, - app_name: str, + charm_name: str, ) -> PhaseResult: """Upgrade a single application via scoped terraform apply. Does not run pre/post actions (those are group-level). + + :param charm_name: charm name (e.g. 'placement-k8s'), resolved + from the juju app name by the CLI. """ if metadata is None: return PhaseResult( @@ -299,53 +374,89 @@ def run_application( error_message="No active hop", ) - # Verify the app exists in some group + # Verify the charm exists in some group group_meta = None for g in metadata.control_plane_groups: - if app_name in g.apps: + if charm_name in g.apps: group_meta = g break if group_meta is None: return PhaseResult( success=False, error_code=UpgradeErrorCode.METADATA_INVALID, - error_message=f"Application {app_name} not found in any group", + error_message=f"Charm {charm_name} not found in any group", ) + # Mark group as in_progress (single-app upgrade doesn't complete the group) + control_plane = hop.phases.control_plane + group_state = control_plane.groups.get(group_meta.name) + if group_state is None: + group_state = Group() + control_plane.groups[group_meta.name] = group_state + if group_state.status != PhaseStatus.COMPLETED: + group_state.status = PhaseStatus.IN_PROGRESS + group_state.started_at = _now_iso() + coordinator.persist_state() + client = self.deployment.get_client() target_args = _terraform_targets_for_charms( - [app_name], group_meta.terraform_targets + [charm_name], group_meta.terraform_targets ) - try: - self.tfhelper.init() - self.tfhelper.update_partial_tfvars_and_apply_tf( - client, - self.manifest, - [app_name], - OPENSTACK_CONFIG_KEY, - tf_apply_extra_args=target_args, + self._override_charm_manifests([charm_name]) + + # Resolve charm name to deployed juju app name for Juju operations + status = self.jhelper.get_model_status(OPENSTACK_MODEL) + juju_app_names = [ + name for name, app in status.apps.items() if app.charm == charm_name + ] + if not juju_app_names: + return PhaseResult( + success=False, + error_code=UpgradeErrorCode.METADATA_INVALID, + error_message=f"No deployed application found for charm {charm_name}", ) + + try: + with console.status(f" {charm_name}: applying terraform plan..."): + self.tfhelper.init() + self.tfhelper.update_partial_tfvars_and_apply_tf( + client, + self.manifest, + [charm_name], + OPENSTACK_CONFIG_KEY, + tf_apply_extra_args=target_args, + ) except TerraformException as e: + click.echo(f" {charm_name}: terraform apply FAILED") return PhaseResult( success=False, error_code=UpgradeErrorCode.CONTROL_PLANE_APPLY_FAILED, - error_message=f"Terraform init/apply failed for {app_name}: {e}", + error_message=f"Terraform init/apply failed for {charm_name}: {e}", ) try: - self.jhelper.wait_until_desired_status( - OPENSTACK_MODEL, - [app_name], - status=["active"], - timeout=600, - ) + with console.status(f" {charm_name}: waiting for convergence..."): + self.jhelper.wait_until_desired_status( + OPENSTACK_MODEL, + juju_app_names, + status=["active"], + timeout=600, + ) except (JujuWaitException, TimeoutError) as e: + click.echo(f" {charm_name}: convergence timeout") return PhaseResult( success=False, error_code=UpgradeErrorCode.CONTROL_PLANE_CONVERGENCE_TIMEOUT, - error_message=f"{app_name} did not converge: {e}", + error_message=f"{charm_name} did not converge: {e}", ) + # If this was the only app in the group, mark group as completed + if len(group_meta.apps) == 1: + group_state.status = PhaseStatus.COMPLETED + group_state.completed_at = _now_iso() + coordinator.persist_state() + click.echo(f" {charm_name}: completed") + return PhaseResult(success=True) def _upgrade_group( @@ -362,14 +473,22 @@ def _upgrade_group( group_meta.ready_timeout_sec, ) - # Resolve charm names to deployed app names for Juju operations + # Resolve charm names to deployed app names, skipping undeployed charms status = self.jhelper.get_model_status(OPENSTACK_MODEL) + deployed_charms = {app.charm for app in status.apps.values()} + charms = [c for c in charms if c in deployed_charms] app_names = [name for name, app in status.apps.items() if app.charm in charms] + if not charms: + LOG.info("group %s has no deployed charms — skipping", group_name) + click.echo(f" {group_name}: no deployed charms — skipping") + return PhaseResult(success=True) LOG.info("resolved charms %s to apps %s", charms, app_names) # Pre-upgrade actions (use app names, not charm names) - pre_result = run_pre_actions(self.jhelper, group_meta.pre_actions) + with console.status(f" {group_name}: running pre-upgrade actions..."): + pre_result = run_pre_actions(self.jhelper, group_meta.pre_actions) if not pre_result.success: + click.echo(f" {group_name}: pre-upgrade actions FAILED") # Still run post-actions as cleanup attempt run_post_actions(self.jhelper, group_meta.post_actions) return pre_result @@ -379,19 +498,22 @@ def _upgrade_group( charms, group_meta.terraform_targets ) + self._override_charm_manifests(charms) try: # ponytail: init re-resolves providers from the snap mirror; a snap # refresh can bump the juju provider version, leaving the .terraform # dir stale and apply failing with "unavailable provider". - self.tfhelper.init() - self.tfhelper.update_partial_tfvars_and_apply_tf( - client, - self.manifest, - charms, - OPENSTACK_CONFIG_KEY, - tf_apply_extra_args=target_args, - ) + with console.status(f" {group_name}: applying terraform plan..."): + self.tfhelper.init() + self.tfhelper.update_partial_tfvars_and_apply_tf( + client, + self.manifest, + charms, + OPENSTACK_CONFIG_KEY, + tf_apply_extra_args=target_args, + ) except TerraformException as e: + click.echo(f" {group_name}: terraform apply FAILED") # Run post-actions as cleanup even on failure run_post_actions(self.jhelper, group_meta.post_actions) return PhaseResult( @@ -401,13 +523,18 @@ def _upgrade_group( ) try: - self.jhelper.wait_until_desired_status( - OPENSTACK_MODEL, - app_names, - status=["active"], - timeout=group_meta.ready_timeout_sec, - ) + with console.status( + f" {group_name}: waiting for convergence " + f"({group_meta.ready_timeout_sec}s)..." + ): + self.jhelper.wait_until_desired_status( + OPENSTACK_MODEL, + app_names, + status=["active"], + timeout=group_meta.ready_timeout_sec, + ) except (JujuWaitException, TimeoutError) as e: + click.echo(f" {group_name}: convergence timeout") # Run post-actions as cleanup even on failure run_post_actions(self.jhelper, group_meta.post_actions) return PhaseResult( @@ -420,10 +547,10 @@ def _upgrade_group( ) # Post-upgrade actions - post_result = run_post_actions(self.jhelper, group_meta.post_actions) + with console.status(f" {group_name}: running post-upgrade actions..."): + post_result = run_post_actions(self.jhelper, group_meta.post_actions) if not post_result.success: - # Group upgraded successfully but post-action failed — - # healthcheck may not be restored. Mark blocked. + click.echo(f" {group_name}: post-upgrade actions FAILED") return PhaseResult( success=False, error_code=UpgradeErrorCode.CONTROL_PLANE_ACTION_FAILED, @@ -435,4 +562,5 @@ def _upgrade_group( ), ) + click.echo(f" {group_name}: completed") return PhaseResult(success=True) diff --git a/sunbeam-python/tests/unit/sunbeam/test_control_plane_groups.py b/sunbeam-python/tests/unit/sunbeam/test_control_plane_groups.py index 678cc919c..7a0e3f651 100644 --- a/sunbeam-python/tests/unit/sunbeam/test_control_plane_groups.py +++ b/sunbeam-python/tests/unit/sunbeam/test_control_plane_groups.py @@ -116,6 +116,16 @@ def mock_deployment(): deployment.get_manifest.return_value = manifest deployment.get_juju_helper.return_value = jhelper deployment.get_client.return_value = client + + # Set up model status with apps matching metadata charm names + status = MagicMock() + keystone_app = MagicMock() + keystone_app.charm = "keystone-k8s" + glance_app = MagicMock() + glance_app.charm = "glance-k8s" + status.apps = {"keystone": keystone_app, "glance": glance_app} + jhelper.get_model_status.return_value = status + return deployment @@ -306,11 +316,22 @@ def test_run_application_passes_target_args(self, mock_deployment, coordinator): metadata = _make_metadata() handler = ControlPlaneHandler(mock_deployment) + # Mock juju status: app "keystone" has charm "keystone-k8s" + jhelper = mock_deployment.get_juju_helper.return_value + status = MagicMock() + keystone_app = MagicMock() + keystone_app.charm = "keystone-k8s" + status.apps = {"keystone": keystone_app} + jhelper.get_model_status.return_value = status + handler.run_application(coordinator, metadata, "keystone-k8s") tfhelper = mock_deployment.get_tfhelper.return_value call = tfhelper.update_partial_tfvars_and_apply_tf.call_args assert call.kwargs["tf_apply_extra_args"] == ["-target=module.keystone"] + # Verify juju wait uses the app name, not the charm name + wait_call = jhelper.wait_until_desired_status.call_args + assert wait_call.args[1] == ["keystone"] def test_plan_group_passes_target_args(self, mock_deployment): """Verify -target args are passed to terraform plan in dry-run.""" From 996f81d49c28af066d35c5c43038606309e7c249 Mon Sep 17 00:00:00 2001 From: Hemanth Nakkina Date: Fri, 28 Aug 2026 16:02:48 +0530 Subject: [PATCH 19/20] feat(upgrades): Update docs with high level specs specs updated in docs section for clsuter upgrades. Assisted-by: z.ai/glm-5.2 Signed-off-by: Hemanth Nakkina --- .../specs/cluster-release-upgrade-design.md | 194 ++++++++++++++++++ 1 file changed, 194 insertions(+) create mode 100644 docs/agents/specs/cluster-release-upgrade-design.md diff --git a/docs/agents/specs/cluster-release-upgrade-design.md b/docs/agents/specs/cluster-release-upgrade-design.md new file mode 100644 index 000000000..bdf52656d --- /dev/null +++ b/docs/agents/specs/cluster-release-upgrade-design.md @@ -0,0 +1,194 @@ +# Cluster Release Upgrade Engine Design + +## Problem + +Sunbeam has no supported path to move a cluster from one OpenStack release to +the next (e.g. 2025.1 → 2026.1). Operators hand-compose charm refreshes, +terraform applies, and juju actions with no state tracking, no ordering +guarantees, and nothing stopping two cluster-mutating operations from running +at once. A failed upgrade leaves a mixed-release cluster with no record of +where it stopped. + +## Architecture + +### Components + +| Component | Location | Responsibility | +|---|---|---| +| `sunbeam cluster upgrade` CLI | `sunbeam/commands/upgrade.py` | `preflight`, `control-plane`, `abandon` subcommands; flag validation; rendering status/plan output | +| `GuardedGroup` | `sunbeam/utils.py` | Blocks every mutating subcommand (`bootstrap`, `add`, `join`, `remove`, `resize`, `destroy`, `configure`, `refresh`, `enable`, `disable`, …) while an upgrade is active | +| Preflight | `sunbeam/upgrades/preflight/` | Sequential checks (snap version, hop validity, cluster health, capacity, MySQL quorum); also creates the active hop in state | +| Metadata loader | `sunbeam/upgrades/metadata.py` | Parses the per-release `upgrade.yml` (groups, terraform targets, actions, steps, timeouts, prerequisites) into a typed schema | +| Coordinator | `sunbeam/upgrades/coordinator.py` | Owns hop lifecycle: lock, load/save state, run a phase via a `PhaseHandler`, transition hop/phase with validity checks | +| State model | `sunbeam/upgrades/state.py` | Typed hop/phase/group/node/step tree, transition tables, `active_hop` index into `hop_history` | +| Control-plane handler | `sunbeam/upgrades/control_plane/` | Executes a group: pre-actions → manifest override → scoped terraform apply → convergence wait → post-actions; also `--retry-group` and `--status` support | +| Upgrade guard helper | `sunbeam/utils.py` | Fail-open check wrapping `clusterd.is_upgrade_active()` | +| Lock primitives | `sunbeam-microcluster/database/upgrade_lock.go` | Single-SQL-row `upgrade_lock` with monotonic fencing token; acquire / refresh / release / verify | +| Lock+state service | `sunbeam-microcluster/sunbeam/upgrade.go` | Token serialization, TTLs, and token-verify-then-write-state in one transaction | +| REST API | `sunbeam-microcluster/api/upgrade.go` | `POST/PUT/DELETE /1.0/upgrade/lock`, `GET/PUT /1.0/upgrade/state`, `GET /1.0/upgrade/active` | +| Python client | `sunbeam/clusterd/` | Acquire/refresh/release lock and get/update state; maps 404/409 to typed exceptions | +| Audit log | `sunbeam/upgrades/observability.py` | DEBUG-level structured log of state changes, lock events, and command invocations | + +### Data model + +`UpgradeState` is a single JSON blob in clusterd under key `upgrade_state`: + +- `hop_history: [Hop]` — every hop ever started. Append-only; the canonical + record. +- `active_hop.hop_history_index` — pointer into `hop_history`. `None` when no + upgrade is active. +- A hop has `status`, `phase` (current phase name), `last_error`, + `metadata_version` (1), `metadata_build_id` (snap revision at hop creation), + and `phases` {`preflight`, `control_plane`, `dataplane`, `storage`, + `finalize`}. +- `control_plane` phase contains `groups: dict[name → Group]` with per-group + status + timestamps. +- `dataplane` phase (schema only for now) contains per-node `step` and + `step_status`, plus `components` tracking previous/target charm channel per + unit. + +Transitions are enforced by `VALID_HOP_TRANSITIONS` / +`VALID_PHASE_TRANSITIONS` tables in the coordinator — `TransitionError` on +violation. + +### Locking + +Locking separates the *what* (state blob) from the *who* (the holder): + +- The `upgrade_lock` table is always present (schema-apply inserts row id=1). +- TTL is 60 s; the CLI heartbeats every 30 s from a daemon thread. +- Acquire when held → HTTP 409; refresh/release with wrong token → 409. +- Token is never reset, so a stale holder's later writes are rejected at the + DB — the *fencing token* pattern. +- The CLI group guard checks lock liveness (holder ≠ empty and not expired), + not hop status: any writing CLI process holds the lock for the duration of + its mutation, which also blocks guard-pass admission concurrently. + +### Metadata-driven orchestration + +`manifests//upgrade.yml` is read generically by the loader; the +coordinator and handlers execute it. Per release it declares, in order: + +- `control_plane_groups`: name, apps, `ready_timeout_sec`, `pre_actions` / + `post_actions` (`{action, apps, scope: leader|all-units}`), + `terraform_targets: {charm → [resource addresses]}`. +- `dataplane` and `storage`: step sequences and timeout defaults (schema only + for now). +- `finalize`: ordered steps of type `engine` or `action`. +- `required_prerequisites` (snap channel, infra components), `compatibility` + (pre/post hop actions) — declared but not yet executed. + +Adding a new release = new `upgrade.yml` + one `RELEASE_TRACKS` entry in +`versions.py`. + +### Control-plane group execution + +For each group (in metadata order, or via `--group` / `--application`): + +1. Filter `apps` to charms actually deployed in the `openstack` model; + resolve charm → deployed app names. Empty → success. +2. Run `pre_actions` via `juju run /leader …`; on failure, post-actions + still run as cleanup. +3. `_override_charm_manifests()` — load the target snap's embedded + `etc/manifests//.yml` and replace each charm's full + `CharmManifest` in the in-memory deployment manifest. +4. `tfhelper.init()` then `update_partial_tfvars_and_apply_tf(..., + tf_apply_extra_args=["-target=…"])` using `terraform_targets`. +5. `wait_until_desired_status(openstack, apps, ["active"], + timeout=ready_timeout_sec)`; `CONTROL_PLANE_CONVERGENCE_TIMEOUT` on + failure. +6. Run `post_actions`; failure → `CONTROL_PLANE_ACTION_FAILED` (apply already + succeeded — message notes manual intervention may be required). + +`--dry-run` iterates groups and calls `plan_group()` — same tfvars update but +`update_partial_tfvars_and_plan_tf`, which does not save tfvars to the DB. + +`--retry-group` requires a FAILED/BLOCKED group, resets it to PENDING, and +re-runs it. `--status` renders per-group status + timestamps. + +### Failure model + +- Whole-blob writes: SIGKILL leaves either old or new state. +- `in_progress` steps on resume are treated as failed and re-executed — + steps must be idempotent (terraform apply, db_sync, snap refresh). +- Lock holder death → TTL expiry; a new holder acquires with higher token; + dead holder's writes are rejected with 409. +- `blocked` status means operator intervention needed; `abandon` is the only + transition out (with a pointer to `sunbeam restore`). + +### Sequencing + +Preflight gates all mutation and ships first. Control-plane before +dataplane: new control plane + old compute is the supported mixed-version +state; the reverse is not. Finalize (deferred integration re-apply, RPC +cache refresh, migrations) only runs after the data plane completes. + +## Alternatives considered + +### Execution and state model + +| Alternative | Rejected because | +|---|---| +| Reconciler-based execution | Workflow-coordinator matches the operator's mental model of an ordered procedure. | +| Canonical `active_hop` record copied off `hop_history` | Keeps a dual-write problem for the hop's entire active life; index reference avoids it. | +| Per-field state updates | Needs per-field crash recovery; whole-blob writes replace all of it. | +| TTL lock without fencing token | Can't distinguish expired-then-reacquired from slow holder (GC pause); monotonic token is the standard fencing pattern. | +| Lock as a `config` key | Config table has no CAS primitive; dedicated table CAS-es the token column. | +| Canonical DataUpgrade charm library | Doesn't match Sunbeam's needs; would require a fork. | + +### Terraform strategy + +| Alternative | Rejected because | +|---|---| +| Full plan decomposition | Monolithic plans are sufficient; scoped `-target` per group is the lighter answer and enables precise `--retry-group`. | +| Channel-only tfvars with cascading fallback (spec v1) | Clusterd manifest holds source-release channels → plan is a silent no-op. Overriding the full `CharmManifest` from the target snap's embedded manifest is the only way the plan sees the new release. | +| Error on unknown charms | Optional features may simply not be deployed; warning+strip accepts partial deployments. | +| Reuse `.terraform` dir | Snap refresh can bump the juju provider version and stale the dir; `terraform init` before apply/plan is cheap insurance. | + +### Charm interaction + +| Alternative | Rejected because | +|---|---| +| `upgrade-charm` hook only | Fires on every refresh; the charm can't distinguish cross-track from in-track. Explicit `pre-upgrade`/`post-upgrade` actions fire only at hop boundaries. | +| Upgrade-mode config flag | Config is persistent and app-level — wrong semantics for transient upgrade state. | +| Drain/undrain actions | Can't control which unit restarts when (parallel pod management). | +| StatefulSet partition patching / kubectl patching | Fights Juju reconciliation; not Sunbeam-native. | + +### Scope and sequencing + +| Alternative | Rejected because | +|---|---| +| Guard check inside each command handler | Fragile — new commands miss it; central `GuardedGroup` intercepts at the click-group level. Fail-open on clusterd-unreachable is deliberate: CLI must not wedge. | +| Fixed convergence timeout | db_sync-heavy nova deployments exceed 10 min on first hop; timeout is per-group in metadata (default 600s, nova 900s). | +| OpenStack API capacity check | Requires admin credentials in preflight; the charm's `running-guests` action needs none. | +| Backup creation/verification in preflight | Artifacts live anywhere and restore is component-specific (MySQL/Vault); engine stays forward-only and prints a disclaimer. | +| Blue-green upgrades | Too complex for the first release. | +| Provider architecture refactor | Independent modernization, not a prerequisite. | + +Juju `OrderedReady` would remove the explicit ordering actions, but is an +upstream feature request out of Sunbeam's control; the pre/post-action +separation is the foundation regardless. + +## Deferred scope + +- Dataplane and storage phase handlers (schema + metadata exist, no handler). +- Finalize phase handler (`verify-upgrade-levels`, `reapply-terraform`, + `upgrade-features`, `validate-end-state`). +- `sunbeam cluster upgrade status` (only `control-plane --status`). +- `--retry-node` / `--rollback-node` flags. +- `required_prerequisites` / `compatibility.pre|post_hop` execution. +- Hop-level PENDING→IN_PROGRESS transition (only `abandon()` touches it today). +- `metadata_build_id` (snap revision) recorded but not yet validated mid-hop. +- messaging-core (rabbitmq) group commented out; no recorded rationale — + pending decision. + +## Consequences + +- New release = new `upgrade.yml` + one `RELEASE_TRACKS` entry. +- Operators dry-validate before mutating (preflight checks + `control-plane + --dry-run` terraform plan). +- Stale CLI cannot corrupt state — rejected at the DB with 409. +- Resume after SIGKILL re-executes the last in-progress step; idempotence is + the contract. +- `--retry-group` = surgical recovery for failed control-plane groups. +- Until dataplane/finalize land, hops never reach COMPLETED. From 7b58fcb7f7eb527407848a0bb21fc634d2cb71e1 Mon Sep 17 00:00:00 2001 From: Hemanth Nakkina Date: Sun, 30 Aug 2026 11:45:11 +0530 Subject: [PATCH 20/20] fix(upgrades): set control-plane phase status and hop.phase on group completion MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The control-plane handler never transitioned phase status or hop.phase, leaving pending forever. W6.3's status command needs these to show correct phase-level progress. Sets: - hop.phase = 'control_plane' on first group/app start - phase status = IN_PROGRESS on first group/app start - phase status = COMPLETED when all groups completed (checked via all_completed guard, not just last-group-is-done) Also adds all_completed guard to run_group() and run_application() completion paths — phase is only marked COMPLETED if every group reports COMPLETED, preventing premature phase completion when a group was retried and failed. --- .../sunbeam/upgrades/control_plane/groups.py | 45 ++++++++++++++ .../unit/sunbeam/test_control_plane_groups.py | 61 ++++++++++++++++--- 2 files changed, 97 insertions(+), 9 deletions(-) diff --git a/sunbeam-python/sunbeam/upgrades/control_plane/groups.py b/sunbeam-python/sunbeam/upgrades/control_plane/groups.py index 7dec487be..0b0e81d49 100644 --- a/sunbeam-python/sunbeam/upgrades/control_plane/groups.py +++ b/sunbeam-python/sunbeam/upgrades/control_plane/groups.py @@ -201,6 +201,12 @@ def run( control_plane = hop.phases.control_plane + # Set phase status on first group start + if control_plane.status == PhaseStatus.PENDING: + control_plane.status = PhaseStatus.IN_PROGRESS + hop.phase = "control_plane" + coordinator.persist_state() + for group_meta in metadata.control_plane_groups: group_name = group_meta.name @@ -234,6 +240,10 @@ def run( LOG.warning("group %s failed: %s", group_name, result.error_message) return result + # All groups done — mark phase completed + control_plane.status = PhaseStatus.COMPLETED + coordinator.persist_state() + return PhaseResult(success=True) def plan_group( @@ -329,6 +339,12 @@ def run_group( group_state = Group() control_plane.groups[group_name] = group_state + # Set phase status on first group start + if control_plane.status == PhaseStatus.PENDING: + control_plane.status = PhaseStatus.IN_PROGRESS + hop.phase = "control_plane" + coordinator.persist_state() + group_state.status = PhaseStatus.IN_PROGRESS group_state.started_at = _now_iso() coordinator.persist_state() @@ -337,6 +353,17 @@ def run_group( if result.success: group_state.status = PhaseStatus.COMPLETED group_state.completed_at = _now_iso() + + # If this was the last group in the metadata and all groups + # completed, mark phase completed + all_completed = all( + gs.status == PhaseStatus.COMPLETED + for gs in control_plane.groups.values() + ) + is_last = group_name == metadata.control_plane_groups[-1].name + if is_last and all_completed: + control_plane.status = PhaseStatus.COMPLETED + coordinator.persist_state() else: group_state.status = PhaseStatus.FAILED group_state.last_error = LastError( @@ -398,6 +425,12 @@ def run_application( group_state.started_at = _now_iso() coordinator.persist_state() + # Set phase status on first app start + if control_plane.status == PhaseStatus.PENDING: + control_plane.status = PhaseStatus.IN_PROGRESS + hop.phase = "control_plane" + coordinator.persist_state() + client = self.deployment.get_client() target_args = _terraform_targets_for_charms( [charm_name], group_meta.terraform_targets @@ -455,6 +488,18 @@ def run_application( group_state.status = PhaseStatus.COMPLETED group_state.completed_at = _now_iso() coordinator.persist_state() + + # If this was the last group in the metadata and all groups + # completed, mark phase completed + all_completed = all( + gs.status == PhaseStatus.COMPLETED + for gs in control_plane.groups.values() + ) + is_last = group_meta.name == metadata.control_plane_groups[-1].name + if is_last and all_completed: + control_plane.status = PhaseStatus.COMPLETED + coordinator.persist_state() + click.echo(f" {charm_name}: completed") return PhaseResult(success=True) diff --git a/sunbeam-python/tests/unit/sunbeam/test_control_plane_groups.py b/sunbeam-python/tests/unit/sunbeam/test_control_plane_groups.py index 7a0e3f651..760500ee2 100644 --- a/sunbeam-python/tests/unit/sunbeam/test_control_plane_groups.py +++ b/sunbeam-python/tests/unit/sunbeam/test_control_plane_groups.py @@ -156,6 +156,18 @@ def test_all_groups_succeed(self, mock_deployment, coordinator): cp = state.current_hop.phases.control_plane assert cp.groups["identity-core"].status == PhaseStatus.COMPLETED assert cp.groups["image"].status == PhaseStatus.COMPLETED + # Phase status set to completed after all groups + assert cp.status == PhaseStatus.COMPLETED + + def test_phase_set_to_control_plane(self, mock_deployment, coordinator): + """Verify hop.phase is set to control_plane on run.""" + metadata = _make_metadata() + state = coordinator.state + handler = ControlPlaneHandler(mock_deployment) + + handler.run(coordinator, metadata, state) + + assert state.current_hop.phase == "control_plane" def test_terraform_apply_called_per_group(self, mock_deployment, coordinator): metadata = _make_metadata() @@ -242,6 +254,8 @@ def test_skips_completed_groups_on_resume(self, mock_deployment, coordinator): tfhelper = mock_deployment.get_tfhelper.return_value # Only image group should be upgraded assert tfhelper.update_partial_tfvars_and_apply_tf.call_count == 1 + # Phase status stays completed (already-in-progress phases resume) + assert state.current_hop.phases.control_plane.status == PhaseStatus.COMPLETED def test_no_metadata_returns_failure(self, mock_deployment, coordinator): state = coordinator.state @@ -333,16 +347,45 @@ def test_run_application_passes_target_args(self, mock_deployment, coordinator): wait_call = jhelper.wait_until_desired_status.call_args assert wait_call.args[1] == ["keystone"] - def test_plan_group_passes_target_args(self, mock_deployment): - """Verify -target args are passed to terraform plan in dry-run.""" + def test_run_group_does_not_complete_phase_with_failed_group( + self, mock_deployment, coordinator + ): + """run_group should not mark phase completed if another group failed.""" metadata = _make_metadata() + state = coordinator.state + + # Mark first group as failed + cp = state.current_hop.phases.control_plane + cp.groups["identity-core"] = Group( + status=PhaseStatus.FAILED, + last_error={"code": "TEST", "message": "test"}, + ) + handler = ControlPlaneHandler(mock_deployment) - group_meta = metadata.control_plane_groups[0] + # Run the second (last) group via run_group + result = handler.run_group(coordinator, metadata, "image") - handler.plan_group(group_meta) + assert result.success is True + # Phase should NOT be marked completed — identity-core is failed + assert cp.status != PhaseStatus.COMPLETED - tfhelper = mock_deployment.get_tfhelper.return_value - plan_call = tfhelper.update_partial_tfvars_and_plan_tf.call_args - assert plan_call.kwargs["tf_plan_extra_args"] == ["-target=module.keystone"] - text_call = tfhelper.terraform_plan_text.call_args - assert text_call.kwargs["extra_args"] == ["-target=module.keystone"] + def test_run_group_completes_phase_when_all_groups_done( + self, mock_deployment, coordinator + ): + """run_group marks phase completed only if all groups completed.""" + metadata = _make_metadata() + state = coordinator.state + + # Mark first group as completed + cp = state.current_hop.phases.control_plane + cp.groups["identity-core"] = Group( + status=PhaseStatus.COMPLETED, + started_at="2025-01-01T00:00:00Z", + completed_at="2025-01-01T00:01:00Z", + ) + + handler = ControlPlaneHandler(mock_deployment) + result = handler.run_group(coordinator, metadata, "image") + + assert result.success is True + assert cp.status == PhaseStatus.COMPLETED