fix(maas): clean up node state before removal

sunbeam-python/sunbeam/core/juju.py

@@ -759,22 +759,22 @@ def run_cmd_on_machine_unit_payload(
     ) -> "jubilant.Task":
         """Run a shell command on a machine unit.
 
-        Returns action results irrespective of the return-code
-        in action results.
-
         :name: unit name
         :model: Name of the model where the application is located
         :cmd: Command to run
         :timeout: Timeout in seconds
         :returns: Command results
-
-        Command execution failures are part of the results with
-        return-code, stdout, stderr.
+        :raises: ExecFailedException if command execution fails
         """
         with self._model(model) as juju:
             try:
                 task = juju.exec(cmd, unit=name, wait=timeout)
-            except jubilant.TaskError as e:
+            except (
+                jubilant.TaskError,
+                jubilant.CLIError,
+                TimeoutError,
+                ValueError,
+            ) as e:
                 raise ExecFailedException(
                     f"Failed to run command {cmd!r} on unit"
                     f" {name!r} in model {model!r}: {e}"
@@ -820,12 +820,19 @@ def run_cmd_on_unit_payload(
         if env:
             args.extend(f"--env={k}={v}" for k, v in env.items())
 
+        stderr = ""
         with self._model(model) as juju:
             try:
                 stdout, _ = juju._cli(*args, "--", *(cmd.split()), log=False)
             except jubilant.CLIError as e:
-                stdout = e.stdout
-        return json.loads(stdout)[name]["results"]
+                stdout, stderr = e.stdout, e.stderr or ""
+        try:
+            return json.loads(stdout)[name]["results"]
+        except (json.JSONDecodeError, KeyError, TypeError) as e:
+            raise ExecFailedException(
+                f"Failed to parse command result for unit {name!r}: "
+                f"{stderr.strip() or e}"
+            ) from e
 
     def run_action(
         self,

sunbeam-python/sunbeam/provider/maas/client.py

@@ -341,6 +341,7 @@ def _convert_raw_machine(machine_raw: dict, root_disk: dict | None) -> dict:
     machine = {
         "system_id": machine_raw["system_id"],
         "hostname": machine_raw["hostname"],
+        "fqdn": machine_raw["fqdn"],
         "roles": list(set(tag_names).intersection(RoleTags.values())),
         "zone": machine_raw["zone"]["name"],
         "status": machine_raw["status_name"],

sunbeam-python/sunbeam/provider/maas/commands.py

@@ -125,6 +125,8 @@
     CheckCinderVolumeDistributionStep,
     DeployCinderVolumeApplicationStep,
     DestroyCinderVolumeApplicationStep,
+    DisableCinderVolumeServicesStep,
+    RemoveCinderVolumeServicesStep,
     RemoveCinderVolumeUnitsStep,
 )
 from sunbeam.steps.clusterd import APPLICATION as CLUSTERD_APPLICATION
@@ -137,6 +139,7 @@
     DeployHypervisorApplicationStep,
     DestroyHypervisorApplicationStep,
     ReapplyHypervisorTerraformPlanStep,
+    RemoveHypervisorReferencesStep,
     RemoveHypervisorUnitStep,
 )
 from sunbeam.steps.juju import (
@@ -172,6 +175,7 @@
     CheckMicrocephDistributionStep,
     DeployMicrocephApplicationStep,
     DestroyMicrocephApplicationStep,
+    RemoveMicrocephOSDsStep,
     RemoveMicrocephUnitsStep,
     SetCephMgrPoolSizeStep,
 )
@@ -1693,6 +1697,9 @@ def remove_node(ctx: click.Context, name: str, force: bool, show_hints: bool) ->
 
     run_plan(check_plan, console, show_hints)
 
+    maas_client = MaasClient.from_deployment(deployment)
+    machine = get_machine(maas_client, name)
+
     plan = [
         MigrateK8SKubeconfigStep(
             client, name, jhelper, deployment.openstack_machines_model
@@ -1701,20 +1708,49 @@ def remove_node(ctx: click.Context, name: str, force: bool, show_hints: bool) ->
         RemoveHypervisorUnitStep(
             client,
             jhelper,
-            deployment,
+            None,
             name,
             deployment.openstack_machines_model,
             force,
         ),
+        DisableCinderVolumeServicesStep(
+            jhelper,
+            deployment,
+            machine["hostname"],
+            machine["fqdn"],
+            force=force,
+        ),
         RemoveCinderVolumeUnitsStep(
             client, name, jhelper, deployment.openstack_machines_model
         ),
+        RemoveCinderVolumeServicesStep(
+            jhelper,
+            deployment,
+            machine["hostname"],
+            machine["fqdn"],
+            force=force,
+        ),
+        RemoveMicrocephOSDsStep(
+            client,
+            name,
+            jhelper,
+            deployment.openstack_machines_model,
+            force=force,
+            hostnames=(machine["hostname"], machine["fqdn"]),
+        ),
         RemoveMicrocephUnitsStep(
             client, name, jhelper, deployment.openstack_machines_model
         ),
         RemoveMicroOVNUnitsStep(
             client, name, jhelper, deployment.openstack_machines_model
         ),
+        RemoveHypervisorReferencesStep(
+            jhelper,
+            deployment,
+            machine["hostname"],
+            machine["fqdn"],
+            force=force,
+        ),
         CordonK8SUnitStep(client, name, jhelper, deployment.openstack_machines_model),
         DrainK8SUnitStep(
             client, name, jhelper, deployment.openstack_machines_model, remove_pvc=True