Support base image deletion

CHANGELOG.md

@@ -19,9 +19,16 @@ and this project adheres to
   - Add support of slurm-quota (#52).
 - cli: Add `deploy --update-os-image` option to force download of base OS image
   when already present on host.
+- cli: Add `clean --remove-base-image` option to remove base OS image on cluster
+  cleanup.
+- cli: Add `images --remove` option to remove a cached base OS image.
 - load: Add random yet realistic job names.
 - lib: Add `deploy --update-os-image` option in bash-completion.
+- lib: Add `clean --remove-base-image` and `images --remove` options in
+  bash-completion.
 - docs: Mention `deploy --update-os-image` option in manpage.
+- docs: Mention `clean --remove-base-image` and `images --remove` options in
+  manpage.
 
 ### Changed
 - conf:

README.md

@@ -248,6 +248,19 @@ When you are done, you can clean up everything for a cluster with this command:
 $ firehpc clean --cluster hpc
 ```
 
+To also remove the cached base OS image (when no other cluster uses the same
+OS):
+
+```
+$ firehpc clean --cluster hpc --remove-base-image
+```
+
+You can also remove a base OS image explicitly:
+
+```
+$ firehpc images --remove debian13
+```
+
 ## Authors
 
 FireHPC is developed by [Rackslab](https://rackslab.io).

docs/man/firehpc.adoc

@@ -78,6 +78,10 @@ This command accepts the following options:
 
 [.cli-opt]#*--cluster*=#[.cli-optval]##_CLUSTER_##::
   Name of the cluster to clean. This option is required.
+
+[.cli-opt]#*--remove-base-image*#::
+  Also remove the cached base OS image used by the cluster. The base image is
+  not removed if another cluster still uses the same OS.
 --
 
 [.cli-opt]#*conf*#::
@@ -183,7 +187,17 @@ settings file.
 [.cli-opt]#*images*#::
 
   List available operating systems and the URL of the corresponding container
-  images to download.
+  images to download. With [.cli-opt]#*--remove*# option, remove a cached base
+  OS image instead.
++
+--
+This command accepts the following options:
+
+[.cli-opt]#*--remove*=#[.cli-optval]##_OS_##::
+  Remove a cached base OS image. The base image is not removed if a cluster
+  still uses this OS. Run `firehpc images` without option to get the list of
+  supported OS.
+--
 
 [.cli-opt]#*list*#::
 

firehpc/cluster.py

@@ -17,10 +17,11 @@
 
 from .templates import Templater
 from .users import UsersDirectory
-from .containers import ContainersManager
+from .containers import BaseImagesManager, ContainersManager
 from .errors import FireHPCRuntimeError
+from .os import OSDatabase
 from .settings import ClusterSettings
-from .state import ClusterState, UserState
+from .state import ClusterState, UserState, os_used_by_clusters
 from .environments import DeploymentEnvironment
 
 if TYPE_CHECKING:
@@ -31,9 +32,19 @@
 logger = logging.getLogger(__name__)
 
 
-def clusters_list(state: Path):
-    """Return list of cluster names present in state directory."""
-    return [path.name for path in UserState(state).clusters.glob("*")]
+def remove_os_base_image(
+    runtime_settings: RuntimeSettings,
+    user_state: UserState,
+    os_key: str,
+    exclude_cluster: Optional[str] = None,
+) -> None:
+    if os_used_by_clusters(user_state, os_key, exclude=exclude_cluster):
+        logger.info(
+            "Base image for OS %s is still used by another cluster, skipping removal",
+            os_key,
+        )
+        return
+    BaseImagesManager(OSDatabase(runtime_settings)).remove(os_key)
 
 
 @dataclass
@@ -80,36 +91,30 @@ def users_directory(self) -> UsersDirectory:
 
     def deploy(
         self,
-        url: str,
+        os: str,
         update_os_image: bool,
         db: RacksDB,
     ) -> None:
         infrastructure = db.infrastructures[self.name]
 
         manager = ContainersManager(self.name)
-
-        base_image_name = os.path.basename(url).split(".")[0]
+        base_images = BaseImagesManager(OSDatabase(self.runtime_settings))
 
         # Check if base image is already present. If not or update_os_image is
         # True, download it. Otherwise, just use it in place.
-        if not manager.image_exists(base_image_name):
-            logger.info("Base image %s must be imported", base_image_name)
-            base_image = manager.download(
-                url,
-                base_image_name,
-            )
+        if not base_images.exists(os):
+            logger.info("Base image %s must be imported", base_images.image_name(os))
+            base_image = base_images.download(os)
         else:
-            logger.info("Base image %s is already imported", base_image_name)
-            base_image = manager.base_image(base_image_name)
+            logger.info("Base image %s is already imported", base_images.image_name(os))
+            base_image = base_images.get(os)
             if update_os_image:
                 logger.info(
-                    "Base image %s must be updated, removing it", base_image_name
+                    "Base image %s must be updated, removing it",
+                    base_images.image_name(os),
                 )
                 base_image.remove()
-                base_image = manager.download(
-                    url,
-                    base_image_name,
-                )
+                base_image = base_images.download(os)
 
         for node in infrastructure.nodes:
             if "admin" in node.tags or not self.cluster_settings.slurm_emulator:
@@ -303,7 +308,15 @@ def insert_in_node_type():
             logger.debug("Removing ansible generated directory %s", generated_path)
             shutil.rmtree(generated_path)
 
-    def clean(self) -> None:
+    def clean(self, remove_base_image: bool = False) -> None:
+        os_key = None
+        if remove_base_image:
+            if self.cluster_settings is None:
+                raise FireHPCRuntimeError(
+                    "Cluster settings are required to remove base image"
+                )
+            os_key = self.cluster_settings.os
+
         manager = ContainersManager(self.name)
 
         manager.stop()
@@ -318,6 +331,14 @@ def clean(self) -> None:
         # Remove cluster state directory
         self.state.clean()
 
+        if remove_base_image:
+            remove_os_base_image(
+                self.runtime_settings,
+                self.state.user_state,
+                os_key,
+                exclude_cluster=self.name,
+            )
+
     def start(self) -> None:
         manager = ContainersManager(self.name)
 

firehpc/containers.py

@@ -19,6 +19,7 @@
 from dasbus.error import DBusError
 
 from .errors import FireHPCRuntimeError
+from .os import OSDatabase
 
 logger = logging.getLogger(__name__)
 
@@ -188,6 +189,38 @@ class BaseImage(Image):
     """Cluster base image"""
 
 
+class BaseImagesManager(DBusObject):
+    INTERFACE = "org.freedesktop.machine1"
+
+    def __init__(self, os_db: OSDatabase) -> None:
+        super().__init__("/org/freedesktop/machine1")
+        self.os_db = os_db
+
+    def exists(self, os: str) -> bool:
+        return self.image_name(os) in [image[0] for image in self.proxy.ListImages()]
+
+    def get(self, os: str) -> BaseImage:
+        return BaseImage.from_machine_image_path(
+            self.proxy.GetImage(self.image_name(os))
+        )
+
+    def download(self, os: str) -> BaseImage:
+        name = self.image_name(os)
+        ImageImporter(self.os_db.url(os), name).transfer()
+        return BaseImage.from_machine_image_path(self.proxy.GetImage(name))
+
+    def remove(self, os: str) -> None:
+        name = self.image_name(os)
+        if not self.exists(os):
+            logger.info("Base image %s is not present, nothing to remove", name)
+            return
+        logger.info("Removing base image %s", name)
+        self.get(os).remove()
+
+    def image_name(self, os: str) -> str:
+        return self.os_db.image_name(os)
+
+
 class ClusterStateModifier(DBusObject):
     INTERFACE = "org.freedesktop.machine1"
 
@@ -436,17 +469,6 @@ def cluster_images(self) -> list:
             if image[0].endswith(f".{self.cluster}.{self.namespace}")
         ]
 
-    def base_image(self, name) -> BaseImage:
-        return BaseImage.from_machine_image_path(self.proxy.GetImage(name))
-
-    def image_exists(self, name) -> bool:
-        return name in [image[0] for image in self.proxy.ListImages()]
-
-    def download(self, url: str, name: str) -> BaseImage:
-        """Download cluster base image"""
-        ImageImporter(url, name).transfer()
-        return BaseImage.from_machine_image_path(self.proxy.GetImage(name))
-
     def clone_base(self, base: BaseImage, node: str) -> None:
         base.clone(f"{node}.{self.cluster}.{self.namespace}")
 

firehpc/exec.py

@@ -14,8 +14,8 @@
 
 from .version import get_version
 from .settings import RuntimeSettings, ClusterSettings
-from .state import default_state_dir, UserState, ClusterState
-from .cluster import EmulatedCluster, clusters_list
+from .state import default_state_dir, UserState, ClusterState, clusters_list
+from .cluster import EmulatedCluster, remove_os_base_image
 from .environments import bootstrap
 from .ssh import SSHClient
 from .errors import FireHPCRuntimeError
@@ -214,6 +214,11 @@ def __init__(self):
             help="Name of the cluster to clean",
             required=True,
         )
+        parser_clean.add_argument(
+            "--remove-base-image",
+            help="Remove base OS image used by the cluster",
+            action="store_true",
+        )
         parser_clean.set_defaults(func=self._execute_clean)
 
         # start command
@@ -252,6 +257,11 @@ def __init__(self):
 
         # images command
         parser_images = subparsers.add_parser("images", help="List available OS images")
+        parser_images.add_argument(
+            "--remove",
+            help="Remove a cached base OS image",
+            metavar="OS",
+        )
         parser_images.set_defaults(func=self._execute_images)
 
         # list command
@@ -387,7 +397,7 @@ def _execute_deploy(self):
             ).users_directory
 
         # Deploy cluster
-        cluster.deploy(os_db.url(self.args.os), self.args.update_os_image, db)
+        cluster.deploy(self.args.os, self.args.update_os_image, db)
         cluster.conf(
             db,
             playbooks=["bootstrap", "site"],
@@ -471,11 +481,15 @@ def _execute_ssh(self):
         ssh.exec(self.args.args)
 
     def _execute_clean(self):
-        # Load cluster settings
         state = ClusterState(self.user_state, self.args.cluster)
+        cluster_settings = None
+        if self.args.remove_base_image:
+            cluster_settings = state.load()
 
-        cluster = EmulatedCluster(self.runtime_settings, self.args.cluster, state)
-        cluster.clean()
+        cluster = EmulatedCluster(
+            self.runtime_settings, self.args.cluster, state, cluster_settings
+        )
+        cluster.clean(remove_base_image=self.args.remove_base_image)
 
     def _execute_status(self):
         # Load cluster settings
@@ -493,7 +507,17 @@ def _execute_status(self):
 
     def _execute_images(self):
         os_db = OSDatabase(self.runtime_settings)
-        print(str(os_db), end="")
+        if self.args.remove is not None:
+            if not os_db.supported(self.args.remove):
+                raise FireHPCRuntimeError(
+                    f"OS {self.args.remove} is not supported, run `firehpc images` to "
+                    "get the list of supported OS"
+                )
+            remove_os_base_image(
+                self.runtime_settings, self.user_state, self.args.remove
+            )
+        else:
+            print(str(os_db), end="")
 
     def _execute_list(self):
         print("\n".join(clusters_list(self.args.state)))

firehpc/os.py

@@ -9,6 +9,7 @@
 from __future__ import annotations
 from typing import TYPE_CHECKING
 import logging
+from os.path import basename
 
 import yaml
 
@@ -33,6 +34,9 @@ def url(self, os: str) -> str:
     def environment(self, os: str) -> str:
         return self.content[os]["environment"]
 
+    def image_name(self, os: str) -> str:
+        return basename(self.url(os)).split(".")[0]
+
     def __str__(self):
         result = ""
         for os, value in self.content.items():

firehpc/state.py

@@ -7,6 +7,7 @@
 import os
 import dataclasses
 from pathlib import Path
+from typing import Optional
 import shutil
 import logging
 
@@ -108,3 +109,20 @@ def load(self):
             raise FireHPCRuntimeError(
                 f"Unable to load cluster settings: {err}"
             ) from err
+
+
+def clusters_list(state: Path):
+    """Return list of cluster names present in state directory."""
+    return [path.name for path in UserState(state).clusters.glob("*")]
+
+
+def os_used_by_clusters(
+    user_state: UserState, os_key: str, exclude: Optional[str] = None
+) -> bool:
+    for cluster in clusters_list(user_state.path):
+        if cluster == exclude:
+            continue
+        settings = ClusterState(user_state, cluster).load()
+        if settings.os == os_key:
+            return True
+    return False