[KT] Fix VM ssh auth and CentOS7 content-release

kt/commands/content_release/impl.py

@@ -6,6 +6,7 @@
 
 from kt.ktlib.config import Config
 from kt.ktlib.kernel_workspace import KernelWorkspace
+from kt.ktlib.kernels import KernelsInfo
 from kt.ktlib.local import LocalCommand
 from kt.ktlib.mock import Mock
 from kt.ktlib.ssh import SshCommand
@@ -302,12 +303,19 @@ def test(cls, kernel_workspace: str):
         # Load kernel workspace
         kernel_workspace_obj = KernelWorkspace.load_from_name(kernel_workspace)
 
+        # Determine package manager based on OS variant
+        config = Config.load()
+        kernels_info = KernelsInfo.from_yaml(config)
+        kernel_name = Vm._extract_kernel_name(kernel_workspace)
+        kernel_info = kernels_info.kernels.get(kernel_name)
+        pkg_mgr = "yum" if (kernel_info and (kernel_info.os_variant or "").startswith("centos7")) else "dnf"
+
         # Setup and spin up the VM (reuses common code from vm command)
         vm_instance = Vm.setup_and_spinup(kernel_workspace_name=kernel_workspace)
 
-        # Wait for dependencies to be installed if VM was just created
+        # Wait for cloud-init to finish (may reboot the VM, e.g. CentOS 7)
         logging.info("Waiting for VM dependencies to be installed...")
-        SshCommand.run(domain=vm_instance.domain, command=["sudo cloud-init status --wait || true"])
+        vm_instance.wait_for_cloud_init()
 
         # Install the built RPMs
         build_files_dir = kernel_workspace_obj.folder / "build_files"
@@ -331,17 +339,22 @@ def test(cls, kernel_workspace: str):
                     command=[
                         'sudo dnf install -y "https://depot.ciq.com/public/files/depot-client/depot/depot.x86_64.rpm"'
                     ],
+                    ssh_key=vm_instance.ssh_key,
                 )
                 logging.info("Depot client installed")
 
                 # Register depot with credentials
                 SshCommand.run(
-                    domain=vm_instance.domain, command=[f"sudo depot register -u {depot_user} -t {depot_token}"]
+                    domain=vm_instance.domain,
+                    command=[f"sudo depot register -u {depot_user} -t {depot_token}"],
+                    ssh_key=vm_instance.ssh_key,
                 )
                 logging.info("Depot registered")
 
                 # Enable fips-legacy-8
-                SshCommand.run(domain=vm_instance.domain, command=["sudo depot enable fips-legacy-8"])
+                SshCommand.run(
+                    domain=vm_instance.domain, command=["sudo depot enable fips-legacy-8"], ssh_key=vm_instance.ssh_key
+                )
                 logging.info("fips-legacy-8 enabled via depot")
             except RuntimeError as e:
                 logging.error(f"Failed to enable depot for fipslegacy-8.6: {e}")
@@ -361,18 +374,22 @@ def test(cls, kernel_workspace: str):
             raise RuntimeError(f"No installable RPMs found in {build_files_dir}")
 
         rpm_paths = " ".join(str(rpm.absolute()) for rpm in install_rpms)
-        # Remove libtraceevent first to avoid file conflicts with perf package
-        install_cmd = (
-            f"sudo dnf remove -y libtraceevent || true && sudo dnf install --skip-broken --allowerasing {rpm_paths} -y"
-        )
-        # install_cmd = f"sudo dnf clean all && sudo dnf install --skip-broken --allowerasing {rpm_paths} -y"
+        if pkg_mgr == "yum":
+            install_cmd = f"sudo yum install -y {rpm_paths}"
+        else:
+            install_cmd = (
+                f"sudo dnf remove -y libtraceevent || true && "
+                f"sudo dnf install --skip-broken --allowerasing {rpm_paths} -y"
+            )
 
         install_log = kernel_workspace_obj.folder.absolute() / "install.log"
         logging.info(f"Installing {len(install_rpms)} RPM(s)")
         logging.info(f"RPM install output will be written to {install_log}")
 
         try:
-            SshCommand.run_with_output(output_file=install_log, domain=vm_instance.domain, command=[install_cmd])
+            SshCommand.run_with_output(
+                output_file=install_log, domain=vm_instance.domain, command=[install_cmd], ssh_key=vm_instance.ssh_key
+            )
             logging.info("RPMs installed successfully")
         except RuntimeError as e:
             logging.error(f"RPM installation failed: {e}")
@@ -407,7 +424,7 @@ def test(cls, kernel_workspace: str):
             "fi'"
         )
         try:
-            SshCommand.run(domain=vm_instance.domain, command=[grubenv_fix_cmd])
+            SshCommand.run(domain=vm_instance.domain, command=[grubenv_fix_cmd], ssh_key=vm_instance.ssh_key)
             logging.info("grubenv symlink fixed if needed")
         except RuntimeError as e:
             logging.error(f"Failed to fix grubenv symlink: {e}")
@@ -419,7 +436,7 @@ def test(cls, kernel_workspace: str):
             kernel_path = f"/boot/vmlinuz-{expected_version}"
             set_default_cmd = f"sudo grubby --set-default={kernel_path}"
             try:
-                SshCommand.run(domain=vm_instance.domain, command=[set_default_cmd])
+                SshCommand.run(domain=vm_instance.domain, command=[set_default_cmd], ssh_key=vm_instance.ssh_key)
                 logging.info("Default boot kernel set successfully")
             except RuntimeError as e:
                 logging.error(f"Failed to set default boot kernel: {e}")
@@ -438,11 +455,12 @@ def test(cls, kernel_workspace: str):
             raise RuntimeError(f"Kernel version mismatch! Expected: {expected_version}, Running: {kernel_version}")
         logging.info("Verified VM is running the newly installed kernel")
 
-        # Run kselftests using the installed kselftests
-        kselftest_log = kernel_workspace_obj.folder.absolute() / f"selftest-{kernel_version}.log"
-        vm_instance.kselftests_internal(kselftest_log)
-
-        # Count passed tests
-        vm_instance.count_kselftest_passed(kselftest_log)
+        # Run kselftests using the installed kselftests (not available on CentOS 7)
+        if pkg_mgr == "yum":
+            logging.info("Skipping kselftests (not available on CentOS 7)")
+        else:
+            kselftest_log = kernel_workspace_obj.folder.absolute() / f"selftest-{kernel_version}.log"
+            vm_instance.kselftests_internal(kselftest_log)
+            vm_instance.count_kselftest_passed(kselftest_log)
 
         logging.info("Test step completed successfully")

kt/commands/vm/impl.py

@@ -1,7 +1,6 @@
 import logging
 
 from kt.ktlib.config import Config
-from kt.ktlib.ssh import SshCommand
 from kt.ktlib.virt import VmCommand
 from kt.ktlib.vm import Vm
 
@@ -43,7 +42,7 @@ def main(
 
     if test:
         logging.info("Waiting for cloud-init to finish...")
-        SshCommand.run(domain=vm_instance.domain, command=["sudo cloud-init status --wait || true"])
+        vm_instance.wait_for_cloud_init()
         vm_instance.test(config=config)
 
     if console:

kt/data/cloud_init_centos7.yaml

@@ -15,6 +15,10 @@ ssh_pwauth: true
 # Ensure the system does not update on boot.
 package_upgrade: False
 
+# Create mount point before mounts module runs
+bootcmd:
+  - mkdir -p SHARED_DIR_PLACEHOLDER
+
 mounts:
   - - NFS_SOURCE_PLACEHOLDER
     - SHARED_DIR_PLACEHOLDER

kt/ktlib/ssh.py

@@ -3,12 +3,14 @@
 
 class SshCommand(CommandRunner):
     COMMAND = "ssh"
-    EXTRA = "-o StrictHostKeyChecking=no"
 
     @classmethod
-    def _build_command(cls, domain: str, command: list[str]) -> list[str]:
-        return [cls.COMMAND, cls.EXTRA, domain] + command
+    def _build_command(cls, domain: str, command: list[str], ssh_key: str | None = None) -> list[str]:
+        cmd = [cls.COMMAND, "-o", "StrictHostKeyChecking=no", "-o", "BatchMode=yes"]
+        if ssh_key:
+            cmd += ["-i", ssh_key]
+        return cmd + [domain] + command
 
     @classmethod
-    def running_kernel_version(cls, domain):
-        return cls.run(domain=domain, command=["uname", "-r"])
+    def running_kernel_version(cls, domain, ssh_key: str | None = None):
+        return cls.run(domain=domain, command=["uname", "-r"], ssh_key=ssh_key)

kt/ktlib/virt.py

@@ -1,3 +1,6 @@
+import logging
+import re
+import time
 from enum import Enum
 
 from pathlib3x import Path
@@ -107,11 +110,23 @@ def domifaddr(cls, vm_name: str) -> list[str]:
 
 
 class VirtHelper:
+    IP_PATTERN = re.compile(r"\d+\.\d+\.\d+\.\d+")
+    IP_POLL_INTERVAL = 5
+    IP_POLL_MAX_ATTEMPTS = 24
+
     @classmethod
     def ip_addr(cls, vm_name: str) -> str:
-        rc = VmCommand.domifaddr(vm_name=vm_name)
-
-        return rc[-1].split("/")[0]
+        for attempt in range(cls.IP_POLL_MAX_ATTEMPTS):
+            output = VmCommand.domifaddr(vm_name=vm_name)
+            for token in output:
+                match = cls.IP_PATTERN.search(token)
+                if match:
+                    return match.group()
+            logging.info(f"Waiting for IP address for {vm_name} (attempt {attempt + 1}/{cls.IP_POLL_MAX_ATTEMPTS})...")
+            time.sleep(cls.IP_POLL_INTERVAL)
+        raise RuntimeError(
+            f"VM {vm_name} did not get an IP address after {cls.IP_POLL_MAX_ATTEMPTS * cls.IP_POLL_INTERVAL}s"
+        )
 
     @classmethod
     def is_running(cls, vm_name: str) -> bool:

kt/ktlib/vm.py

@@ -232,6 +232,7 @@ def _setup_cloud_init(self, config: Config, no_depot: bool = False):
         base_path_str = str(config.base_path.absolute())
         if self._is_centos7():
             nfs_source = f"{Constants.LIBVIRT_HOST_IP}:{base_path_str}"
+            data["bootcmd"][0] = f"mkdir -p {base_path_str}"
             data["mounts"][0][0] = nfs_source
             data["mounts"][0][1] = base_path_str
             data["mounts"][1][0] = base_path_str
@@ -351,19 +352,19 @@ def spin_up(self, config: Config, vcpus: int = 12, memory: int = 32768, no_depot
 
             self._create_image(config=config, vcpus=vcpus, memory=memory, no_depot=no_depot)
             self._wait_for_running()
-            return VmInstance(name=self.name, kernel_workspace=self.kernel_workspace)
+            return VmInstance(name=self.name, kernel_workspace=self.kernel_workspace, config=config)
 
         logging.info(f"Vm {self.name} already exists")
 
         if VirtHelper.is_running(vm_name=self.name):
             logging.info(f"Vm {self.name} is running, nothing to do")
-            return VmInstance(name=self.name, kernel_workspace=self.kernel_workspace)
+            return VmInstance(name=self.name, kernel_workspace=self.kernel_workspace, config=config)
 
         logging.info(f"Vm {self.name} is not running, starting it")
         VmCommand.start(vm_name=self.name)
         self._wait_for_running()
 
-        return VmInstance(name=self.name, kernel_workspace=self.kernel_workspace)
+        return VmInstance(name=self.name, kernel_workspace=self.kernel_workspace, config=config)
 
     def destroy(self):
         if VirtHelper.is_running(vm_name=self.name):
@@ -381,19 +382,52 @@ class VmInstance:
     ssh_domain: str
     kernel_workspace: KernelWorkspace
 
-    def __init__(self, name: str, kernel_workspace: KernelWorkspace):
+    def __init__(self, name: str, kernel_workspace: KernelWorkspace, config: Config):
         self.name = name
         ip_addr = VirtHelper.ip_addr(vm_name=self.name)
-        username = os.environ["USER"]
-        self.domain = f"{username}@{ip_addr}"
+        self.domain = f"{config.user}@{ip_addr}"
         self.kernel_workspace = kernel_workspace
+        ssh_pub = str(config.ssh_key)
+        self.ssh_key = ssh_pub.removesuffix(".pub") if ssh_pub.endswith(".pub") else ssh_pub
+        self._wait_for_ssh()
+
+    def _wait_for_ssh(self):
+        for attempt in range(Constants.VM_POLL_MAX_ATTEMPTS):
+            try:
+                SshCommand.run(domain=self.domain, command=["true"], ssh_key=self.ssh_key)
+                logging.info(f"SSH connection to {self.domain} established")
+                return
+            except RuntimeError:
+                logging.info(
+                    f"Waiting for SSH on {self.domain} (attempt {attempt + 1}/{Constants.VM_POLL_MAX_ATTEMPTS})..."
+                )
+                time.sleep(Constants.VM_POLL_INTERVAL_SECONDS)
+        raise RuntimeError(
+            f"SSH to {self.domain} not available after "
+            f"{Constants.VM_POLL_MAX_ATTEMPTS * Constants.VM_POLL_INTERVAL_SECONDS}s"
+        )
+
+    def wait_for_cloud_init(self):
+        """ "Wait for cloud-init to finish on the VM. This method will block until cloud-init has completed its tasks."""
+        try:
+            SshCommand.run(
+                domain=self.domain,
+                command=["sudo cloud-init status --wait || true"],
+                ssh_key=self.ssh_key,
+            )
+        except RuntimeError as e:
+            if "closed by remote host" in str(e):
+                logging.info("VM rebooted during cloud-init, waiting for it to come back...")
+                self._wait_for_ssh()
+            else:
+                raise
 
     def reboot(self):
         logging.debug("Rebooting vm")
 
         command = ["sudo", "reboot"]
         try:
-            SshCommand.run(domain=self.domain, command=command)
+            SshCommand.run(domain=self.domain, command=command, ssh_key=self.ssh_key)
         except RuntimeError as e:
             if "closed by remote host" in str(e):
                 pass
@@ -416,7 +450,7 @@ def kselftests(self, config):
         output_file = self.kernel_workspace.folder.absolute() / Path(f"kselftest-{self.current_head_sha_short()}.log")
         ssh_cmd = f"cd {self.kernel_workspace.src_worktree.folder.absolute()} &&  sudo {script}"
 
-        SshCommand.run_with_output(output_file=output_file, domain=self.domain, command=[ssh_cmd])
+        SshCommand.run_with_output(output_file=output_file, domain=self.domain, command=[ssh_cmd], ssh_key=self.ssh_key)
 
     def kselftests_internal(self, output_file: Path):
         """
@@ -434,7 +468,9 @@ def kselftests_internal(self, output_file: Path):
         kselftest_cmd = "sudo /usr/libexec/kselftests/run_kselftest.sh"
 
         try:
-            SshCommand.run_with_output(output_file=output_file, domain=self.domain, command=[kselftest_cmd])
+            SshCommand.run_with_output(
+                output_file=output_file, domain=self.domain, command=[kselftest_cmd], ssh_key=self.ssh_key
+            )
             logging.info("Kselftests completed successfully")
         except RuntimeError as e:
             logging.error(f"Kselftests failed: {e}")
@@ -468,7 +504,7 @@ def running_kernel_version(self):
         Returns:
             str: The kernel version string (e.g., "5.14.0-284.30.1+23.1.el9_2_ciq.x86_64")
         """
-        return SshCommand.running_kernel_version(domain=self.domain)
+        return SshCommand.running_kernel_version(domain=self.domain, ssh_key=self.ssh_key)
 
     def expected_kernel_version(self):
         """
@@ -499,7 +535,7 @@ def build_kernel(self, config):
         )
         ssh_cmd = f"cd {self.kernel_workspace.src_worktree.folder.absolute()} &&  {build_script} -n"
 
-        SshCommand.run_with_output(output_file=output_file, domain=self.domain, command=[ssh_cmd])
+        SshCommand.run_with_output(output_file=output_file, domain=self.domain, command=[ssh_cmd], ssh_key=self.ssh_key)
 
     def test(self, config):
         if self.expected_kernel_version():