Dpdk: roll back broken installs and reuse downloaded assets

A failure part way through a source installation left the node with a
half installed dpdk or rdma-core, and every later test on that node
failed for an unrelated reason. Wrap the install steps so a failure
uninstalls what was applied, removes the extracted source, and marks
the node dirty if even the cleanup fails, then re-raises the original
error.

The asset removal guards that were specific to the rdma-core installer
now live on the base Installer as _delete_assets, so every installer
gets the same protection against deleting '/' or the working path.

Downloads and extraction are also skipped when the asset is already on
the node, and dpdk-stable is fetched from the github mirror, which is
far more reliable than dpdk.org from Azure.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
Copilot-Session: 5d5f58ad-b9df-4420-ad37-22caee78e925

Author: mcgov

lisa/microsoft/testsuites/dpdk/common.py

@@ -17,7 +17,7 @@
 from lisa.tools.lscpu import CpuArchitecture
 from lisa.util import UnsupportedDistroException
 
-DPDK_STABLE_GIT_REPO = "https://dpdk.org/git/dpdk-stable"
+DPDK_STABLE_GIT_REPO = "https://github.com/dpdk/dpdk-stable.git"
 
 # azure routing table magic subnet prefix
 # signals 'route all traffic on this subnet'
@@ -150,6 +150,7 @@ def download(self) -> PurePath:
                 self._tar_url,
                 overwrite=False,
                 file_path=str(work_path),
+                skip_exists=True,
             )
             remote_path = node.get_pure_path(tarfile)
             self.tar_filename = remote_path.name
@@ -167,12 +168,13 @@ def download(self) -> PurePath:
         # force name as tarfile name
         # add option to skip files which already exist on disk
         # in the event we have already extracted this specific tar
-        node.tools[Tar].extract(
-            file=str(remote_path),
-            dest_dir=str(work_path),
-            gzip=True,
-            skip_existing_files=True,
-        )
+        if not node.shell.exists(self.asset_path):
+            node.tools[Tar].extract(
+                file=str(remote_path),
+                dest_dir=str(work_path),
+                gzip=True,
+                skip_existing_files=True,
+            )
         return self.asset_path
 
 
@@ -186,7 +188,8 @@ class Installer:
     # First we download the assets to ensure asset_path is set
     # even if we end up skipping re-installation
     def _setup_node(self) -> None:
-        self._download_assets()
+        if not hasattr(self, "asset_path"):
+            self._download_assets()
 
     # check if the package is already installed:
     # Is the package installed from source? Or from the package manager?
@@ -209,6 +212,40 @@ def _install(self) -> None:
     def _uninstall(self) -> None:
         raise NotImplementedError(f"_clean_previous_installation {self._err_msg}")
 
+    def _asset_path_exists(self) -> bool:
+        return hasattr(self, "asset_path") and self._node.shell.exists(self.asset_path)
+
+    def _delete_assets(self) -> None:
+        if not self._asset_path_exists():
+            return
+        asset_path = self.asset_path
+        delattr(self, "asset_path")
+        working_path = str(self._node.get_working_path())
+        assert_that(str(asset_path)).described_as(
+            "Test bug: Installer source path was empty during attempted cleanup!"
+        ).is_not_empty()
+        assert_that(str(asset_path)).described_as(
+            "Test bug: Installer source path was set to root dir '/' "
+            "during attempted cleanup!"
+        ).is_not_equal_to("/")
+        assert_that(str(asset_path)).described_as(
+            f"Test bug: Installer source path {asset_path} was set to working path "
+            f"'{working_path}' during attempted cleanup!"
+        ).is_not_equal_to(working_path)
+        self._node.execute(f"rm -rf {str(asset_path)}", shell=True)
+
+    def _rollback_installation(self) -> None:
+        try:
+            if self._check_if_installed():
+                self._uninstall()
+            self._delete_assets()
+        except Exception as err:
+            self._node.log.debug(
+                f"Installer cleanup failed; marking node dirty. {str(err)}"
+            )
+            self._node.mark_dirty()
+            raise err
+
     # install the dependencies
     def _install_dependencies(self) -> None:
         if self._os_dependencies is not None:
@@ -230,9 +267,20 @@ def _should_install(self, required_version: Optional[VersionInfo] = None) -> boo
     def do_installation(self, required_version: Optional[VersionInfo] = None) -> None:
         self._setup_node()
         if self._should_install():
-            self._uninstall()
-            self._install_dependencies()
-            self._install()
+            # any issues here could result in a broken installation.
+            # If the node is still usable, we don't want to discard it.
+            # So attempt to roll back a broken installation and re-raise the problem.
+            # This avoids re-deployments and ensures a transient issue causing a broken
+            # installation doesn't propagate failures into future tests on
+            # that same node.
+            try:
+                self._download_assets()
+                self._uninstall()
+                self._install_dependencies()
+                self._install()
+            except Exception as e:
+                self._rollback_installation()
+                raise e
 
     def __init__(
         self,
@@ -289,7 +337,7 @@ def force_dpdk_default_source(variables: Dict[str, Any]) -> None:
         variables["dpdk_source"] = DPDK_STABLE_GIT_REPO
 
 
-_UBUNTU_LTS_VERSIONS = ["24.4.0", "22.4.0", "20.4.0", "18.4.0"]
+_UBUNTU_LTS_VERSIONS = ["26.4.0", "24.4.0", "22.4.0", "20.4.0", "18.4.0"]
 
 
 # see https://ubuntu.com/about/release-cycle

lisa/microsoft/testsuites/dpdk/rdmacore.py

@@ -1,4 +1,3 @@
-from assertpy import assert_that
 from microsoft.testsuites.dpdk.common import (
     DependencyInstaller,
     Installer,
@@ -14,7 +13,7 @@
 
 RDMA_CORE_MANA_DEFAULT_SOURCE = (
     "https://github.com/linux-rdma/rdma-core/"
-    "releases/download/v50.1/rdma-core-50.1.tar.gz"
+    "releases/download/v59.0/rdma-core-59.0.tar.gz"
 )
 RDMA_CORE_SOURCE_DEPENDENCIES = DependencyInstaller(
     [
@@ -166,20 +165,6 @@ def _uninstall(self) -> None:
         self._node.tools[Make].run(
             parameters="uninstall", shell=True, sudo=True, cwd=self.asset_path
         )
-        working_path = str(self._node.get_working_path())
-        assert_that(str(self.asset_path)).described_as(
-            "RDMA Installer source path was empty during attempted cleanup!"
-        ).is_not_empty()
-        assert_that(str(self.asset_path)).described_as(
-            "RDMA Installer source path was set to root dir "
-            "'/' during attempted cleanup!"
-        ).is_not_equal_to("/")
-        assert_that(str(self.asset_path)).described_as(
-            f"RDMA Installer source path {self.asset_path} was set to "
-            f"working path '{working_path}' during attempted cleanup!"
-        ).is_not_equal_to(working_path)
-        # remove source code directory
-        self._node.execute(f"rm -rf {str(self.asset_path)}", shell=True)
 
     def get_installed_version(self) -> VersionInfo:
         version: VersionInfo = self._node.tools[Pkgconfig].get_package_version(