tests: fix snapshot tests for older versions

Previously, the snapshot tests that were supposed to retrieve
the older Firecracker binaries from S3 were not filtering the artifacts
by version number.

This results in test failures for patch releases, where the test
will use both older and newer binaries.

Added support in the testing framework for this and fixed the
respective tests.

Signed-off-by: alindima <[email protected]>

Author: alindima

tests/framework/artifacts.py

@@ -12,6 +12,7 @@
 import boto3
 import botocore.client
 from host_tools.snapshot_helper import merge_memory_bitmaps
+from framework.utils import compare_release_versions
 
 
 ARTIFACTS_LOCAL_ROOT = "/tmp/ci-artifacts"
@@ -316,9 +317,9 @@ def _fetch_artifacts(self,
                 # Filter by userprovided keyword if any.
                 and (keyword is None or keyword in file.key)
             ):
-                artifacts.append(artifact_class(self.bucket,
-                                                file.key,
-                                                artifact_type=artifact_type))
+                artifacts.append(artifact_class(
+                    self.bucket, file.key, artifact_type=artifact_type)
+                )
         return artifacts
 
     def snapshots(self, keyword=None):
@@ -347,7 +348,7 @@ def snapshots(self, keyword=None):
             # Filter out the snapshot artifacts root folder.
             # Select only files with specified keyword.
             if (key[-1] == "/" and key != prefix and
-               (keyword is None or keyword in snapshot_dir.key)):
+                    (keyword is None or keyword in snapshot_dir.key)):
                 artifact_type = ArtifactType.SNAPSHOT
                 artifacts.append(SnapshotArtifact(self.bucket,
                                                   key,
@@ -365,16 +366,29 @@ def microvms(self, keyword=None):
             keyword=keyword
         )
 
-    def firecrackers(self, keyword=None):
+    def firecrackers(self, keyword=None, older_than=None):
         """Return fc/jailer artifacts for the current arch."""
-        return self._fetch_artifacts(
+        firecrackers = self._fetch_artifacts(
             ArtifactCollection.ARTIFACTS_BINARIES,
             ArtifactCollection.FC_EXTENSION,
             ArtifactType.FC,
             FirecrackerArtifact,
             keyword=keyword
         )
 
+        if older_than is not None:
+            return filter(
+                lambda fc:
+                    compare_release_versions(
+                        os.path.basename(fc.key).replace(
+                            ArtifactCollection.FC_EXTENSION, '')[1:],
+                        older_than
+                    ) < 0,
+                firecrackers
+            )
+
+        return firecrackers
+
     def kernels(self, keyword=None):
         """Return kernel artifacts for the current arch."""
         return self._fetch_artifacts(

tests/framework/utils.py

@@ -206,3 +206,44 @@ def get_cpu_vendor():
     if 'AuthenticAMD' in brand_str:
         return CpuVendor.AMD
     return CpuVendor.INTEL
+
+
+def get_firecracker_version_from_toml():
+    """
+    Return the version of the firecracker crate, from Cargo.toml.
+
+    Usually different from the output of `./firecracker --version`, if
+    the code has not been released.
+    """
+    cmd = "cd ../src/firecracker && cargo pkgid | cut -d# -f2 | cut -d: -f2"
+
+    rc, stdout, _ = run_cmd(cmd)
+    assert rc == 0
+
+    return stdout
+
+
+def compare_release_versions(first, second):
+    """
+    Compare two versions with format `X.Y.Z`.
+
+    :param first: first version string
+    :param second: second version string
+    :returns: 0 if equal, <0 if first < second, >0 if second < first
+    """
+    first = first.split('.')
+    second = second.split('.')
+
+    first = list(map(int, first))
+    second = list(map(int, second))
+
+    if first[0] == second[0]:
+        if first[1] == second[1]:
+            if first[2] == second[2]:
+                return 0
+
+            return first[2] - second[2]
+
+        return first[1] - second[1]
+
+    return first[0] - second[0]

tests/integration_tests/functional/test_snapshot_version.py

@@ -9,6 +9,7 @@
 from framework.artifacts import ArtifactCollection
 from framework.builder import MicrovmBuilder, SnapshotBuilder, SnapshotType
 from framework.microvms import C3micro
+from framework.utils import get_firecracker_version_from_toml
 import host_tools.network as net_tools  # pylint: disable=import-error
 
 
@@ -24,7 +25,8 @@ def test_restore_from_past_versions(bin_cloner_path):
     # Fetch all firecracker binaries.
     # With each binary create a snapshot and try to restore in current
     # version.
-    firecracker_artifacts = artifacts.firecrackers()
+    firecracker_artifacts = artifacts.firecrackers(
+        older_than=get_firecracker_version_from_toml())
     for firecracker in firecracker_artifacts:
         firecracker.download()
         jailer = firecracker.jailer()
@@ -90,12 +92,13 @@ def test_restore_in_past_versions(bin_cloner_path):
     logger = logging.getLogger("snapshot_version")
 
     artifacts = ArtifactCollection(_test_images_s3_bucket())
-    # Fetch all snapshots artifacts.
+    # Fetch all snapshots artifacts of past versions.
     # "fc_release" is the key that should be used for per release snapshot
     # artifacts. Such snapshots are created at release time and target the
     # current version. We are going to restore all these snapshots with current
     # testing build.
-    firecracker_artifacts = artifacts.firecrackers()
+    firecracker_artifacts = artifacts.firecrackers(
+        older_than=get_firecracker_version_from_toml())
     for firecracker in firecracker_artifacts:
         firecracker.download()
         jailer = firecracker.jailer()

tests/integration_tests/performance/test_snapshot_perf.py

@@ -12,7 +12,8 @@
 from framework.matrix import TestMatrix, TestContext
 from framework.microvms import C3micro
 from framework.builder import MicrovmBuilder, SnapshotBuilder, SnapshotType
-from framework.utils import CpuVendor, get_cpu_vendor
+from framework.utils import CpuVendor, get_cpu_vendor, \
+    get_firecracker_version_from_toml
 import host_tools.network as net_tools  # pylint: disable=import-error
 import host_tools.logging as log_tools
 
@@ -147,9 +148,9 @@ def _test_snapshot_resume_latency(context):
 
     for i in range(SAMPLE_COUNT):
         microvm, metrics_fifo = vm_builder.build_from_snapshot(
-                                                snapshot,
-                                                True,
-                                                enable_diff_snapshots)
+            snapshot,
+            True,
+            enable_diff_snapshots)
 
         # Attempt to connect to resumed microvm.
         ssh_connection = net_tools.SSHConnection(microvm.ssh_config)
@@ -267,7 +268,8 @@ def test_older_snapshot_resume_latency(bin_cloner_path):
     # Fetch all firecracker binaries.
     # With each binary create a snapshot and try to restore in current
     # version.
-    firecracker_artifacts = artifacts.firecrackers()
+    firecracker_artifacts = artifacts.firecrackers(
+        older_than=get_firecracker_version_from_toml())
     for firecracker in firecracker_artifacts:
         firecracker.download()
         jailer = firecracker.jailer()