smoke test: support ceph storage pools

test/integration/smoke/test_backup_recovery_nas.py

@@ -24,7 +24,11 @@
 from marvin.lib.common import (get_domain, get_zone, get_template)
 from nose.plugins.attrib import attr
 from marvin.codes import FAILED
+import os
 import time
+from urllib.parse import urlsplit
+
+SUPPORTED_PRIMARY_STORAGE_POOL_TYPES = ['networkfilesystem', 'rbd']
 
 class TestNASBackupAndRecovery(cloudstackTestCase):
 
@@ -39,19 +43,56 @@ def setUpClass(cls):
         cls.services["mode"] = cls.zone.networktype
         cls.hypervisor = cls.testClient.getHypervisorInfo()
         cls.domain = get_domain(cls.api_client)
+        cls._cleanup = []
+
+        if cls.hypervisor.lower() != 'kvm':
+            cls.skipTest(cls, reason="Test can be run only on KVM hypervisor")
+
         cls.template = get_template(cls.api_client, cls.zone.id, cls.services["ostype"])
         if cls.template == FAILED:
             assert False, "get_template() failed to return template with description %s" % cls.services["ostype"]
         cls.services["small"]["zoneid"] = cls.zone.id
         cls.services["small"]["template"] = cls.template.id
-        cls._cleanup = []
 
-        if cls.hypervisor.lower() != 'kvm':
-            cls.skipTest(cls, reason="Test can be run only on KVM hypervisor")
-
-        cls.storage_pool = StoragePool.list(cls.api_client)[0]
-        if cls.storage_pool.type.lower() != 'networkfilesystem':
-            cls.skipTest(cls, reason="Test can be run only if the primary storage is of type NFS")
+        # Pick a pool that's actually usable, not just list()[0] -- environments that
+        # added Ceph/RBD storage after the zone's original NFS primary storage keep the
+        # old NFS pool around in Disabled state, and it still sorts first. Falling back
+        # to index 0 there silently exercises the disabled NFS pool's path as if it were
+        # the primary storage in use, rather than the RBD pool VMs actually deploy on.
+        storage_pools = StoragePool.list(cls.api_client)
+        usable_pools = [p for p in storage_pools if getattr(p, 'state', 'Up') == 'Up']
+        cls.storage_pool = usable_pools[0] if usable_pools else storage_pools[0]
+        if cls.storage_pool.type.lower() not in SUPPORTED_PRIMARY_STORAGE_POOL_TYPES:
+            cls.skipTest(cls, reason="Test can be run only if the primary storage is of type NFS or RBD (Ceph)")
+
+        # The NAS backup repository needs an NFS export to mount. When the primary
+        # storage is itself NFS, its own path can double as that export (the
+        # historical behaviour). When the primary storage is Ceph/RBD, the primary
+        # storage location can't be reused as a NAS export, so fall back to the
+        # "nfs" test data entry -- the same NFS mount point test_primary_storage.py
+        # uses to create its temporary NFS primary storage pool, and something every
+        # marvin environment already has configured (services["nfs"]["url"], e.g.
+        # "nfs://nfs/export/automation/1/testprimary"). An explicit
+        # "nas_backup_repository_address" test data entry or NAS_BACKUP_REPO_ADDRESS
+        # environment variable, if set, takes precedence over both.
+        if cls.storage_pool.type.lower() == 'networkfilesystem':
+            default_nas_repository_address = cls.storage_pool.ipaddress + ":" + cls.storage_pool.path
+        else:
+            nfs_test_data = cls.services.get("nfs")
+            if nfs_test_data and nfs_test_data.get("url"):
+                nfs_url = urlsplit(nfs_test_data["url"])
+                default_nas_repository_address = "%s:%s" % (nfs_url.hostname, nfs_url.path)
+            else:
+                default_nas_repository_address = None
+        cls.nas_repository_address = cls.services.get("nas_backup_repository_address") \
+            or os.environ.get("NAS_BACKUP_REPO_ADDRESS") \
+            or default_nas_repository_address
+        if not cls.nas_repository_address:
+            cls.skipTest(cls, reason="No NAS backup repository export configured. Set "
+                                     "'nas_backup_repository_address' in the test data, the "
+                                     "NAS_BACKUP_REPO_ADDRESS environment variable, or the standard "
+                                     "'nfs' test data entry, when the primary storage is not NFS "
+                                     "(e.g. Ceph/RBD)")
 
         # Check backup configuration values, set them to enable the nas provider
         backup_enabled_cfg = Configurations.list(cls.api_client, name='backup.framework.enabled')
@@ -72,14 +113,22 @@ def setUpClass(cls):
 
         cls._cleanup = [cls.account]
 
-        # Create NAS backup repository and offering. Use the same directory as the storage pool
+        # Create NAS backup repository and offering.
         cls.backup_repository = BackupRepository.add(cls.api_client, zoneid=cls.zone.id, name="Nas",
-                                                     address=cls.storage_pool.ipaddress + ":" + cls.storage_pool.path,
+                                                     address=cls.nas_repository_address,
                                                      provider="nas", type="nfs",)
         cls._cleanup.append(cls.backup_repository)
+        # Match the external offering to the repository just created above by externalid
+        # (== the repository's own id for the nas provider) rather than blindly taking
+        # index 0 -- a stray repository left over from an earlier interrupted run (e.g.
+        # one whose backups didn't get cleaned up, so its own teardown couldn't remove
+        # it either) sorts alongside the new one, and index 0 has no guarantee of being
+        # the one this run owns.
         cls.provider_offerings = BackupOffering.listExternal(cls.api_client, cls.zone.id)
-        cls.backup_offering = BackupOffering.importExisting(cls.api_client, cls.zone.id, cls.provider_offerings[0].externalid,
-                                                            cls.provider_offerings[0].name, cls.provider_offerings[0].description)
+        matching_offerings = [o for o in cls.provider_offerings if o.externalid == cls.backup_repository.id]
+        provider_offering = matching_offerings[0] if matching_offerings else cls.provider_offerings[0]
+        cls.backup_offering = BackupOffering.importExisting(cls.api_client, cls.zone.id, provider_offering.externalid,
+                                                            provider_offering.name, provider_offering.description)
         cls._cleanup.append(cls.backup_offering)
 
         cls.offering = ServiceOffering.create(cls.api_client,cls.services["service_offerings"]["small"])
@@ -303,12 +352,26 @@ def _backup_type(self, backup):
         # Backup objects expose `type`; for chained backups it's "INCREMENTAL", else "FULL".
         return getattr(backup, 'type', 'FULL') or 'FULL'
 
+    def _require_incremental_capable_storage(self):
+        """
+        Incremental NAS backups rely on QEMU dirty bitmaps / libvirt checkpoints, which
+        only exist on file-based qcow2 storage -- see
+        NASBackupProvider.allVolumesOnCheckpointCapableStorage(), which forces every VM
+        on RBD/Ceph (or Linstor) onto the legacy full-only path server-side. On such
+        storage every backup comes back FULL regardless of cadence, so these chain/type
+        assertions can't pass (and some would pass vacuously without exercising the
+        chain logic at all). Skip rather than fail when running against RBD.
+        """
+        if self.storage_pool.type.lower() == 'rbd':
+            self.skipTest("Incremental backups are not supported on RBD/Ceph primary Storage")
+
     @attr(tags=["advanced", "backup"], required_hardware="true")
     def test_incremental_chain_cadence(self):
         """
         With nas.backup.full.every=3, the sequence of backups should be
         FULL, INCREMENTAL, INCREMENTAL, FULL, INCREMENTAL, ...
         """
+        self._require_incremental_capable_storage()
         self.backup_offering.assignOffering(self.apiclient, self.vm.id)
         original_full_every = self._get_full_every()
         self._set_full_every(3)
@@ -358,6 +421,7 @@ def test_incremental_after_vm_restart(self):
             FULL + marker1  ->  stop/start the VM (wipes the checkpoint registry)
             ->  INCREMENTAL + marker2  ->  restore the tip  ->  both markers present.
         """
+        self._require_incremental_capable_storage()
         self.backup_offering.assignOffering(self.apiclient, self.vm.id)
         original_full_every = self._get_full_every()
         # High cadence so the post-restart backup is INCREMENTAL, not a periodic FULL.
@@ -430,6 +494,7 @@ def test_restore_from_incremental(self):
         Take FULL + 2 INCREMENTAL backups, each with a marker file. Restore from the
         latest incremental and verify all three markers are present (chain flatten).
         """
+        self._require_incremental_capable_storage()
         self.backup_offering.assignOffering(self.apiclient, self.vm.id)
         original_full_every = self._get_full_every()
         self._set_full_every(5)
@@ -479,6 +544,7 @@ def test_delete_middle_incremental_repairs_chain(self):
         The chain repair should rebase INC2 onto FULL, and the final restore
         should still produce a working VM with all expected blocks.
         """
+        self._require_incremental_capable_storage()
         self.backup_offering.assignOffering(self.apiclient, self.vm.id)
         original_full_every = self._get_full_every()
         self._set_full_every(5)
@@ -531,6 +597,7 @@ def test_delete_full_with_children_is_deferred(self):
         FULL is hidden from the backup list while its child survives, and it is
         physically swept once the last descendant is deleted.
         """
+        self._require_incremental_capable_storage()
         self.backup_offering.assignOffering(self.apiclient, self.vm.id)
         original_full_every = self._get_full_every()
         self._set_full_every(5)
@@ -568,6 +635,7 @@ def test_stopped_vm_falls_back_to_full(self):
         would call for an incremental, the agent must fall back to a full and start a
         new chain. The incrementalFallback flag should be reflected in backup.type=FULL.
         """
+        self._require_incremental_capable_storage()
         self.backup_offering.assignOffering(self.apiclient, self.vm.id)
         original_full_every = self._get_full_every()
         self._set_full_every(2)  # next backup after the first should be incremental
@@ -594,3 +662,79 @@ def test_stopped_vm_falls_back_to_full(self):
         finally:
             self._set_full_every(original_full_every)
             self.backup_offering.removeOffering(self.apiclient, self.vm.id)
+
+    # ------------------------------------------------------------------
+    # Restore-volume-and-attach regression (PR apache/cloudstack#14007)
+    # ------------------------------------------------------------------
+    # This test exercises the fixed path end to end via restoreVolumeFromBackupAndAttachToVM,
+    # on whichever primary storage this environment is running with (NFS or Ceph/RBD).
+
+    @attr(tags=["advanced", "backup"], required_hardware="true")
+    def test_restore_volume_and_attach_to_vm(self):
+        """
+        Test restoring the ROOT and DATADISK volumes of a backup and attaching them
+        to a different Instance (restoreVolumeFromBackupAndAttachToVM).
+        """
+        target_vm = None
+        self.backup_offering.assignOffering(self.apiclient, self.vm.id)
+        try:
+            ssh_client_vm = self.vm.get_ssh_client(reconnect=True)
+            ssh_client_vm.execute("echo restore-attach-marker > /root/restore_attach_marker.txt; sync")
+
+            Backup.create(self.apiclient, self.vm.id, "restore_attach_backup")
+
+            backups = Backup.list(self.apiclient, self.vm.id)
+            self.assertEqual(len(backups), 1, "There should exist only one backup for the VM")
+            backup = backups[0]
+
+            volumes = Volume.list(self.apiclient, virtualmachineid=self.vm.id, listall=True)
+            self.assertTrue(isinstance(volumes, list), "List volumes should return a valid list")
+            root_disk_id = None
+            data_disk_id = None
+            for volume in volumes:
+                if volume.type == 'ROOT':
+                    root_disk_id = volume.id
+                elif volume.type == 'DATADISK':
+                    data_disk_id = volume.id
+            self.assertIsNotNone(root_disk_id, "The backed up VM should have a ROOT volume")
+
+            # Target Instance that will receive the restored volumes. The nas provider
+            # (unlike KBOSS) requires the target Instance to be stopped before a volume
+            # can be restored and attached to it.
+            target_vm = VirtualMachine.create(
+                self.apiclient, self.services["small"], accountid=self.account.name,
+                domainid=self.account.domainid, serviceofferingid=self.offering.id,
+                mode=self.services["mode"]
+            )
+            target_vm.stop(self.apiclient, forced=True)
+
+            # Restore and attach the ROOT volume backup as an extra disk.
+            Backup.restoreVolumeFromBackupAndAttachToVM(
+                self.apiclient, backupid=backup.id, volumeid=root_disk_id, virtualmachineid=target_vm.id
+            )
+            target_volumes = Volume.list(self.apiclient, virtualmachineid=target_vm.id, listall=True)
+            self.assertTrue(isinstance(target_volumes, list), "List volumes should return a valid list")
+            self.assertEqual(2, len(target_volumes),
+                "Target Instance should have its own ROOT volume plus the restored volume")
+
+            if data_disk_id:
+                # Restore and attach the DATADISK volume backup as well.
+                Backup.restoreVolumeFromBackupAndAttachToVM(
+                    self.apiclient, backupid=backup.id, volumeid=data_disk_id, virtualmachineid=target_vm.id
+                )
+                target_volumes = Volume.list(self.apiclient, virtualmachineid=target_vm.id, listall=True)
+                self.assertEqual(3, len(target_volumes),
+                    "Target Instance should have 3 volumes after restoring both the ROOT and DATADISK backups")
+
+            # Start the target Instance to verify the restored disk(s) are actually
+            # usable and libvirt accepted the attach-device/attach-disk calls.
+            target_vm.start(self.apiclient)
+
+            Backup.delete(self.apiclient, backup.id)
+        finally:
+            if target_vm is not None:
+                try:
+                    target_vm.delete(self.apiclient)
+                except Exception:
+                    pass
+            self.backup_offering.removeOffering(self.apiclient, self.vm.id)

test/integration/smoke/test_direct_download.py

@@ -218,7 +218,7 @@ def setUpClass(cls):
                 zoneid=cls.zone.id
             )
             for pool in storage_pools:
-                if not cls.nfsStorageFound and pool.type == "NetworkFilesystem":
+                if not cls.nfsStorageFound and pool.type in ("NetworkFilesystem", "RBD"):
                     cls.nfsStorageFound = True
                     cls.nfsPoolId = pool.id
                 elif not cls.localStorageFound and pool.type == "Filesystem":
@@ -298,7 +298,7 @@ def deployVM(self, offering) :
     @skipTestIf("nfsKvmNotAvailable")
     @attr(tags=["advanced", "basic", "eip", "advancedns", "sg"], required_hardware="false")
     def test_01_deploy_vm_from_direct_download_template_nfs_storage(self):
-        """Test Deploy VM from direct download template on NFS storage
+        """Test Deploy VM from direct download template on NFS or RBD storage
         """
 
         # Create service offering for local storage using storage tags

test/integration/smoke/test_over_provisioning.py

@@ -60,10 +60,10 @@ def test_UpdateStorageOverProvisioningFactor(self):
                             "The environment don't have storage pools required for test")
 
         for pool in storage_pools:
-            if pool.type == "NetworkFilesystem" or pool.type == "VMFS" or pool.type == "PowerFlex":
+            if pool.type in ("NetworkFilesystem", "VMFS", "PowerFlex", "RBD"):
                 break
 
-        if pool.type != "NetworkFilesystem" and pool.type != "VMFS" and pool.type != "PowerFlex":
+        if pool.type not in ("NetworkFilesystem", "VMFS", "PowerFlex", "RBD"):
             raise self.skipTest("Storage overprovisioning currently not supported on " + pool.type + " pools")
 
         self.poolId = pool.id

test/integration/smoke/test_snapshots.py

@@ -262,7 +262,7 @@ def test_02_list_snapshots_with_removed_data_store(self):
         volume_pool_response = list_storage_pools(self.apiclient,
                                                     id=vol_res[0].storageid)
         volume_pool = volume_pool_response[0]
-        if volume_pool.type.lower() != 'networkfilesystem':
+        if volume_pool.type not in ("NetworkFilesystem", "RBD"):
             self.skipTest("This test is not supported for volume created on storage pool type %s" % volume_pool.type)
         clusters = list_clusters(
             self.apiclient,
@@ -291,9 +291,8 @@ def test_02_list_snapshots_with_removed_data_store(self):
             'Up',
             "Check primary storage state"
         )
-        self.assertEqual(
-            storage.type,
-            'NetworkFilesystem',
+        self.assertTrue(
+            storage.type in ("NetworkFilesystem", "RBD"),
             "Check storage pool type"
         )
         storage_pools_response = list_storage_pools(self.apiclient,

test/integration/smoke/test_vm_life_cycle.py

@@ -48,6 +48,7 @@
                                get_suitable_test_template,
                                get_test_ovf_templates,
                                list_hosts,
+                               list_storage_pools,
                                get_vm_vapp_configs)
 from marvin.codes import FAILED, PASS
 from nose.plugins.attrib import attr
@@ -1707,12 +1708,17 @@ def get_target_host(self, virtualmachineid):
 
         return target_hosts[0]
 
-    def get_target_pool(self, volid):
-        target_pools = StoragePool.listForMigration(self.apiclient, id=volid)
+    def get_target_pool(self, vol):
+        target_pools = StoragePool.listForMigration(self.apiclient, id=vol.id)
 
         if target_pools is None or len(target_pools) == 0:
             self.skipTest("Not enough storage pools found for migration")
 
+        volume = Volume.list(self.apiclient, id=vol.id)[0]
+        source_pool = list_storage_pools(self.apiclient, id=volume.storageid)[0]
+        if source_pool.type == 'RBD' and target_pools[0].type == 'RBD':
+            self.skipTest("Live VM migration between RBD pools is unsupported")
+
         return target_pools[0]
 
     def get_vm_volumes(self, id):
@@ -1751,7 +1757,7 @@ def test_01_migrate_VM_and_root_volume(self):
 
         root_volume = self.get_vm_volumes(vm.id)[0]
 
-        target_pool = self.get_target_pool(root_volume.id)
+        target_pool = self.get_target_pool(root_volume)
 
         target_host = self.get_target_host(vm.id)
 
@@ -1789,9 +1795,9 @@ def test_02_migrate_VM_with_two_data_disks(self):
 
         root_volume = self.get_vm_volumes(vm.id)[0]
 
-        target_pool = self.get_target_pool(root_volume.id)
-        volume1.target_pool = self.get_target_pool(volume1.id)
-        volume2.target_pool = self.get_target_pool(volume2.id)
+        target_pool = self.get_target_pool(root_volume)
+        volume1.target_pool = self.get_target_pool(volume1)
+        volume2.target_pool = self.get_target_pool(volume2)
 
         target_host = self.get_target_host(vm.id)
 

test/integration/smoke/test_vm_snapshots.py

@@ -47,6 +47,11 @@ def setUpClass(cls):
             cls.unsupportedHypervisor = True
             return
 
+        list_volume_pool_response = list_storage_pools(cls.apiclient)
+        volume_pool = list_volume_pool_response[0]
+        if volume_pool.type == "RBD":
+            cls.skipTest(cls, reason="VM snapshot is unsupported for VMs on RBD storage pool")
+
         cls.services = testClient.getParsedTestDataConfig()
         # Get Zone, Domain and templates
         cls.domain = get_domain(cls.apiclient)