StorPool support importing standalone disk volumes and importVm from SHARED

api/src/main/java/org/apache/cloudstack/storage/volume/VolumeImportUnmanageService.java

@@ -19,6 +19,8 @@
 
 import com.cloud.hypervisor.Hypervisor;
 import com.cloud.storage.Storage;
+import com.cloud.storage.StoragePool;
+import com.cloud.storage.Volume;
 import com.cloud.utils.component.PluggableService;
 import org.apache.cloudstack.api.command.admin.volume.ListVolumesForImportCmd;
 import org.apache.cloudstack.api.command.admin.volume.ImportVolumeCmd;
@@ -37,7 +39,8 @@ public interface VolumeImportUnmanageService extends PluggableService, Configura
             Arrays.asList(Hypervisor.HypervisorType.KVM, Hypervisor.HypervisorType.VMware);
 
     List<Storage.StoragePoolType> SUPPORTED_STORAGE_POOL_TYPES_FOR_KVM = Arrays.asList(Storage.StoragePoolType.NetworkFilesystem,
-            Storage.StoragePoolType.Filesystem, Storage.StoragePoolType.RBD, Storage.StoragePoolType.SharedMountPoint);
+            Storage.StoragePoolType.Filesystem, Storage.StoragePoolType.RBD, Storage.StoragePoolType.SharedMountPoint,
+            Storage.StoragePoolType.StorPool);
 
     ConfigKey<Boolean> AllowImportVolumeWithBackingFile = new ConfigKey<>(Boolean.class,
             "allow.import.volume.with.backing.file",
@@ -54,4 +57,10 @@ public interface VolumeImportUnmanageService extends PluggableService, Configura
 
     boolean unmanageVolume(long volumeId);
 
+    /**
+     * Lets a storage driver (e.g. StorPool) react to a volume becoming managed/unmanaged, from
+     * this service or from another import path (e.g. VM import). No-op if unsupported.
+     */
+    void notifyStorageDriverOfVolumeManagementChange(StoragePool pool, Volume volume, boolean imported);
+
 }

engine/api/src/main/java/org/apache/cloudstack/engine/subsystem/api/storage/PrimaryDataStoreDriver.java

@@ -18,10 +18,12 @@
  */
 package org.apache.cloudstack.engine.subsystem.api.storage;
 
+import java.util.List;
 import java.util.Map;
 
 import org.apache.cloudstack.framework.async.AsyncCompletionCallback;
 import org.apache.cloudstack.storage.command.CommandResult;
+import org.apache.cloudstack.storage.volume.VolumeOnStorageTO;
 
 import com.cloud.host.Host;
 import com.cloud.offering.DiskOffering;
@@ -221,4 +223,23 @@ default boolean informStorageForDiskOfferingChange() {
     }
 
     default void updateStorageWithTheNewDiskOffering(Volume volume, DiskOffering newDiskOffering) {}
+
+    /**
+     * intended for storage that can enumerate its own volumes available for import without hypervisor involvement
+     * @param volumePath when not blank, restricts the result to the volume with this path, if found
+     * @param keyword when not blank, restricts the result to volumes whose name contains this keyword
+     * @param forUnmanage when volume will be unmanaged from CloudStack
+     * @return null if the storage driver does not support listing volumes for import on its own,
+     * in which case the caller falls back to querying a host via GetVolumesOnStorageCommand
+    */
+    default List<VolumeOnStorageTO> listVolumesForImport(StoragePool pool, String volumePath, String keyword, boolean forUnmanage) {
+        return null;
+    }
+
+    /**
+     * Called after a volume is imported into or unmanaged from CloudStack, so the driver can
+     * mark/unmark it accordingly (e.g. tag it) for future {@link #listVolumesForImport} calls.
+     */
+    default void volumeManagementChanged(StoragePool pool, Volume volume, boolean imported) {
+    }
 }

plugins/storage/volume/storpool/src/main/java/com/cloud/hypervisor/kvm/storage/StorPoolStorageAdaptor.java

@@ -109,12 +109,13 @@ public boolean deleteStoragePool(KVMStoragePool pool) {
         return deleteStoragePool(pool.getUuid());
     }
 
-    private static long getDeviceSize(final String devPath) {
+    private static long getDeviceSize(String devPath) {
         SP_LOG("StorPoolStorageAdaptor.getDeviceSize: path=%s", devPath);
 
         if (getVolumeNameFromPath(devPath, true) == null) {
             return 0;
         }
+        devPath = devPath.trim();
         File file = new File(devPath);
         if (!file.exists()) {
             return 0;

plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/datastore/driver/StorPoolPrimaryDataStoreDriver.java

@@ -20,12 +20,16 @@
 
 import com.cloud.storage.dao.SnapshotDetailsVO;
 
+import java.util.ArrayList;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 
 import javax.inject.Inject;
 
+import com.google.gson.JsonArray;
+import com.google.gson.JsonObject;
+
 import com.cloud.storage.dao.StoragePoolHostDao;
 import com.cloud.storage.dao.VMTemplateDetailsDao;
 import com.cloud.storage.dao.VolumeDao;
@@ -82,9 +86,11 @@
 import org.apache.cloudstack.storage.to.TemplateObjectTO;
 import org.apache.cloudstack.storage.to.VolumeObjectTO;
 import org.apache.cloudstack.storage.volume.VolumeObject;
+import org.apache.cloudstack.storage.volume.VolumeOnStorageTO;
 
 import org.apache.commons.collections4.CollectionUtils;
 import org.apache.commons.collections4.MapUtils;
+import org.apache.commons.lang3.StringUtils;
 
 import com.cloud.agent.api.Answer;
 import com.cloud.agent.api.storage.ResizeVolumeAnswer;
@@ -105,6 +111,7 @@
 import com.cloud.host.Host;
 import com.cloud.host.HostVO;
 import com.cloud.host.dao.HostDao;
+import com.cloud.hypervisor.Hypervisor;
 import com.cloud.hypervisor.kvm.storage.StorPoolStorageAdaptor;
 import com.cloud.offering.DiskOffering;
 import com.cloud.server.ResourceTag;
@@ -117,6 +124,7 @@
 import com.cloud.storage.ResizeVolumePayload;
 import com.cloud.storage.Snapshot;
 import com.cloud.storage.SnapshotVO;
+import com.cloud.storage.Storage;
 import com.cloud.storage.Storage.StoragePoolType;
 import com.cloud.storage.StorageManager;
 import com.cloud.storage.StoragePool;
@@ -130,6 +138,7 @@
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
+import org.jetbrains.annotations.NotNull;
 
 public class StorPoolPrimaryDataStoreDriver implements PrimaryDataStoreDriver {
 
@@ -353,6 +362,7 @@ private void updateVolume(DataStore dataStore, String path, VolumeInfo vinfo) {
         volume.setPoolId(dataStore.getId());
         volume.setPath(path);
         volume.setPoolType(StoragePoolType.StorPool);
+        volume.setFormat(Storage.ImageFormat.QCOW2);
         volumeDao.update(volume.getId(), volume);
     }
 
@@ -895,7 +905,7 @@ public void copyAsync(DataObject srcData, DataObject dstData, AsyncCompletionCal
                     template = conn.getTemplateName();
                 }
 
-                StorPoolVolumeDef volumeDef = new StorPoolVolumeDef(null, size, tags, parentName, null, template, null, null, null);
+                StorPoolVolumeDef volumeDef = new StorPoolVolumeDef(null, size, tags, parentName, vinfo.getMaxIops(), template, null, null, null);
                 resp = StorPoolUtil.volumeCreate(volumeDef, conn);
 
                 if (resp.getError() == null) {
@@ -1367,6 +1377,120 @@ public Pair<Long, Long> getVolumeStats(StoragePool storagePool, String volumeId)
         return null;
     }
 
+    /**
+     * Lists StorPool volumes available for import (globalId name, no "cs" tag unless
+     * {@code forUnmanage}). When {@code volumePath} is given, looks up that one volume directly
+     * instead of browsing the whole pool; when {@code forUnmanage} is also true, an already
+     * "cs"-tagged volume is still returned, since the caller is unmanageVolume() looking for the
+     * very volume it is about to drop from CloudStack.
+     */
+    @Override
+    public List<VolumeOnStorageTO> listVolumesForImport(StoragePool pool, String volumePath, String keyword, boolean forUnmanage) {
+        StorPoolUtil.spLog("StorPoolPrimaryDataStoreDriver.listVolumesForImport: pool=%s, volumePath=%s, keyword=%s, forUnmanage=%s", pool.getName(), volumePath, keyword, forUnmanage);
+        SpConnectionDesc conn = StorPoolUtil.getSpConnection(pool.getUuid(), pool.getId(), storagePoolDetailsDao, primaryStoreDao);
+        List<VolumeOnStorageTO> volumes = new ArrayList<>();
+
+        if (StringUtils.isNotBlank(volumePath)) {
+            return getVolumeOnStorageByPath(pool, volumePath, conn, volumes, forUnmanage);
+        }
+
+        SpApiResponse resp = StorPoolUtil.volumesListResponse(conn);
+        if (resp.getError() != null) {
+            StorPoolUtil.spLog("StorPoolPrimaryDataStoreDriver.listVolumesForImport: could not list volumes for pool=%s. Error: %s", pool.getName(), resp.getError());
+            return volumes;
+        }
+        return getVolumeOnStorage(pool, keyword, resp, volumes);
+    }
+
+    @NotNull
+    private List<VolumeOnStorageTO> getVolumeOnStorage(StoragePool pool, String keyword, SpApiResponse resp, List<VolumeOnStorageTO> volumes) {
+        JsonArray volumesList = resp.fullJson.getAsJsonObject().getAsJsonArray("data");
+
+        for (int i = 0; i < volumesList.size(); i++) {
+            JsonObject volume = volumesList.get(i).getAsJsonObject();
+            String rawName = volume.get("name").getAsString();
+            if (isVolumeDeleted(volume) || !isEligibleForBulkImport(volume, rawName)) {
+                continue;
+            }
+
+            String name = rawName.substring(1);
+
+            if (StringUtils.isNotBlank(keyword) && !name.contains(keyword)) {
+                continue;
+            }
+            volumes.add(toVolumeOnStorageTO(volume, name));
+        }
+        StorPoolUtil.spLog("StorPoolPrimaryDataStoreDriver.listVolumesForImport: found %s volume(s) for pool=%s", volumes.size(), pool.getName());
+        return volumes;
+    }
+
+    @NotNull
+    private List<VolumeOnStorageTO> getVolumeOnStorageByPath(StoragePool pool, String volumePath, SpConnectionDesc conn, List<VolumeOnStorageTO> volumes, boolean forUnmanage) {
+        String name = StorPoolStorageAdaptor.getVolumeNameFromPath(volumePath, true);
+        SpApiResponse resp = StorPoolUtil.volumeDescribeResponse(name, conn);
+        if (resp.getError() != null) {
+            StorPoolUtil.spLog("StorPoolPrimaryDataStoreDriver.listVolumesForImport: could not describe volume=%s on pool=%s. Error: %s", name, pool.getName(), resp.getError());
+            return volumes;
+        }
+        JsonArray data = resp.fullJson.getAsJsonObject().getAsJsonArray("data");
+        if (data == null || data.isEmpty()) {
+            return volumes;
+        }
+        JsonObject volume = data.get(0).getAsJsonObject();
+        String rawName = volume.get("name").getAsString();
+        if (isVolumeDeleted(volume) || !isGlobalIdVolumeName(rawName) || (!forUnmanage && isTaggedCs(volume))) {
+            return volumes;
+        }
+        volumes.add(toVolumeOnStorageTO(volume, rawName.substring(1)));
+        return volumes;
+    }
+
+    private static boolean isVolumeDeleted(JsonObject volume) {
+        return volume.has("deleted") && volume.get("deleted").getAsBoolean();
+    }
+
+    private static boolean isGlobalIdVolumeName(String rawName) {
+        return !rawName.startsWith("*") && !rawName.contains("@") && rawName.startsWith("~");
+    }
+
+    private static boolean isTaggedCs(JsonObject volume) {
+        JsonObject tags = volume.has("tags") && !volume.get("tags").isJsonNull() ? volume.getAsJsonObject("tags") : null;
+        return tags != null && tags.has("cs");
+    }
+
+    private static boolean isEligibleForBulkImport(JsonObject volume, String rawName) {
+        return isGlobalIdVolumeName(rawName) && !isTaggedCs(volume);
+    }
+
+    private VolumeOnStorageTO toVolumeOnStorageTO(JsonObject volume, String name) {
+        String path = StorPoolUtil.devPath(name);
+        long size = volume.get("size").getAsLong();
+        long spaceUsed = volume.has("spaceUsed") ? volume.get("spaceUsed").getAsLong() : size;
+        StorPoolUtil.spLog("listVolumesForImport: found volume=%s, path=%s, size=%s, spaceUsed=%s", name, path, size, spaceUsed);
+        return new VolumeOnStorageTO(Hypervisor.HypervisorType.KVM, path, name, path,
+                Storage.ImageFormat.QCOW2.toString(), spaceUsed, size);
+    }
+
+    @Override
+    public void volumeManagementChanged(StoragePool pool, Volume volume, boolean imported) {
+        StorPoolUtil.spLog("StorPoolPrimaryDataStoreDriver.volumeManagementChanged: pool=%s, volume=%s, imported=%s", pool.getName(), volume.getPath(), imported);
+        SpConnectionDesc conn = StorPoolUtil.getSpConnection(pool.getUuid(), pool.getId(), storagePoolDetailsDao, primaryStoreDao);
+        String name = StorPoolStorageAdaptor.getVolumeNameFromPath(volume.getPath(), true);
+        Map<String, String> tags;
+        if (imported) {
+            String tier = volume.getDiskOfferingId() != null ? getTierFromOfferingDetail(volume.getDiskOfferingId()) : null;
+            Long instanceId = volume.getInstanceId();
+            tags = StorPoolHelper.addStorPoolTags(volume.getUuid(), getVMInstanceUUID(instanceId), "volume", getVcPolicyTag(instanceId), tier);
+        } else {
+            tags = StorPoolHelper.addStorPoolTags("", "", "", "", "");
+        }
+        StorPoolVolumeDef spVolume = new StorPoolVolumeDef(name, null, tags, null, null, null, null, null, null);
+        SpApiResponse resp = StorPoolUtil.volumeUpdate(spVolume, conn);
+        if (resp.getError() != null) {
+            StorPoolUtil.spLog("StorPoolPrimaryDataStoreDriver.volumeManagementChanged: could not update tags on volume=%s. Error: %s", name, resp.getError());
+        }
+    }
+
     public boolean canHostAccessStoragePool(Host host, StoragePool pool) {
         return false;
     }
@@ -1398,11 +1522,11 @@ private void updateVolumeWithTags(StoragePoolVO poolVO, VolumeInfo vInfo) {
             SpConnectionDesc conn = StorPoolUtil.getSpConnection(poolVO.getUuid(), poolVO.getId(), storagePoolDetailsDao, primaryStoreDao);
             String volName = StorPoolStorageAdaptor.getVolumeNameFromPath(vInfo.getPath(), true);
             Pair<String, String> templateAndTier = getTemplateAndTier(vInfo, conn);
-            Map<String, String> tags = StorPoolHelper.addStorPoolTags(volName, getVMInstanceUUID(vInfo.getInstanceId()), "volume", getVcPolicyTag(vInfo.getInstanceId()), templateAndTier.first());
+            Map<String, String> tags = StorPoolHelper.addStorPoolTags(vInfo.getUuid(), getVMInstanceUUID(vInfo.getInstanceId()), "volume", getVcPolicyTag(vInfo.getInstanceId()), templateAndTier.first());
             if (vInfo.getDeviceId() != null) {
                 tags.put("disk", vInfo.getDeviceId().toString());
             }
-            StorPoolVolumeDef spVolume = new StorPoolVolumeDef(volName, null, tags, null, null, templateAndTier.second(), null, null, null);
+            StorPoolVolumeDef spVolume = new StorPoolVolumeDef(volName, null, tags, null, null, null, null, null, null);
             StorPoolUtil.spLog("Updating volume's tags [%s] with template [%s]", tags, templateAndTier.second());
             SpApiResponse resp = StorPoolUtil.volumeUpdate(spVolume, conn);
             if (resp.getError() != null) {

plugins/storage/volume/storpool/src/main/java/org/apache/cloudstack/storage/datastore/util/StorPoolUtil.java

@@ -452,8 +452,16 @@ public static JsonArray snapshotsListAllClusters(SpConnectionDesc conn) {
         return obj.getAsJsonObject(DATA).getAsJsonArray(CLUSTERS);
     }
 
+    public static SpApiResponse volumesListResponse(SpConnectionDesc conn) {
+        return GET("MultiCluster/VolumesList", conn);
+    }
+
+    public static SpApiResponse volumeDescribeResponse(String name, SpConnectionDesc conn) {
+        return GET("MultiCluster/Volume/" + name, conn);
+    }
+
     public static JsonArray volumesList(SpConnectionDesc conn) {
-        SpApiResponse resp = GET("MultiCluster/VolumesList", conn);
+        SpApiResponse resp = volumesListResponse(conn);
         JsonObject obj = resp.fullJson.getAsJsonObject();
         return obj.getAsJsonArray(DATA);
     }