Copy template to target KVM host if needed when migrating local <> local storage (apache#3154)

* Migrate template to target host if needed.

Fix KVM VM local storage live migration by migrating its template to the
target host if needed.

* Address reviewer and add method that updates the DB template reference

* Remove deprecated Config.PrimaryStorageDownloadWait

* Code formating of @Inject to follow checkstyle

Author: GabrielBrascher

engine/components-api/src/main/java/com/cloud/storage/StorageManager.java

@@ -101,6 +101,9 @@ public interface StorageManager extends StorageService {
             ConfigKey.Scope.Cluster,
             null);
 
+    ConfigKey<Integer> PRIMARY_STORAGE_DOWNLOAD_WAIT = new ConfigKey<Integer>("Storage", Integer.class, "primary.storage.download.wait", "10800",
+            "In second, timeout for download template to primary storage", false);
+
     /**
      * Returns a comma separated list of tags for the specified storage pool
      * @param poolId

engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/AncientDataMotionStrategy.java

@@ -62,6 +62,7 @@
 import com.cloud.host.Host;
 import com.cloud.hypervisor.Hypervisor;
 import com.cloud.storage.DataStoreRole;
+import com.cloud.storage.StorageManager;
 import com.cloud.storage.Storage.StoragePoolType;
 import com.cloud.storage.StoragePool;
 import com.cloud.storage.VolumeVO;
@@ -145,8 +146,7 @@ private Scope pickCacheScopeForCopy(DataObject srcData, DataObject destData) {
     }
 
     protected Answer copyObject(DataObject srcData, DataObject destData, Host destHost) {
-        String value = configDao.getValue(Config.PrimaryStorageDownloadWait.toString());
-        int _primaryStorageDownloadWait = NumbersUtil.parseInt(value, Integer.parseInt(Config.PrimaryStorageDownloadWait.getDefaultValue()));
+        int primaryStorageDownloadWait = StorageManager.PRIMARY_STORAGE_DOWNLOAD_WAIT.value();
         Answer answer = null;
         DataObject cacheData = null;
         DataObject srcForCopy = srcData;
@@ -156,7 +156,8 @@ protected Answer copyObject(DataObject srcData, DataObject destData, Host destHo
                 srcForCopy = cacheData = cacheMgr.createCacheObject(srcData, destScope);
             }
 
-            CopyCommand cmd = new CopyCommand(srcForCopy.getTO(), addFullCloneFlagOnVMwareDest(destData.getTO()), _primaryStorageDownloadWait, VirtualMachineManager.ExecuteInSequence.value());
+            CopyCommand cmd = new CopyCommand(srcForCopy.getTO(), addFullCloneFlagOnVMwareDest(destData.getTO()), primaryStorageDownloadWait,
+                    VirtualMachineManager.ExecuteInSequence.value());
             EndPoint ep = destHost != null ? RemoteHostEndPoint.getHypervisorHostEndPoint(destHost) : selector.select(srcForCopy, destData);
             if (ep == null) {
                 String errMsg = "No remote endpoint to send command, check if host or ssvm is down?";

engine/storage/datamotion/src/main/java/org/apache/cloudstack/storage/motion/KvmNonManagedStorageDataMotionStrategy.java

@@ -29,22 +29,36 @@
 import org.apache.cloudstack.engine.subsystem.api.storage.TemplateDataFactory;
 import org.apache.cloudstack.engine.subsystem.api.storage.TemplateInfo;
 import org.apache.cloudstack.engine.subsystem.api.storage.VolumeInfo;
+import org.apache.cloudstack.engine.subsystem.api.storage.ObjectInDataStoreStateMachine;
+import org.apache.cloudstack.storage.command.CopyCommand;
+import org.apache.cloudstack.storage.datastore.DataStoreManagerImpl;
 import org.apache.cloudstack.storage.datastore.db.StoragePoolVO;
+import org.apache.cloudstack.storage.to.TemplateObjectTO;
+import org.apache.commons.lang.StringUtils;
+import org.apache.log4j.Logger;
 
 import com.cloud.agent.api.Answer;
 import com.cloud.agent.api.MigrateCommand;
 import com.cloud.agent.api.MigrateCommand.MigrateDiskInfo;
 import com.cloud.agent.api.storage.CreateAnswer;
 import com.cloud.agent.api.storage.CreateCommand;
 import com.cloud.agent.api.to.VirtualMachineTO;
+import com.cloud.exception.AgentUnavailableException;
+import com.cloud.exception.OperationTimedoutException;
 import com.cloud.host.Host;
 import com.cloud.hypervisor.Hypervisor.HypervisorType;
 import com.cloud.storage.DataStoreRole;
 import com.cloud.storage.DiskOfferingVO;
 import com.cloud.storage.Storage.StoragePoolType;
+import com.cloud.storage.StorageManager;
+import com.cloud.storage.StoragePool;
+import com.cloud.storage.VMTemplateStoragePoolVO;
+import com.cloud.storage.VMTemplateStorageResourceAssoc;
 import com.cloud.storage.VolumeVO;
+import com.cloud.storage.dao.VMTemplatePoolDao;
 import com.cloud.utils.exception.CloudRuntimeException;
 import com.cloud.vm.DiskProfile;
+import com.cloud.vm.VirtualMachineManager;
 
 /**
  * Extends {@link StorageSystemDataMotionStrategy}, allowing KVM hosts to migrate VMs with the ROOT volume on a non managed local storage pool.
@@ -54,6 +68,14 @@ public class KvmNonManagedStorageDataMotionStrategy extends StorageSystemDataMot
 
     @Inject
     private TemplateDataFactory templateDataFactory;
+    @Inject
+    private VMTemplatePoolDao vmTemplatePoolDao;
+    @Inject
+    private DataStoreManagerImpl dataStoreManagerImpl;
+    @Inject
+    private VirtualMachineManager virtualMachineManager;
+
+    private static final Logger LOGGER = Logger.getLogger(KvmNonManagedStorageDataMotionStrategy.class);
 
     /**
      * Uses the canHandle from the Super class {@link StorageSystemDataMotionStrategy}. If the storage pool is of file and the internalCanHandle from {@link StorageSystemDataMotionStrategy} CANT_HANDLE, returns the StrategyPriority.HYPERVISOR strategy priority. otherwise returns CANT_HANDLE.
@@ -95,7 +117,7 @@ protected String generateDestPath(VirtualMachineTO vmTO, VolumeVO srcVolume, Hos
         String templateUuid = getTemplateUuid(destVolumeInfo.getTemplateId());
         CreateCommand rootImageProvisioningCommand = new CreateCommand(diskProfile, templateUuid, destStoragePool, true);
 
-        Answer rootImageProvisioningAnswer = _agentMgr.easySend(destHost.getId(), rootImageProvisioningCommand);
+        Answer rootImageProvisioningAnswer = agentManager.easySend(destHost.getId(), rootImageProvisioningCommand);
 
         if (rootImageProvisioningAnswer == null) {
             throw new CloudRuntimeException(String.format("Migration with storage of vm [%s] failed while provisioning root image", vmTO.getName()));
@@ -140,4 +162,77 @@ protected void setVolumePath(VolumeVO volume) {
     protected boolean shouldMigrateVolume(StoragePoolVO sourceStoragePool, Host destHost, StoragePoolVO destStoragePool) {
         return sourceStoragePool.getPoolType() == StoragePoolType.Filesystem;
     }
+
+    /**
+     * If the template is not on the target primary storage then it copies the template.
+     */
+    @Override
+    protected void copyTemplateToTargetFilesystemStorageIfNeeded(VolumeInfo srcVolumeInfo, StoragePool srcStoragePool, DataStore destDataStore, StoragePool destStoragePool,
+            Host destHost) {
+        VMTemplateStoragePoolVO sourceVolumeTemplateStoragePoolVO = vmTemplatePoolDao.findByPoolTemplate(destStoragePool.getId(), srcVolumeInfo.getTemplateId());
+        if (sourceVolumeTemplateStoragePoolVO == null && destStoragePool.getPoolType() == StoragePoolType.Filesystem) {
+            DataStore sourceTemplateDataStore = dataStoreManagerImpl.getImageStore(srcVolumeInfo.getDataCenterId());
+            TemplateInfo sourceTemplateInfo = templateDataFactory.getTemplate(srcVolumeInfo.getTemplateId(), sourceTemplateDataStore);
+            TemplateObjectTO sourceTemplate = new TemplateObjectTO(sourceTemplateInfo);
+
+            LOGGER.debug(String.format("Could not find template [id=%s, name=%s] on the storage pool [id=%s]; copying the template to the target storage pool.",
+                    srcVolumeInfo.getTemplateId(), sourceTemplateInfo.getName(), destDataStore.getId()));
+
+            TemplateInfo destTemplateInfo = templateDataFactory.getTemplate(srcVolumeInfo.getTemplateId(), destDataStore);
+            final TemplateObjectTO destTemplate = new TemplateObjectTO(destTemplateInfo);
+            Answer copyCommandAnswer = sendCopyCommand(destHost, sourceTemplate, destTemplate, destDataStore);
+
+            if (copyCommandAnswer != null && copyCommandAnswer.getResult()) {
+                updateTemplateReferenceIfSuccessfulCopy(srcVolumeInfo, srcStoragePool, destTemplateInfo, destDataStore);
+            }
+        }
+    }
+
+    /**
+     *  Update the template reference on table "template_spool_ref" (VMTemplateStoragePoolVO).
+     */
+    protected void updateTemplateReferenceIfSuccessfulCopy(VolumeInfo srcVolumeInfo, StoragePool srcStoragePool, TemplateInfo destTemplateInfo, DataStore destDataStore) {
+        VMTemplateStoragePoolVO srcVolumeTemplateStoragePoolVO = vmTemplatePoolDao.findByPoolTemplate(srcStoragePool.getId(), srcVolumeInfo.getTemplateId());
+        VMTemplateStoragePoolVO destVolumeTemplateStoragePoolVO = new VMTemplateStoragePoolVO(destDataStore.getId(), srcVolumeInfo.getTemplateId());
+        destVolumeTemplateStoragePoolVO.setDownloadPercent(100);
+        destVolumeTemplateStoragePoolVO.setDownloadState(VMTemplateStorageResourceAssoc.Status.DOWNLOADED);
+        destVolumeTemplateStoragePoolVO.setState(ObjectInDataStoreStateMachine.State.Ready);
+        destVolumeTemplateStoragePoolVO.setTemplateSize(srcVolumeTemplateStoragePoolVO.getTemplateSize());
+        destVolumeTemplateStoragePoolVO.setLocalDownloadPath(destTemplateInfo.getUuid());
+        destVolumeTemplateStoragePoolVO.setInstallPath(destTemplateInfo.getUuid());
+        vmTemplatePoolDao.persist(destVolumeTemplateStoragePoolVO);
+    }
+
+    /**
+     * Sends the CopyCommand to migrate the template to the dest host.
+     */
+    protected Answer sendCopyCommand(Host destHost, TemplateObjectTO sourceTemplate, TemplateObjectTO destTemplate, DataStore destDataStore) {
+        boolean executeInSequence = virtualMachineManager.getExecuteInSequence(HypervisorType.KVM);
+        CopyCommand copyCommand = new CopyCommand(sourceTemplate, destTemplate, StorageManager.PRIMARY_STORAGE_DOWNLOAD_WAIT.value(), executeInSequence);
+        try {
+            Answer copyCommandAnswer = agentManager.send(destHost.getId(), copyCommand);
+            logInCaseOfTemplateCopyFailure(copyCommandAnswer, sourceTemplate, destDataStore);
+            return copyCommandAnswer;
+        } catch (AgentUnavailableException | OperationTimedoutException e) {
+            throw new CloudRuntimeException(generateFailToCopyTemplateMessage(sourceTemplate, destDataStore), e);
+        }
+    }
+
+    private String generateFailToCopyTemplateMessage(TemplateObjectTO sourceTemplate, DataStore destDataStore) {
+        return String.format("Failed to copy template [id=%s, name=%s] to the primary storage pool [id=%s].", sourceTemplate.getId(),
+                sourceTemplate.getName(), destDataStore.getId());
+    }
+
+    /**
+     * Logs in debug mode the copy command failure if the CopyCommand Answer has result as false.
+     */
+    protected void logInCaseOfTemplateCopyFailure(Answer copyCommandAnswer, TemplateObjectTO sourceTemplate, DataStore destDataStore) {
+        if (copyCommandAnswer != null && !copyCommandAnswer.getResult()) {
+            String failureDetails = StringUtils.EMPTY;
+            if (copyCommandAnswer.getDetails() != null) {
+                failureDetails = " Details: " + copyCommandAnswer.getDetails();
+            }
+            LOGGER.error(generateFailToCopyTemplateMessage(sourceTemplate, destDataStore) + failureDetails);
+        }
+    }
 }