Snapshot state normalization for failure due to malformed secondary storage heuristic

Author: erikbocks

server/src/main/java/com/cloud/storage/snapshot/SnapshotManagerImpl.java

@@ -1633,11 +1633,17 @@ public SnapshotInfo takeSnapshot(VolumeInfo volume) throws ResourceAllocationExc
         boolean backupSnapToSecondary = isBackupSnapshotToSecondaryForZone(volume.getDataCenterId());
 
         if (isKvmAndFileBasedStorage && backupSnapToSecondary) {
-            DataStore imageStore = snapshotSrv.findSnapshotImageStore(snapshot);
-            if (imageStore == null) {
-                throw new CloudRuntimeException(String.format("Could not find any secondary storage to allocate snapshot [%s].", snapshot));
+            try {
+                DataStore imageStore = snapshotSrv.findSnapshotImageStore(snapshot);
+                if (imageStore == null) {
+                    throw new CloudRuntimeException(String.format("Could not find any secondary storage to allocate snapshot [%s].", snapshot));
+                }
+                snapshot.setImageStore(imageStore);
+            } catch (CloudRuntimeException ex) {
+                logger.error("There was an error while selecting image store for snapshot: {}", ex.getMessage());
+                handleErrorInUncreatedSnapshot(snapshotId);
+                throw new CloudRuntimeException(ex.getMessage());
             }
-            snapshot.setImageStore(imageStore);
         }
 
         updateSnapshotPayload(volume.getPoolId(), payload, isKvmAndFileBasedStorage, clusterId);
@@ -1738,6 +1744,12 @@ private void postSnapshotDirectlyToSecondary(SnapshotInfo snapshot, SnapshotInfo
         snapshotDetailsDao.removeDetail(snapshotOnPrimary.getId(), AsyncJob.Constants.MS_ID);
     }
 
+    private void handleErrorInUncreatedSnapshot(Long snapshotId) {
+        SnapshotVO snapshot = _snapshotDao.findById(snapshotId);
+        snapshot.setState(Snapshot.State.Error);
+        _snapshotDao.persist(snapshot);
+    }
+
     @Override
     public boolean isHypervisorKvmAndFileBasedStorage(VolumeInfo volumeInfo, StoragePool storagePool) {
         Set<StoragePoolType> fileBasedStores = Set.of(Storage.StoragePoolType.SharedMountPoint, Storage.StoragePoolType.NetworkFilesystem, Storage.StoragePoolType.Filesystem);