Merge branch 'fix-local-migrate-state-roll-back' into 'master'

Add rollback for local volume migrate state change

Closes ZSTAC-10519

See merge request zstackio/zstack!2396

Author: gitlab

plugin/localstorage/src/main/java/org/zstack/storage/primary/local/LocalStorageBase.java

@@ -7,6 +7,8 @@
 import org.zstack.core.componentloader.PluginRegistry;
 import org.zstack.core.db.*;
 import org.zstack.core.db.SimpleQuery.Op;
+import org.zstack.core.defer.Defer;
+import org.zstack.core.defer.Deferred;
 import org.zstack.core.thread.AsyncThread;
 import org.zstack.core.thread.ChainTask;
 import org.zstack.core.thread.SyncTaskChain;
@@ -49,6 +51,7 @@
 import org.zstack.storage.primary.local.MigrateBitsStruct.ResourceInfo;
 import org.zstack.utils.CollectionUtils;
 import org.zstack.utils.DebugUtils;
+import org.zstack.utils.ShellUtils;
 import org.zstack.utils.Utils;
 import org.zstack.utils.function.Function;
 import org.zstack.utils.gson.JSONObjectUtil;
@@ -268,7 +271,8 @@ class MigrateStruct {
             private boolean isRootVolume = false;
             private OverlayMessage message;
             private String vmUuid;
-            private boolean isMigrated = false;
+            private boolean volumeStatusChanged = false;
+            private boolean vmStateChanged = false;
 
             public String getVmOriginState() {
                 return vmOriginState;
@@ -304,20 +308,28 @@ public void setVmUuid(String vmUuid) {
                 this.vmUuid = vmUuid;
             }
 
-            public boolean isMigrated() {
-                return isMigrated;
+            public boolean isVolumeStatusChanged() {
+                return volumeStatusChanged;
             }
 
-            public void setMigrated(boolean migrated) {
-                isMigrated = migrated;
+            public void setVolumeStatusChanged(boolean volumeStatusChanged) {
+                this.volumeStatusChanged = volumeStatusChanged;
+            }
+
+            public boolean isVmStateChanged() {
+                return vmStateChanged;
+            }
+
+            public void setVmStateChanged(boolean vmStateChanged) {
+                this.vmStateChanged = vmStateChanged;
             }
         }
 
         MigrateStruct struct = new MigrateStruct();
         VolumeStatus originStatus = Q.New(VolumeVO.class).select(VolumeVO_.status).eq(VolumeVO_.uuid, msg.getVolumeUuid()).findValue();
         FlowChain chain = new SimpleFlowChain();
         chain.setName(String.format("local-storage-%s-migrate-volume-%s-to-host-%s", msg.getPrimaryStorageUuid(), msg.getVolumeUuid(), msg.getDestHostUuid()));
-        chain.then(new NoRollbackFlow() {
+        chain.then(new Flow() {
             @Override
             public void run(FlowTrigger trigger, Map data) {
                 String __name__ = "change-volume-status-to-migrating";
@@ -334,11 +346,25 @@ public void run(MessageReply reply) {
                             return;
                         }
 
+                        struct.setVolumeStatusChanged(true);
                         trigger.next();
                     }
                 });
             }
-        }).then(new NoRollbackFlow() {
+
+            @Override
+            public void rollback(FlowRollback trigger, Map data) {
+                if (struct.isVolumeStatusChanged()) {
+                    ChangeVolumeStatusMsg rollbackMsg  = new ChangeVolumeStatusMsg();
+                    rollbackMsg.setStatus(originStatus);
+                    rollbackMsg.setVolumeUuid(msg.getVolumeUuid());
+                    bus.makeTargetServiceIdByResourceUuid(rollbackMsg, VolumeConstant.SERVICE_ID, msg.getVolumeUuid());
+                    bus.send(rollbackMsg);
+                }
+
+                trigger.rollback();
+            }
+        }).then(new Flow() {
             @Override
             public void run(FlowTrigger trigger, Map data) {
                 String __name__ = "change-vm-state-to-volume-migrating";
@@ -370,10 +396,24 @@ public void run(MessageReply reply) {
                             return;
                         }
 
+                        struct.setVmStateChanged(true);
                         trigger.next();
                     }
                 });
             }
+
+            @Override
+            public void rollback(FlowRollback trigger, Map data) {
+                if (struct.isVmStateChanged()) {
+                    ChangeVmStateMsg rollbackMsg = new ChangeVmStateMsg();
+                    rollbackMsg.setStateEvent(struct.getVmOriginState());
+                    rollbackMsg.setVmInstanceUuid(struct.getVmUuid());
+                    bus.makeTargetServiceIdByResourceUuid(rollbackMsg, VmInstanceConstant.SERVICE_ID, struct.getVmUuid());
+                    bus.send(rollbackMsg);
+                }
+
+                trigger.rollback();
+            }
         }).then(new NoRollbackFlow() {
             @Override
             public void run(FlowTrigger trigger, Map data) {
@@ -412,7 +452,6 @@ public void run(MessageReply reply) {
 
                         MigrateVolumeOnLocalStorageReply mr = reply.castReply();
                         evt.setInventory(mr.getInventory());
-                        struct.setMigrated(true);
                         trigger.next();
                     }
                 });

test/src/test/groovy/org/zstack/test/integration/storage/primary/local/LocalStorageVolumeMigrateFailureRollbackCase.groovy

@@ -0,0 +1,79 @@
+package org.zstack.test.integration.storage.primary.local
+
+import org.zstack.core.db.Q
+import org.zstack.header.vm.VmInstanceState
+import org.zstack.header.vm.VmInstanceVO
+import org.zstack.header.vm.VmInstanceVO_
+import org.zstack.header.volume.VolumeStatus
+import org.zstack.header.volume.VolumeVO
+import org.zstack.header.volume.VolumeVO_
+import org.zstack.sdk.HostInventory
+import org.zstack.sdk.LocalStorageMigrateVolumeAction
+import org.zstack.sdk.VmInstanceInventory
+import org.zstack.sdk.VolumeInventory
+import org.zstack.storage.primary.local.LocalStorageKvmBackend
+import org.zstack.test.integration.storage.Env
+import org.zstack.test.integration.storage.primary.PrimaryStorageTest
+import org.zstack.testlib.EnvSpec
+import org.zstack.testlib.HttpError
+import org.zstack.testlib.SubCase
+
+/**
+ * Created by kayo on 2018/3/23.
+ */
+class LocalStorageVolumeMigrateFailureRollbackCase extends SubCase {
+    EnvSpec env
+
+    @Override
+    void clean() {
+        env.delete()
+    }
+
+    @Override
+    void setup() {
+        useSpring(PrimaryStorageTest.springSpec)
+    }
+
+    @Override
+    void environment() {
+        env = Env.localStorageOneVmWithOneDataVolumeEnv()
+    }
+
+    @Override
+    void test() {
+        env.create {
+            testVolumeStatusAndVmStateRollback()
+        }
+    }
+
+    void testVolumeStatusAndVmStateRollback() {
+        HostInventory hostInventory = env.inventoryByName("kvm1")
+        VmInstanceInventory vm1 = (VmInstanceInventory) env.inventoryByName("vm")
+        VolumeInventory dataVolume = vm1.getAllVolumes().find { i -> i.getUuid() != vm1.getRootVolumeUuid() }
+        detachDataVolumeFromVm {
+            uuid = dataVolume.getUuid()
+            vmUuid = vm1.getUuid()
+        }
+
+        stopVmInstance {
+            uuid = vm1.uuid
+        }
+
+        env.afterSimulator(LocalStorageKvmBackend.CHECK_MD5_PATH) { rsp ->
+            throw new HttpError(403, "on purpose")
+        }
+
+        def action = new LocalStorageMigrateVolumeAction()
+        action.sessionId = adminSession()
+        action.destHostUuid = hostInventory.uuid
+        action.volumeUuid = vm1.getRootVolumeUuid()
+        LocalStorageMigrateVolumeAction.Result ret = action.call()
+
+        assert ret.error != null
+
+        retryInSecs {
+            assert Q.New(VmInstanceVO.class).eq(VmInstanceVO_.uuid, vm1.getUuid()).eq(VmInstanceVO_.state, VmInstanceState.Stopped).isExists()
+            assert Q.New(VolumeVO.class).eq(VolumeVO_.uuid, vm1.getRootVolumeUuid()).eq(VolumeVO_.status, VolumeStatus.Ready).isExists()
+        }
+    }
+}