[core] handle force destroy request

Author: miltalex

core/environment/manager.go

@@ -118,7 +118,7 @@ func (envs *Manager) CreateEnvironment(workflowPath string, userVars map[string]
 	return env.id, err
 }
 
-func (envs *Manager) TeardownEnvironment(environmentId uuid.UUID) error {
+func (envs *Manager) TeardownEnvironment(environmentId uuid.UUID, force bool) error {
 	envs.mu.Lock()
 	defer envs.mu.Unlock()
 
@@ -127,7 +127,7 @@ func (envs *Manager) TeardownEnvironment(environmentId uuid.UUID) error {
 		return err
 	}
 
-	if env.CurrentState() != "STANDBY" {
+	if env.CurrentState() != "STANDBY" && !force {
 		return errors.New(fmt.Sprintf("cannot teardown environment in state %s", env.CurrentState()))
 	}
 

core/server.go

@@ -299,6 +299,24 @@ func (m *RpcServer) DestroyEnvironment(cxt context.Context, req *pb.DestroyEnvir
 		return nil, status.Newf(codes.NotFound, "environment not found: %s", err.Error()).Err()
 	}
 
+	// if Force immediately disband the environment (unlocking all tasks) and run the cleanup. 
+	if req.Force {
+		err = m.state.environments.TeardownEnvironment(env.Id(), req.Force)
+		if err != nil {
+			return &pb.DestroyEnvironmentReply{}, status.New(codes.Internal, err.Error()).Err()
+		}
+
+		tasksForEnv := env.Workflow().GetTasks().GetTaskIds()
+		killed, running, err := m.doCleanupTasks(tasksForEnv)
+		ctr := &pb.CleanupTasksReply{KilledTasks: killed, RunningTasks: running}
+		if err != nil {
+			log.WithError(err).Error("task cleanup error")
+			return &pb.DestroyEnvironmentReply{CleanupTasksReply: ctr}, status.New(codes.Internal, err.Error()).Err()
+		}
+
+		return &pb.DestroyEnvironmentReply{CleanupTasksReply: ctr}, nil
+	}
+
 	if req.AllowInRunningState && env.CurrentState() == "RUNNING" {
 		err = env.TryTransition(environment.MakeTransition(m.state.taskman, pb.ControlEnvironmentRequest_STOP_ACTIVITY))
 		if err != nil {
@@ -328,7 +346,7 @@ func (m *RpcServer) DestroyEnvironment(cxt context.Context, req *pb.DestroyEnvir
 		}
 	}
 
-	err = m.state.environments.TeardownEnvironment(env.Id())
+	err = m.state.environments.TeardownEnvironment(env.Id(), req.Force)
 	if err != nil {
 		return &pb.DestroyEnvironmentReply{}, status.New(codes.Internal, err.Error()).Err()
 	}

core/signals.go

@@ -99,7 +99,7 @@ func manageKillSignals(state *internalState) {
 		}
 
 		// Teardown Enviroment
-		err = state.environments.TeardownEnvironment(uid)
+		err = state.environments.TeardownEnvironment(uid, false)
 		if err != nil {
 			log.WithPrefix("termination").WithError(err).Error(fmt.Sprintf("cannot teardown enviroment %s", uid.String()))
 		}