feat: support cancelling restores

Author: shreddedbacon

controller-test.sh

@@ -444,7 +444,7 @@ echo '
 {"properties":{"delivery_mode":2},"routing_key":"ci-local-controller-kubernetes:misc",
     "payload":"{
         \"misc\":{
-            \"miscResource\":\"eyJtZXRhZGF0YSI6eyJuYW1lIjoicmVzdG9yZS1iZjA3MmEwLXVxeHFvNCJ9LCJzcGVjIjp7InNuYXBzaG90IjoiYmYwNzJhMDllMTc3MjZkYTU0YWRjNzk5MzZlYzg3NDU1MjE5OTM1OTlkNDEyMTFkZmM5NDY2ZGZkNWJjMzJhNSIsInJlc3RvcmVNZXRob2QiOnsiczMiOnt9fSwiYmFja2VuZCI6eyJzMyI6eyJidWNrZXQiOiJiYWFzLW5naW54LWV4YW1wbGUifSwicmVwb1Bhc3N3b3JkU2VjcmV0UmVmIjp7ImtleSI6InJlcG8tcHciLCJuYW1lIjoiYmFhcy1yZXBvLXB3In19fX0=\"
+            \"miscResource\":\"eyJtZXRhZGF0YSI6eyJuYW1lIjoicmVzdG9yZS1iZjA3MmEwOWUxNzcyNmRhNTRhZGM3OTkzNmVjODc0NTUyMTk5MzU5OWQ0MTIxMWRmYzk0NjZkZmQ1YmMzMmE1In0sInNwZWMiOnsic25hcHNob3QiOiJiZjA3MmEwOWUxNzcyNmRhNTRhZGM3OTkzNmVjODc0NTUyMTk5MzU5OWQ0MTIxMWRmYzk0NjZkZmQ1YmMzMmE1IiwicmVzdG9yZU1ldGhvZCI6eyJzMyI6e319LCJiYWNrZW5kIjp7InMzIjp7ImJ1Y2tldCI6ImJhYXMtbmdpbngtZXhhbXBsZSJ9LCJyZXBvUGFzc3dvcmRTZWNyZXRSZWYiOnsia2V5IjoicmVwby1wdyIsIm5hbWUiOiJiYWFzLXJlcG8tcHcifX19fQ==\"
         },
         \"key\":\"deploytarget:restic:backup:restore\",
         \"environment\":{
@@ -463,7 +463,7 @@ echo ""
 sleep 10
 CHECK_COUNTER=1
 kubectl -n nginx-example-main get restores.k8up.io
-until $(kubectl -n nginx-example-main get restores.k8up.io restore-bf072a0-uqxqo4 &> /dev/null)
+until $(kubectl -n nginx-example-main get restores.k8up.io restore-bf072a09e17726da54adc79936ec8745521993599d41211dfc9466dfd5bc32a5 &> /dev/null)
 do
 if [ $CHECK_COUNTER -lt 14 ]; then
     let CHECK_COUNTER=CHECK_COUNTER+1
@@ -478,7 +478,7 @@ else
     exit 1
 fi
 done
-kubectl -n nginx-example-main get restores.k8up.io restore-bf072a0-uqxqo4 -o yaml | kubectl-neat > test-resources/results/k8upv1-cluster.yaml
+kubectl -n nginx-example-main get restores.k8up.io restore-bf072a09e17726da54adc79936ec8745521993599d41211dfc9466dfd5bc32a5 -o yaml | kubectl-neat > test-resources/results/k8upv1-cluster.yaml
 if cmp --silent -- "test-resources/results/k8upv1.yaml" "test-resources/results/k8upv1-cluster.yaml"; then
     echo "Resulting restores match"
 else
@@ -495,6 +495,46 @@ else
     exit 1
 fi
 
+# test that a cancellation works when a cancellation message is received from rabbitmq, and that the restore resource is deleted from the namespace
+echo "==> Trigger a lagoon restore cancellation using rabbitmq"
+echo '
+{"properties":{"delivery_mode":2},"routing_key":"ci-local-controller-kubernetes:misc",
+    "payload":"{
+        \"misc\":{
+            \"miscResource\":\"eyJyZXN0b3JlTmFtZSI6InJlc3RvcmUtYmYwNzJhMDllMTc3MjZkYTU0YWRjNzk5MzZlYzg3NDU1MjE5OTM1OTlkNDEyMTFkZmM5NDY2ZGZkNWJjMzJhNSIsImJhY2t1cElkIjoiYmYwNzJhMDllMTc3MjZkYTU0YWRjNzk5MzZlYzg3NDU1MjE5OTM1OTlkNDEyMTFkZmM5NDY2ZGZkNWJjMzJhNSJ9\"
+        },
+        \"key\":\"deploytarget:restic:cancel:restore\",
+        \"environment\":{
+            \"name\":\"main\",
+            \"openshiftProjectName\":\"nginx-example-main\"
+        },
+        \"project\":{
+            \"name\":\"nginx-example\"
+        },
+        \"advancedTask\":{}
+    }",
+"payload_encoding":"string"
+}' >payload.json
+curl -s -u guest:guest -H "Accept: application/json" -H "Content-Type:application/json" -X POST -d @payload.json http://172.17.0.1:15672/api/exchanges/%2f/lagoon-tasks/publish
+echo ""
+sleep 10
+# check that the restore resource gets removed
+until ! $(kubectl -n nginx-example-main get restores.k8up.io restore-bf072a09e17726da54adc79936ec8745521993599d41211dfc9466dfd5bc32a5 &> /dev/null)
+do
+if [ $CHECK_COUNTER -lt 14 ]; then
+    let CHECK_COUNTER=CHECK_COUNTER+1
+    echo "Restore not deleted yet"
+    sleep 5
+else
+    echo "Timeout of 70seconds for restore to be deleted"
+    check_controller_log
+    tear_down
+    echo "================ END ================"
+    echo "============== FAILED ==============="
+    exit 1
+fi
+done
+
 echo "==> Delete the environment"
 echo '
 {"properties":{"delivery_mode":2},"routing_key":"ci-local-controller-kubernetes:remove",

internal/helpers/helper_types.go

@@ -1,5 +1,13 @@
 package helpers
 
+import (
+	"context"
+
+	apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
 // LagoonEnvironmentVariable is used to define Lagoon environment variables.
 type LagoonEnvironmentVariable struct {
 	Name  string `json:"name"`
@@ -31,3 +39,27 @@ type LagoonAPIConfiguration struct {
 	SSHHost   string
 	SSHPort   string
 }
+
+func K8UPVersions(ctx context.Context, c client.Client) (bool, bool, error) {
+	k8upv1alpha1Exists := false
+	k8upv1Exists := false
+	crdv1alpha1 := &apiextensionsv1.CustomResourceDefinition{}
+	if err := c.Get(context.TODO(), types.NamespacedName{Name: "restores.backup.appuio.ch"}, crdv1alpha1); err != nil {
+		if err := IgnoreNotFound(err); err != nil {
+			return k8upv1alpha1Exists, k8upv1Exists, err
+		}
+	}
+	if crdv1alpha1.ObjectMeta.Name == "restores.backup.appuio.ch" {
+		k8upv1alpha1Exists = true
+	}
+	crdv1 := &apiextensionsv1.CustomResourceDefinition{}
+	if err := c.Get(context.TODO(), types.NamespacedName{Name: "restores.k8up.io"}, crdv1); err != nil {
+		if err := IgnoreNotFound(err); err != nil {
+			return k8upv1alpha1Exists, k8upv1Exists, err
+		}
+	}
+	if crdv1.ObjectMeta.Name == "restores.k8up.io" {
+		k8upv1Exists = true
+	}
+	return k8upv1alpha1Exists, k8upv1Exists, nil
+}

internal/messenger/consumer.go

@@ -336,14 +336,25 @@ func (m *Messenger) Consumer(targetName string) { //error {
 					return
 				}
 			case "deploytarget:restic:backup:restore", "kubernetes:restic:backup:restore":
+				v1alpha1, v1, err := helpers.K8UPVersions(ctx, m.Client)
+				if err != nil {
+					//@TODO: send msg back to lagoon and update task to failed?
+					message.Ack(false) // ack to remove from queue
+					return
+				}
+				if !v1alpha1 && !v1 {
+					// k8up not installed
+					message.Ack(false) // ack to remove from queue
+					return
+				}
 				opLog.Info(
 					fmt.Sprintf(
 						"Received backup restoration for project %s, environment %s",
 						jobSpec.Project.Name,
 						jobSpec.Environment.Name,
 					),
 				)
-				err := m.ResticRestore(namespace, jobSpec)
+				err = m.ResticRestore(ctx, namespace, jobSpec, v1alpha1, v1, false)
 				if err != nil {
 					opLog.Error(err,
 						fmt.Sprintf(
@@ -356,6 +367,39 @@ func (m *Messenger) Consumer(targetName string) { //error {
 					message.Ack(false) // ack to remove from queue
 					return
 				}
+			case "deploytarget:restic:cancel:restore":
+				v1alpha1, v1, err := helpers.K8UPVersions(ctx, m.Client)
+				if err != nil {
+					//@TODO: send msg back to lagoon and update task to failed?
+					message.Ack(false) // ack to remove from queue
+					return
+				}
+				if !v1alpha1 && !v1 {
+					// k8up not installed
+					message.Ack(false) // ack to remove from queue
+					return
+				}
+				// if this is a request to cancel a restore attempt
+				opLog.Info(
+					fmt.Sprintf(
+						"Received restore cancellation for project %s, environment %s",
+						jobSpec.Project.Name,
+						jobSpec.Environment.Name,
+					),
+				)
+				err = m.ResticRestore(ctx, namespace, jobSpec, v1alpha1, v1, true)
+				if err != nil {
+					opLog.Error(err,
+						fmt.Sprintf(
+							"Cancel restore for project %s, environment %s failed",
+							jobSpec.Project.Name,
+							jobSpec.Environment.Name,
+						),
+					)
+					//@TODO: send msg back to lagoon and update task to failed?
+					message.Ack(false) // ack to remove from queue
+					return
+				}
 			case "deploytarget:route:migrate", "kubernetes:route:migrate", "openshift:route:migrate":
 				opLog.Info(
 					fmt.Sprintf(