CC load test improvements.

Makefile

@@ -49,4 +49,4 @@ generate-e2e-client:
 .PHONY: generate-e2e-client
 
 deploy-loadtest: release
-	IMAGE_REPOSITORY=$(DOCKER_REPOSITORY) IMAGE_TAG=$(VERSION) ./hack/loadtest/deploy.sh
+	IMAGE_REPOSITORY=$(DOCKER_REPOSITORY) IMAGE_TAG=$(VERSION) ./hack/loadtest/deploy.sh $(LOADTEST_VALUES_FILE)

cmd/testserver/run.go

@@ -51,36 +51,55 @@ func run(ctx context.Context) error {
 	// Choose scenarios below by adding/removing/etc. instances of scenarios.XXX()
 	// All scenarios in the list run in parallel (but not necessarily at the same time if preparation takes different time).
 	testScenarios := []scenarios.TestScenario{
-		scenarios.CheckNodeDeletedStuck(300, logger),
+		scenarios.CheckNodeStatus(5000, logger),
 	}
 
-	var wg sync.WaitGroup
-	wg.Add(len(testScenarios))
-	errs := make(chan error, len(testScenarios))
+	logger.Info("Starting continuous test scenario execution")
 
-	for i, test := range testScenarios {
-		go func() {
-			defer wg.Done()
-			logger.Info(fmt.Sprintf("Starting test scenario %d", i))
+	iteration := 0
+	for {
+		iteration++
+		logger.Info(fmt.Sprintf("Starting iteration %d", iteration))
 
-			err := scenarios.RunScenario(ctx, test, testServer, logger, clientSet)
-			errs <- err
-		}()
-	}
+		var wg sync.WaitGroup
+		wg.Add(len(testScenarios))
+		errs := make(chan error, len(testScenarios))
 
-	logger.Info("Waiting for test scenarios to finish")
-	wg.Wait()
+		for i, test := range testScenarios {
+			go func(scenarioIndex int, scenario scenarios.TestScenario) {
+				defer wg.Done()
+				logger.Info(fmt.Sprintf("Starting test scenario %d in iteration %d", scenarioIndex, iteration))
 
-	close(errs)
-	receivedErrors := make([]error, 0)
-	for err := range errs {
-		if err != nil {
-			receivedErrors = append(receivedErrors, err)
+				err := scenarios.RunScenario(ctx, scenario, testServer, logger, clientSet)
+				errs <- err
+			}(i, test)
+		}
+
+		logger.Info(fmt.Sprintf("Waiting for test scenarios to finish in iteration %d", iteration))
+		wg.Wait()
+
+		close(errs)
+		receivedErrors := make([]error, 0)
+		for err := range errs {
+			if err != nil {
+				receivedErrors = append(receivedErrors, err)
+			}
+		}
+
+		if len(receivedErrors) > 0 {
+			logger.Error(fmt.Sprintf("Iteration %d completed with (%d) errors: %v", iteration, len(receivedErrors), errors.Join(receivedErrors...)))
+		} else {
+			logger.Info(fmt.Sprintf("Iteration %d completed successfully", iteration))
 		}
-	}
-	logger.Info(fmt.Sprintf("All test scenarios are done, received (%d) errors, exiting", len(receivedErrors)))
 
-	return errors.Join(receivedErrors...)
+		logger.Info("Waiting 1 minute before next iteration")
+		select {
+		case <-time.After(5 * time.Minute):
+		case <-ctx.Done():
+			logger.Info("Context canceled, stopping test scenarios")
+			return ctx.Err()
+		}
+	}
 }
 
 func createK8SClients(cfg loadtest.Config, logger *slog.Logger) (*kubernetes.Clientset, *dynamic.DynamicClient, *apiextensionsclientset.Clientset, helm.Client, error) {

hack/loadtest/deploy.sh

@@ -1,5 +1,8 @@
 #!/usr/bin/env bash
 
+# Optional values file can be passed as first argument
+VALUES_FILE="${1:-}"
+
 CC_IMAGE_REPOSITORY="${IMAGE_REPOSITORY:-us-docker.pkg.dev/castai-hub/library/cluster-controller}"
 CC_IMAGE_TAG="${IMAGE_TAG:-latest}"
 LOAD_TEST_IMAGE_REPOSITORY="${LOAD_TEST_IMAGE_REPOSITORY:-$CC_IMAGE_REPOSITORY}"
@@ -14,23 +17,35 @@ echo "Deploying kwok"
 helm repo add kwok https://kwok.sigs.k8s.io/charts/
 helm repo update kwok
 
-helm upgrade --namespace castai-agent --create-namespace  --install kwok kwok/kwok --set replicas="$KWOK_REPLICAS"
-helm upgrade --namespace castai-agent --create-namespace  --install kwok-stages kwok/stage-fast
-helm upgrade --namespace castai-agent --create-namespace  --install kwok-metrics kwok/metrics-usage
+helm upgrade --namespace castai-agent --create-namespace --install kwok kwok/kwok --set replicas="$KWOK_REPLICAS"
+helm upgrade --namespace castai-agent --create-namespace --install kwok-stages kwok/stage-fast
+helm upgrade --namespace castai-agent --create-namespace --install kwok-metrics kwok/metrics-usage
 
 if [ "$DEPLOY_CLUSTER_CONTROLLER" = "true" ]; then
   echo "Deploying cluster controller"
-  helm upgrade --namespace castai-agent --create-namespace --install  cluster-controller castai-helm/castai-cluster-controller \
-    --set castai.apiKey="dummy" \
-    --set castai.apiURL="http://castai-loadtest-agent-service.castai-agent.svc.cluster.local.:8080" \
-    --set castai.clusterID="00000000-0000-0000-0000-000000000000" \
-    --set image.repository="$CC_IMAGE_REPOSITORY" \
-    --set image.tag="$CC_IMAGE_TAG" \
-    --set image.pullPolicy="Always" \
+  HELM_ARGS=(
+    --namespace castai-agent
+    --create-namespace
+    --install cluster-controller
+    castai-helm/castai-cluster-controller
+    --set castai.apiKey="dummy"
+    --set castai.apiURL="http://castai-loadtest-agent-service.castai-agent.svc.cluster.local.:8080"
+    --set castai.clusterID="00000000-0000-0000-0000-000000000000"
+    --set image.repository="$CC_IMAGE_REPOSITORY"
+    --set image.tag="$CC_IMAGE_TAG"
+    --set image.pullPolicy="Always"
     --set autoscaling.enabled="true"
+  )
+
+  if [ -n "$VALUES_FILE" ]; then
+    echo "Using values file: $VALUES_FILE"
+    HELM_ARGS+=(--values "$VALUES_FILE")
+  fi
+
+  helm upgrade "${HELM_ARGS[@]}"
 fi
 
 echo "Deploying load testing components"
-kubectl kustomize "$SCRIPT_DIR" | \
-  LOADTEST_REPOSITORY="$LOAD_TEST_IMAGE_REPOSITORY" LOADTEST_TAG="$LOAD_TEST_IMAGE_TAG" envsubst \$LOADTEST_REPOSITORY,\$LOADTEST_TAG | \
+kubectl kustomize "$SCRIPT_DIR" |
+  LOADTEST_REPOSITORY="$LOAD_TEST_IMAGE_REPOSITORY" LOADTEST_TAG="$LOAD_TEST_IMAGE_TAG" envsubst \$LOADTEST_REPOSITORY,\$LOADTEST_TAG |
   kubectl apply -f -