From dbb07d7d1aa3f2510308a185e6aa3e86f8bbe33f Mon Sep 17 00:00:00 2001
From: Maksim Fedotov <maksim.fedotov@flant.com>
Date: Wed, 2 Apr 2025 11:56:50 +0300
Subject: [PATCH 01/48] feat(ci): add automatic e2e tests

Signed-off-by: Maksim Fedotov <maksim.fedotov@flant.com>
---
 .github/scripts/js/ci.js                |  28 +++
 .github/scripts/js/constants.js         | 110 ++++++++++
 .github/scripts/js/e2e-commit-status.js | 264 ++++++++++++++++++++++++
 .github/workflows/dev_module_build.yml  |  38 +++-
 .github/workflows/run_e2e.yml           | 178 ++++++++++++++++
 5 files changed, 617 insertions(+), 1 deletion(-)
 create mode 100644 .github/scripts/js/constants.js
 create mode 100644 .github/scripts/js/e2e-commit-status.js
 create mode 100644 .github/workflows/run_e2e.yml

diff --git a/.github/scripts/js/ci.js b/.github/scripts/js/ci.js
index 91c11372f7..67b97e7ed2 100644
--- a/.github/scripts/js/ci.js
+++ b/.github/scripts/js/ci.js
@@ -25,6 +25,9 @@
  * @returns {object}
  */
 
+const {
+	userClusterLabels
+  } = require('./constants');
 const { dumpError } = require('./error');
 const extractCommandFromComment = (comment) => {
 	// Split comment to lines.
@@ -65,6 +68,31 @@ const reactToComment = async ({github, context, comment_id, content}) => {
 	});
 };
 module.exports.reactToComment = reactToComment;
+
+const checkUserClusterLabel = async ({prLabels}) => {
+    const userLabelsInPR = prLabels
+        .map(label => label.name)
+        .filter(labelName => userClusterLabels[labelName]);
+    return userLabelsInPR;
+};
+module.exports.checkUserClusterLabel = checkUserClusterLabel;
+
+const getClusterUser = async ({context, core}) => {
+  const prLabels = context.payload.pull_request.labels;
+  let userLabelsInPR = await checkUserClusterLabel({prLabels});
+  if (userLabelsInPR.length === 0) {
+    core.info('No user labels found in PR, using PR author\'s cluster');
+    const prAuthorId = context.payload.pull_request.user.id;
+    core.info(`PR author: ${prAuthorId}`);
+    return prAuthorId.toString();
+    // const authorLabel = [{'name': `e2e/user/${prAuthor}`}];
+    // retry check for PR author's cluster label
+  } else if (userLabelsInPR.length > 1) {
+    return core.setFailed(`Error: PR has multiple user labels: ${userLabelsInPR.join(', ')}`);
+  }
+  return userClusterLabels[userLabelsInPR].id
+};
+module.exports.getClusterUser = getClusterUser;
   
 /**
  * Start workflow using workflow_dispatch event.
diff --git a/.github/scripts/js/constants.js b/.github/scripts/js/constants.js
new file mode 100644
index 0000000000..4633e50114
--- /dev/null
+++ b/.github/scripts/js/constants.js
@@ -0,0 +1,110 @@
+// Copyright 2022 Flant JSC
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//     http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+//@ts-check
+
+const skipE2eLabel = 'skip/e2e';
+const abortFailedE2eCommand = '/e2e/abort';
+module.exports.skipE2eLabel = skipE2eLabel;
+module.exports.abortFailedE2eCommand = abortFailedE2eCommand;
+
+// Labels available for pull requests.
+const labels = {
+  // E2E
+  'e2e/run': { type: 'e2e-run', provider: 'static' },
+  // Allow running workflows for external PRs.
+  'status/ok-to-test': { type: 'ok-to-test' },
+
+};
+module.exports.knownLabels = labels;
+const userClusterLables = {
+  'e2e/user/nevermarine' : { type: 'e2e-user', id: '72984806'},
+  'e2e/user/Isteb4k' : { type: 'e2e-user', id: '93128416'}
+};
+module.exports.userClusterLabels = userClusterLables;
+
+// Label to detect if issue is a release issue.
+const releaseIssueLabel = 'issue/release';
+module.exports.releaseIssueLabel = releaseIssueLabel;
+
+const slashCommands = {
+  deploy: ['deploy/alpha', 'deploy/beta', 'deploy/early-access', 'deploy/stable', 'deploy/rock-solid'],
+  suspend: ['suspend/alpha', 'suspend/beta', 'suspend/early-access', 'suspend/stable', 'suspend/rock-solid']
+};
+module.exports.knownSlashCommands = slashCommands;
+
+module.exports.labelsSrv = {
+  /**
+   * Search for known label name using its type and property:
+   * - search by provider property for e2e-run labels
+   * - search by env property for deploy-web labels
+   *
+   * @param {object} inputs
+   * @param {string} inputs.labelType
+   * @param {string} inputs.labelSubject
+   * @returns {string}
+   */
+  findLabel: ({ labelType, labelSubject }) => {
+    return (Object.entries(labels).find(([name, info]) => {
+      if (info.type === labelType) {
+        if (labelType === 'e2e-run') {
+          return info.provider === labelSubject;
+        }
+        if (labelType === 'deploy-web') {
+          return info.env === labelSubject;
+        }
+
+        return true;
+      }
+      return false;
+    }) || [''])[0];
+  }
+};
+
+// Providers for e2e tests.
+const providers = Object.entries(labels)
+  .filter(([name, info]) => info.type === 'e2e-run')
+  .map(([name, info]) => info.provider)
+  .sort();
+module.exports.knownProviders = providers;
+
+// Channels available for deploy.
+const channels = [
+  //
+  'alpha',
+  'beta',
+  'early-access',
+  'stable',
+  'rock-solid'
+];
+
+module.exports.knownChannels = channels;
+
+const criNames = Object.entries(labels)
+  .filter(([name, info]) => info.type === 'e2e-use' && !!info.cri)
+  .map(([name, info]) => info.cri);
+module.exports.knownCRINames = criNames;
+
+const kubernetesVersions = Object.entries(labels)
+  .filter(([name, info]) => info.type === 'e2e-use' && !!info.ver)
+  .map(([name, info]) => info.ver)
+  .sort();
+module.exports.knownKubernetesVersions = kubernetesVersions;
+
+module.exports.e2eDefaults = {
+  criName: 'Containerd',
+  edition: 'FE',
+  multimaster: false,
+  cis: false
+};
+
+const editions = ['CE', 'EE', 'FE', 'BE', 'SE', 'SE-plus'];
+module.exports.knownEditions = editions;
diff --git a/.github/scripts/js/e2e-commit-status.js b/.github/scripts/js/e2e-commit-status.js
new file mode 100644
index 0000000000..41464b4f80
--- /dev/null
+++ b/.github/scripts/js/e2e-commit-status.js
@@ -0,0 +1,264 @@
+// Copyright 2022 Flant JSC
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//     http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+const {
+  sleep
+} = require('./time');
+
+/**
+ * Build workflow run url for add it to commit status target url
+ * @param {object} inputs
+ * @param {object} inputs.core - A reference to the '@actions/core' package.
+ * @param {object} inputs.context - A reference to context https://github.com/actions/toolkit/blob/main/packages/github/src/context.ts#L6
+ * @returns string
+ */
+function workflowUrl({core, context}) {
+  core.debug(`workflowUrl context: ${JSON.stringify(context)}`);
+  const {serverUrl, repo, runId} = context;
+  const repository = repo.repo;
+  const owner = repo.owner;
+  const url = `${serverUrl}/${owner}/${repository}/actions/runs/${runId}`;
+  core.debug(`workflowUrl url: ${url}`);
+  return url
+}
+
+/**
+ * Wrap github.rest.repos.createCommitStatus. Returns true if status was set
+ * Use STATUS_TARGET_COMMIT env var as target commit sha
+ * @param {object} inputs
+ * @param {object} inputs.core - A reference to the '@actions/core' package.
+ * @param {object} inputs.github - A pre-authenticated octokit/rest.js client with pagination plugins.
+ * @param {object} inputs.context - A reference to context https://github.com/actions/toolkit/blob/main/packages/github/src/context.ts#L6
+ * @param {object} inputs.status - A state object for send
+ * @param {string} inputs.status.state - A state type as 'success' (it is mark in GitHub ui)
+ * @param {string} inputs.status.description - A description for commit status
+ * @param {string|undefined} inputs.status.url - A target url for commit status (Details link in GitHub ui)
+ * @param {string} inputs.status.commitSha - A commit for set status
+ * @returns Promise<bool>
+ */
+async function sendCreateCommitStatus({github, context, core, status}) {
+  const {state, description, url, commitSha} = status
+  core.debug(`sendCreateCommitStatus target commit: ${commitSha}`);
+
+  for(let i = 0; i < 3; i++) {
+    const response = await github.rest.repos.createCommitStatus({
+      owner: context.repo.owner,
+      repo: context.repo.repo,
+      sha: commitSha,
+      state: state,
+      description: description,
+      target_url: url,
+      context: 'E2e test'
+    });
+
+    core.debug(`rest.repos.createCommitStatus response: ${JSON.stringify(response)}`);
+    if (response.status === 201) {
+      core.debug(`rest.repos.createCommitStatus response status is 201. Returns true`);
+      return true;
+    }
+
+    // wait 3s for retry request
+    await sleep(3000);
+  }
+
+  return false
+}
+
+/**
+ * Set `waiting for start e2e` status (pending) Uses with push commit
+ * Use STATUS_TARGET_COMMIT env var as target commit sha
+ * @param {object} inputs
+ * @param {object} inputs.core - A reference to the '@actions/core' package.
+ * @param {object} inputs.github - A pre-authenticated octokit/rest.js client with pagination plugins.
+ * @param {object} inputs.context - A reference to context https://github.com/actions/toolkit/blob/main/packages/github/src/context.ts#L6
+ * @param {string} inputs.commitSha - sha commit for set status
+ * @returns Promise<bool>
+ */
+async function setWait ({github, context, core, commitSha}) {
+  return sendCreateCommitStatus({
+    github,
+    context,
+    core,
+    status: {
+      commitSha,
+      state: 'pending',
+      description: 'Waiting for run e2e test'
+    }
+  })
+}
+
+/**
+ * Set `e2e was failed` status (failed) when e2e was failed
+ * Use STATUS_TARGET_COMMIT env var as target commit sha
+ * @param {object} inputs
+ * @param {object} inputs.core - A reference to the '@actions/core' package.
+ * @param {object} inputs.github - A pre-authenticated octokit/rest.js client with pagination plugins.
+ * @param {object} inputs.context - A reference to context https://github.com/actions/toolkit/blob/main/packages/github/src/context.ts#L6
+ * @param {string} inputs.commitSha - sha commit for set status
+ * @returns Promise<bool>
+ */
+async function setFail({github, context, core, commitSha}){
+  return sendCreateCommitStatus({
+    github,
+    context,
+    core,
+    status: {
+      commitSha,
+      state: 'failure',
+      description: 'E2e test was failed',
+      url: workflowUrl({core, context}),
+    }
+  })
+}
+
+/**
+ * Set `e2e was passed` status (failed) when e2e was failed
+ * Use STATUS_TARGET_COMMIT env var as target commit sha
+ * @param {object} inputs
+ * @param {object} inputs.core - A reference to the '@actions/core' package.
+ * @param {object} inputs.github - A pre-authenticated octokit/rest.js client with pagination plugins.
+ * @param {object} inputs.context - A reference to context https://github.com/actions/toolkit/blob/main/packages/github/src/context.ts#L6
+ * @param {string} inputs.commitSha - sha commit for set status
+ * @returns Promise<bool>
+ */
+function setSuccess ({github, context, core, commitSha}) {
+  return sendCreateCommitStatus({
+    github,
+    context,
+    core,
+    status: {
+      commitSha,
+      state: 'success',
+      description: 'E2e test was passed',
+      url: workflowUrl({core, context}),
+    }
+  })
+}
+
+/**
+ * Set `e2e was failed` status (success) when e2e was failed
+ * Unfortunately we do not have 'skip' status and use 'success'
+ * Use STATUS_TARGET_COMMIT env var as target commit sha
+ * @param {object} inputs
+ * @param {object} inputs.core - A reference to the '@actions/core' package.
+ * @param {object} inputs.github - A pre-authenticated octokit/rest.js client with pagination plugins.
+ * @param {object} inputs.context - A reference to context https://github.com/actions/toolkit/blob/main/packages/github/src/context.ts#L6
+ * @param {string} inputs.commitSha - sha commit for set status
+ * @returns Promise<bool>
+ */
+async function setSkip({github, context, core, commitSha}){
+  return sendCreateCommitStatus({
+    github,
+    context,
+    core,
+    status: {
+      commitSha,
+      state: 'success',
+      description: 'E2e test was skipped',
+    },
+  })
+}
+
+/**
+ * Set commit status when label set/unset.
+ * Check label for skipping e2e test and e2e tests should skip set success status
+ * If status was not set then fail job
+ *
+ * Used in build-and-test_dev workflow
+ *
+ * Use STATUS_TARGET_COMMIT env var as target commit sha
+ * @param {object} inputs
+ * @param {object} inputs.core - A reference to the '@actions/core' package.
+ * @param {object} inputs.github - A pre-authenticated octokit/rest.js client with pagination plugins.
+ * @param {object} inputs.context - A reference to context https://github.com/actions/toolkit/blob/main/packages/github/src/context.ts#L6
+ * @param {boolean} inputs.labeled - true - PR was labeled, false - unlabeled
+ * @param {string} inputs.commitSha - sha commit for set status
+ * @returns Promise<void>
+ */
+async function onLabeledForSkip({github, context, core, labeled, commitSha}) {
+  const statusSetFunc = (labeled) ? setSkip : setWait;
+
+  const done = await statusSetFunc({github, context, core, commitSha});
+  if (!done) {
+    core.setFailed('e2e requirement status was not set.');
+  }
+}
+
+/**
+ * Set commit status when commit was pushed.
+ * Check label for skipping e2e test and e2e tests should skip set success status
+ * If status was not set then fail job
+ *
+ * Used in e2e_run* workflow
+ *
+ * Use STATUS_TARGET_COMMIT env var as target commit sha
+ * Use STATUS_TARGET_COMMIT env var as job status
+ * Use  env var as target commit sha
+ * @param {object} inputs
+ * @param {object} inputs.core - A reference to the '@actions/core' package.
+ * @param {object} inputs.github - A pre-authenticated octokit/rest.js client with pagination plugins.
+ * @param {object} inputs.context - A reference to context https://github.com/actions/toolkit/blob/main/packages/github/src/context.ts#L6
+ * @returns Promise<void>
+ */
+async function setStatusAfterE2eRun({github, context, core}) {
+  const jobStatus = process.env.JOB_STATUS;
+  const commitSha = process.env.STATUS_TARGET_COMMIT;
+
+  let setStateFunc = null;
+  if (jobStatus === 'failure' || jobStatus === 'cancelled') {
+    setStateFunc = setFail;
+  } else if (jobStatus === 'success') {
+    setStateFunc = await setSuccess;
+  } else {
+    core.setFailed(`e2e requirement status was not set. Job status ${jobStat}`)
+    return
+  }
+
+  const success = setStateFunc({github, context, core, commitSha})
+  if (!success) {
+    core.setFailed(`e2e requirement status was not set. Job status ${jobStat}`)
+  }
+}
+
+/**
+ * Set commit status when commit was pushed.
+ * Check label for skipping e2e test and e2e tests should skip set success status
+ * If status was not set then fail job
+ *
+ * Used in build-and-test_dev workflow
+ *
+ * Use STATUS_TARGET_COMMIT env var as target commit sha
+ * Use PR_LABELS env var as list of PR labels
+ * @param {object} inputs
+ * @param {object} inputs.core - A reference to the '@actions/core' package.
+ * @param {object} inputs.github - A pre-authenticated octokit/rest.js client with pagination plugins.
+ * @param {object} inputs.context - A reference to context https://github.com/actions/toolkit/blob/main/packages/github/src/context.ts#L6
+ * @returns Promise<void>
+ */
+async function setInitialStatus ({github, context, core}) {
+  core.info(`Labels json: ${process.env.PR_LABELS}`);
+
+  const labels = JSON.parse(process.env.PR_LABELS);
+  const commitSha = process.env.STATUS_TARGET_COMMIT;
+
+  core.debug(`Labels: ${labels ? JSON.stringify(labels.map((l) => l.name)) : 'no labels'}`);
+
+  const shouldSkip = labels ? labels.some((l) => l.name === "skip/e2e") : false;
+  core.debug(`Should skip e2e: ${shouldSkip}`);
+
+  return onLabeledForSkip({github, context, core, labeled: shouldSkip, commitSha})
+}
+
+module.exports = {
+  setStatusAfterE2eRun,
+  setInitialStatus,
+  onLabeledForSkip
+}
diff --git a/.github/workflows/dev_module_build.yml b/.github/workflows/dev_module_build.yml
index cc1c4ecb8a..956fc42784 100644
--- a/.github/workflows/dev_module_build.yml
+++ b/.github/workflows/dev_module_build.yml
@@ -336,7 +336,43 @@ jobs:
           module_source: ${{ vars.DEV_MODULE_SOURCE}}
           module_name: ${{ vars.MODULE_NAME }}
           module_tag: "$MODULES_MODULE_TAG"
-
+  pull_request_info:
+    name: Get PR info
+    runs-on: ubuntu-latest
+    outputs:
+      labels: ${{ steps.pr_labels.outputs.labels }}
+    steps:
+      - name: Get PR labels
+        id: pr_labels
+        uses: actions/github-script@v6.4.1
+        with:
+          script: |
+            const prNumber = context.payload.pull_request.number;
+            const { data: labels } = await github.rest.issues.listLabelsOnIssue({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              issue_number: prNumber
+            });
+            core.setOutput('labels', JSON.stringify(labels));
+  set_e2e_requirement_status:
+    name: Set 'waiting for e2e' commit status
+    needs:
+      - pull_request_info
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@v3.5.2
+      - name: Set commit status after e2e run
+        id: set_e2e_requirement_status
+        uses: actions/github-script@v6.4.1
+        env:
+          STATUS_TARGET_COMMIT: ${{ github.event.pull_request.head.sha }}
+          PR_LABELS: ${{ needs.pull_request_info.outputs.labels }}
+        with:
+          github-token: ${{secrets.CHANGELOG_ACCESS_TOKEN}}
+          script: |
+            const e2eStatus = require('./.github/scripts/js/e2e-commit-status');
+            await e2eStatus.setInitialStatus({github, context, core});
   cve_scan_on_pr:
     name: Trivy images check
     runs-on: ${{ fromJSON(needs.set_vars.outputs.runner_type)}}
diff --git a/.github/workflows/run_e2e.yml b/.github/workflows/run_e2e.yml
new file mode 100644
index 0000000000..f15989b8e9
--- /dev/null
+++ b/.github/workflows/run_e2e.yml
@@ -0,0 +1,178 @@
+name: Run e2e
+on:
+  pull_request_target:
+    types: 
+      - labeled
+
+env:
+  GO_VERSION: "1.22.7"
+
+jobs:
+  skip_e2e:
+    if: ${{ github.event.label.name == 'skip/e2e' }}
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Skip E2E tests
+        id: skip_e2e
+        uses: actions/github-script@v6
+        with:
+          script: |
+            const e2eStatus = require('./.github/scripts/js/e2e-commit-status');
+            e2eStatus.onLabeledForSkip({
+              github,
+              context,
+              core,
+              labeled: true,
+              commitSha: context.payload.pull_request.head.sha
+            })
+
+  run_e2e:
+    if: ${{ github.event.label.name == 'e2e/run' }}
+    name: Run E2E tests
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Select user
+        id: select_user
+        uses: actions/github-script@v6
+        env:
+          KUBECONFIGS: ${{ secrets.K8S_CLUSTER_SECRET }}
+        with:
+          script: |
+            const ci = require('./.github/scripts/js/ci');
+            const userId = await ci.getClusterUser({context, core})
+            // core.setOutput("user_id", user_id)
+            // core.exportVariable("user_id", user_id);
+            const kubeconfigs = JSON.parse(process.env.KUBECONFIGS);
+            const kubeconfig = kubeconfigs.find(config => config.id === userId)?.kubeconfig;
+            if (!kubeconfig) {
+              core.setFailed(`No kubeconfig found for user with ID ${userId}.`);
+            } else {
+              core.info(`Found kubeconfig for user with ID ${userId}`);
+              core.setSecret(kubeconfig);
+              core.setOutput('kubeconfig_data', kubeconfig);
+            }
+      
+      - name: Install Deckhouse-cli
+        run: |
+          echo "Install d8"
+          curl -fsSL -o d8-install.sh https://raw.githubusercontent.com/deckhouse/deckhouse-cli/main/d8-install.sh
+          bash d8-install.sh
+
+      - name: Set up Go ${{ env.GO_VERSION }}
+        uses: actions/setup-go@v5
+        with:
+          go-version: "${{ env.GO_VERSION }}"
+
+      - name: Install Task
+        uses: arduino/setup-task@v2
+
+      - name: Install ginkgo
+        working-directory: ./tests/e2e/
+        run: |
+          echo "Install ginkgo"
+          GINKGO_VERSION=$(go list -f '{{.Version}}' -m github.com/onsi/ginkgo/v2)
+          go install "github.com/onsi/ginkgo/v2/ginkgo@${GINKGO_VERSION}"
+
+      - name: Install crane
+        run: |
+          go install github.com/google/go-containerregistry/cmd/crane@latest
+
+      - name: Install Deckhouse-cli
+        run: |
+          echo "Install d8"
+          curl -fsSL -o d8-install.sh https://raw.githubusercontent.com/deckhouse/deckhouse-cli/main/d8-install.sh
+          bash d8-install.sh
+
+      - uses: azure/k8s-set-context@v4
+        with:
+          method: kubeconfig
+          kubeconfig: ${{ steps.select_user.outputs.kubeconfig_data }}
+          
+      - name: Checkout cluster to revision
+        env:
+          v12n_tag: pr${{ github.event.pull_request.number }}
+        run: |
+          d8 k patch mpo virtualization --type merge -p '{"spec":{"imageTag":"$v12n_tag"}}'
+          local images_hash=$(crane export "dev-registry.deckhouse.io/sys/deckhouse-oss/modules/virtualization:$v12n_tag" - | tar -Oxf - images_digests.json)
+          local v12n_pods=$(kubectl -n d8-virtualization get pods -o json | jq -c)
+          local retry_count=0
+          local max_retries=120
+          local sleep_interval=5 
+
+          while true; do
+            all_hashes_found=true
+
+            # Fetch current pods information
+            local v12n_pods=$(kubectl -n d8-virtualization get pods -o json | jq -c)
+
+            # Process each image entry
+            while IFS= read -r image_entry; do
+              local image=$(echo "$image_entry" | jq -r '.key')
+              local hash=$(echo "$image_entry" | jq -r '.value')
+
+              if [[ "${image,,}" =~ (libguestfs|predeletehook) ]]; then
+                continue
+              fi
+
+              if echo "$v12n_pods" | grep -q "$hash"; then
+                echo "- ✅ $image $hash"
+              else
+                echo "- 🟥 $image $hash"
+                all_hashes_found=false
+              fi
+            done < <(echo "$images_hash" | jq -c '. | to_entries | sort_by(.key)[]')
+
+            # If all hashes are found, break the loop
+            if [ "$all_hashes_found" = true ]; then
+              echo "All image hashes found in pods."
+              break
+            fi 
+
+            ((retry_count++))
+            echo "Some hashes are missing, rechecking... Attempt: $retry_count"
+
+            # Check if the retry limit has been reached
+            if [ "$retry_count" -ge "$max_retries" ]; then
+              echo "Error: Timeout reached after $((retry_count * sleep_interval)) seconds. Some image hashes are still missing."
+              exit 1
+            fi
+            # Wait for the specified interval before retrying
+            sleep "$sleep_interval"
+          done
+
+      - name: Download dependencies
+        working-directory: ./tests/e2e/
+        run: |
+          echo "Download dependencies"
+          go mod download
+
+      - name: Run E2E
+        id: e2e-tests
+        working-directory: ./tests/e2e/
+        run: |
+          task run:ci -v
+
+  last_comment:
+    name: Update comment on finish
+    needs:
+      - run_e2e
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@v3.5.2
+
+      - name: Set commit status after e2e run
+        id: set_e2e_requirement_status
+        if: ${{ always() }}
+        uses: actions/github-script@v6.4.1
+        env:
+          JOB_STATUS: ${{ job.status }}
+          STATUS_TARGET_COMMIT: ${{ github.event.pull_request.head.sha }}
+        with:
+          github-token: ${{secrets.CHANGELOG_ACCESS_TOKEN}}
+          script: |
+            const e2eStatus = require('./.github/scripts/js/e2e-commit-status');
+
+            await e2eStatus.setStatusAfterE2eRun({github, context, core});
\ No newline at end of file

From 0eda30e8d9e5c7235e75f1f8b1fa24942762222c Mon Sep 17 00:00:00 2001
From: Maksim Fedotov <maksim.fedotov@flant.com>
Date: Wed, 2 Apr 2025 12:25:30 +0300
Subject: [PATCH 02/48] change pull_request_target to pull_request

Signed-off-by: Maksim Fedotov <maksim.fedotov@flant.com>
---
 .github/workflows/run_e2e.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/run_e2e.yml b/.github/workflows/run_e2e.yml
index f15989b8e9..08c38d316e 100644
--- a/.github/workflows/run_e2e.yml
+++ b/.github/workflows/run_e2e.yml
@@ -1,6 +1,7 @@
 name: Run e2e
 on:
-  pull_request_target:
+  # pull_request_target:
+  pull_request:
     types: 
       - labeled
 

From 48067285445cc083a0def882b24d458d010db4c5 Mon Sep 17 00:00:00 2001
From: Maksim Fedotov <maksim.fedotov@flant.com>
Date: Wed, 2 Apr 2025 14:18:27 +0300
Subject: [PATCH 03/48] change token name

Signed-off-by: Maksim Fedotov <maksim.fedotov@flant.com>
---
 .github/workflows/dev_module_build.yml | 2 +-
 .github/workflows/run_e2e.yml          | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/dev_module_build.yml b/.github/workflows/dev_module_build.yml
index 956fc42784..aa3bf1279d 100644
--- a/.github/workflows/dev_module_build.yml
+++ b/.github/workflows/dev_module_build.yml
@@ -369,7 +369,7 @@ jobs:
           STATUS_TARGET_COMMIT: ${{ github.event.pull_request.head.sha }}
           PR_LABELS: ${{ needs.pull_request_info.outputs.labels }}
         with:
-          github-token: ${{secrets.CHANGELOG_ACCESS_TOKEN}}
+          github-token: ${{secrets.RELEASE_PLEASE_TOKEN}}
           script: |
             const e2eStatus = require('./.github/scripts/js/e2e-commit-status');
             await e2eStatus.setInitialStatus({github, context, core});
diff --git a/.github/workflows/run_e2e.yml b/.github/workflows/run_e2e.yml
index 08c38d316e..86697aaee9 100644
--- a/.github/workflows/run_e2e.yml
+++ b/.github/workflows/run_e2e.yml
@@ -172,7 +172,7 @@ jobs:
           JOB_STATUS: ${{ job.status }}
           STATUS_TARGET_COMMIT: ${{ github.event.pull_request.head.sha }}
         with:
-          github-token: ${{secrets.CHANGELOG_ACCESS_TOKEN}}
+          github-token: ${{secrets.RELEASE_PLEASE_TOKEN}}
           script: |
             const e2eStatus = require('./.github/scripts/js/e2e-commit-status');
 

From 0b2a21721202af5fad85ed2a82e6b78e4005e24c Mon Sep 17 00:00:00 2001
From: Maksim Fedotov <maksim.fedotov@flant.com>
Date: Wed, 2 Apr 2025 14:38:56 +0300
Subject: [PATCH 04/48] add time

Signed-off-by: Maksim Fedotov <maksim.fedotov@flant.com>
---
 .github/scripts/time.js | 14 ++++++++++++++
 1 file changed, 14 insertions(+)
 create mode 100644 .github/scripts/time.js

diff --git a/.github/scripts/time.js b/.github/scripts/time.js
new file mode 100644
index 0000000000..ac3df2b4f8
--- /dev/null
+++ b/.github/scripts/time.js
@@ -0,0 +1,14 @@
+// Copyright 2022 Flant JSC
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//     http://www.apache.org/licenses/LICENSE-2.0
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+module.exports.sleep = async (ms) => {
+  return new Promise(resolve => setTimeout(resolve, ms))
+}

From 35e68fdcab419edc4a6df8617989ac25e17c7b55 Mon Sep 17 00:00:00 2001
From: Maksim Fedotov <maksim.fedotov@flant.com>
Date: Wed, 2 Apr 2025 15:14:36 +0300
Subject: [PATCH 05/48] verbosity=3

Signed-off-by: Maksim Fedotov <maksim.fedotov@flant.com>
---
 .github/workflows/run_e2e.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/run_e2e.yml b/.github/workflows/run_e2e.yml
index 86697aaee9..0a11826cff 100644
--- a/.github/workflows/run_e2e.yml
+++ b/.github/workflows/run_e2e.yml
@@ -95,7 +95,7 @@ jobs:
         env:
           v12n_tag: pr${{ github.event.pull_request.number }}
         run: |
-          d8 k patch mpo virtualization --type merge -p '{"spec":{"imageTag":"$v12n_tag"}}'
+          d8 k patch mpo virtualization --type merge -p '{"spec":{"imageTag":"$v12n_tag"}}' -v3
           local images_hash=$(crane export "dev-registry.deckhouse.io/sys/deckhouse-oss/modules/virtualization:$v12n_tag" - | tar -Oxf - images_digests.json)
           local v12n_pods=$(kubectl -n d8-virtualization get pods -o json | jq -c)
           local retry_count=0

From 41d428bf4ba5fbc3c2a9674ac1f4cf04ce5d6341 Mon Sep 17 00:00:00 2001
From: Maksim Fedotov <maksim.fedotov@flant.com>
Date: Wed, 2 Apr 2025 15:16:08 +0300
Subject: [PATCH 06/48] move time

Signed-off-by: Maksim Fedotov <maksim.fedotov@flant.com>
---
 .github/scripts/{ => js}/time.js | 0
 1 file changed, 0 insertions(+), 0 deletions(-)
 rename .github/scripts/{ => js}/time.js (100%)

diff --git a/.github/scripts/time.js b/.github/scripts/js/time.js
similarity index 100%
rename from .github/scripts/time.js
rename to .github/scripts/js/time.js

From a39fb6234a3560fc445c75e50537c28e863f3a7f Mon Sep 17 00:00:00 2001
From: Maksim Fedotov <maksim.fedotov@flant.com>
Date: Wed, 2 Apr 2025 15:59:03 +0300
Subject: [PATCH 07/48] remove double d8 install

Signed-off-by: Maksim Fedotov <maksim.fedotov@flant.com>
---
 .github/workflows/run_e2e.yml | 6 ------
 1 file changed, 6 deletions(-)

diff --git a/.github/workflows/run_e2e.yml b/.github/workflows/run_e2e.yml
index 0a11826cff..b0fa203789 100644
--- a/.github/workflows/run_e2e.yml
+++ b/.github/workflows/run_e2e.yml
@@ -80,12 +80,6 @@ jobs:
         run: |
           go install github.com/google/go-containerregistry/cmd/crane@latest
 
-      - name: Install Deckhouse-cli
-        run: |
-          echo "Install d8"
-          curl -fsSL -o d8-install.sh https://raw.githubusercontent.com/deckhouse/deckhouse-cli/main/d8-install.sh
-          bash d8-install.sh
-
       - uses: azure/k8s-set-context@v4
         with:
           method: kubeconfig

From 8f13b9d354ce3c1cdbc89d507d2df12345cf90ee Mon Sep 17 00:00:00 2001
From: Maksim Fedotov <maksim.fedotov@flant.com>
Date: Wed, 2 Apr 2025 16:40:27 +0300
Subject: [PATCH 08/48] add setup

Signed-off-by: Maksim Fedotov <maksim.fedotov@flant.com>
---
 .github/workflows/run_e2e.yml | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/.github/workflows/run_e2e.yml b/.github/workflows/run_e2e.yml
index b0fa203789..f2159ca725 100644
--- a/.github/workflows/run_e2e.yml
+++ b/.github/workflows/run_e2e.yml
@@ -84,6 +84,12 @@ jobs:
         with:
           method: kubeconfig
           kubeconfig: ${{ steps.select_user.outputs.kubeconfig_data }}
+
+      - uses: deckhouse/modules-actions/setup@v2
+        with:
+          registry: ${{ vars.DEV_REGISTRY }}
+          registry_login: ${{ vars.DEV_MODULES_REGISTRY_LOGIN }}
+          registry_password: ${{ secrets.DEV_MODULES_REGISTRY_PASSWORD }}
           
       - name: Checkout cluster to revision
         env:

From df2f3f4fdac0a56c2851ffe146ba05693f0050b3 Mon Sep 17 00:00:00 2001
From: Maksim Fedotov <maksim.fedotov@flant.com>
Date: Wed, 2 Apr 2025 16:57:30 +0300
Subject: [PATCH 09/48] remove locals

Signed-off-by: Maksim Fedotov <maksim.fedotov@flant.com>
---
 .github/workflows/run_e2e.yml | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/.github/workflows/run_e2e.yml b/.github/workflows/run_e2e.yml
index f2159ca725..20075161d2 100644
--- a/.github/workflows/run_e2e.yml
+++ b/.github/workflows/run_e2e.yml
@@ -96,22 +96,22 @@ jobs:
           v12n_tag: pr${{ github.event.pull_request.number }}
         run: |
           d8 k patch mpo virtualization --type merge -p '{"spec":{"imageTag":"$v12n_tag"}}' -v3
-          local images_hash=$(crane export "dev-registry.deckhouse.io/sys/deckhouse-oss/modules/virtualization:$v12n_tag" - | tar -Oxf - images_digests.json)
-          local v12n_pods=$(kubectl -n d8-virtualization get pods -o json | jq -c)
-          local retry_count=0
-          local max_retries=120
-          local sleep_interval=5 
+          images_hash=$(crane export "dev-registry.deckhouse.io/sys/deckhouse-oss/modules/virtualization:$v12n_tag" - | tar -Oxf - images_digests.json)
+          v12n_pods=$(kubectl -n d8-virtualization get pods -o json | jq -c)
+          retry_count=0
+          max_retries=120
+          sleep_interval=5 
 
           while true; do
             all_hashes_found=true
 
             # Fetch current pods information
-            local v12n_pods=$(kubectl -n d8-virtualization get pods -o json | jq -c)
+            v12n_pods=$(kubectl -n d8-virtualization get pods -o json | jq -c)
 
             # Process each image entry
             while IFS= read -r image_entry; do
-              local image=$(echo "$image_entry" | jq -r '.key')
-              local hash=$(echo "$image_entry" | jq -r '.value')
+              image=$(echo "$image_entry" | jq -r '.key')
+              hash=$(echo "$image_entry" | jq -r '.value')
 
               if [[ "${image,,}" =~ (libguestfs|predeletehook) ]]; then
                 continue

From cb77d9dc43c2d8ac5004a3a71ef34a8a2b4e5a74 Mon Sep 17 00:00:00 2001
From: Maksim Fedotov <maksim.fedotov@flant.com>
Date: Wed, 2 Apr 2025 17:22:55 +0300
Subject: [PATCH 10/48] fix arithmetic

Signed-off-by: Maksim Fedotov <maksim.fedotov@flant.com>
---
 .github/workflows/run_e2e.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/run_e2e.yml b/.github/workflows/run_e2e.yml
index 20075161d2..8de36bc291 100644
--- a/.github/workflows/run_e2e.yml
+++ b/.github/workflows/run_e2e.yml
@@ -131,7 +131,7 @@ jobs:
               break
             fi 
 
-            ((retry_count++))
+            retry_count=$((retry_count + 1))
             echo "Some hashes are missing, rechecking... Attempt: $retry_count"
 
             # Check if the retry limit has been reached

From c969980cd6afab7b53c7d389ff4d69c03945106e Mon Sep 17 00:00:00 2001
From: Maksim Fedotov <maksim.fedotov@flant.com>
Date: Thu, 3 Apr 2025 10:32:18 +0300
Subject: [PATCH 11/48] add bash

Signed-off-by: Maksim Fedotov <maksim.fedotov@flant.com>
---
 .github/workflows/run_e2e.yml | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/.github/workflows/run_e2e.yml b/.github/workflows/run_e2e.yml
index 8de36bc291..fb1dc57da6 100644
--- a/.github/workflows/run_e2e.yml
+++ b/.github/workflows/run_e2e.yml
@@ -8,6 +8,10 @@ on:
 env:
   GO_VERSION: "1.22.7"
 
+defaults:
+  run:
+    shell: bash
+
 jobs:
   skip_e2e:
     if: ${{ github.event.label.name == 'skip/e2e' }}

From 53a492a2fd2f6084f4d83ca67bff528ff02b7962 Mon Sep 17 00:00:00 2001
From: Maksim Fedotov <maksim.fedotov@flant.com>
Date: Thu, 3 Apr 2025 17:50:18 +0300
Subject: [PATCH 12/48] comment out crane

Signed-off-by: Maksim Fedotov <maksim.fedotov@flant.com>
---
 .github/workflows/run_e2e.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/run_e2e.yml b/.github/workflows/run_e2e.yml
index fb1dc57da6..c759651a60 100644
--- a/.github/workflows/run_e2e.yml
+++ b/.github/workflows/run_e2e.yml
@@ -80,9 +80,9 @@ jobs:
           GINKGO_VERSION=$(go list -f '{{.Version}}' -m github.com/onsi/ginkgo/v2)
           go install "github.com/onsi/ginkgo/v2/ginkgo@${GINKGO_VERSION}"
 
-      - name: Install crane
-        run: |
-          go install github.com/google/go-containerregistry/cmd/crane@latest
+      # - name: Install crane
+      #   run: |
+      #     go install github.com/google/go-containerregistry/cmd/crane@latest
 
       - uses: azure/k8s-set-context@v4
         with:

From 9e900be833641fd21c2396fe079510bc3416af2f Mon Sep 17 00:00:00 2001
From: Maksim Fedotov <maksim.fedotov@flant.com>
Date: Thu, 3 Apr 2025 17:54:08 +0300
Subject: [PATCH 13/48] fix var

Signed-off-by: Maksim Fedotov <maksim.fedotov@flant.com>
---
 .github/workflows/run_e2e.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/run_e2e.yml b/.github/workflows/run_e2e.yml
index c759651a60..a701034338 100644
--- a/.github/workflows/run_e2e.yml
+++ b/.github/workflows/run_e2e.yml
@@ -99,7 +99,7 @@ jobs:
         env:
           v12n_tag: pr${{ github.event.pull_request.number }}
         run: |
-          d8 k patch mpo virtualization --type merge -p '{"spec":{"imageTag":"$v12n_tag"}}' -v3
+          d8 k patch mpo virtualization --type merge -p "{\"spec\":{\"imageTag\":\"$v12n_tag\"}}" 
           images_hash=$(crane export "dev-registry.deckhouse.io/sys/deckhouse-oss/modules/virtualization:$v12n_tag" - | tar -Oxf - images_digests.json)
           v12n_pods=$(kubectl -n d8-virtualization get pods -o json | jq -c)
           retry_count=0

From 4b0afef4d2908db5f915091d1ea958e5d2170f9f Mon Sep 17 00:00:00 2001
From: Maksim Fedotov <maksim.fedotov@flant.com>
Date: Fri, 4 Apr 2025 09:36:29 +0300
Subject: [PATCH 14/48] set context via js

Signed-off-by: Maksim Fedotov <maksim.fedotov@flant.com>
---
 .github/workflows/run_e2e.yml | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/.github/workflows/run_e2e.yml b/.github/workflows/run_e2e.yml
index a701034338..f81627a469 100644
--- a/.github/workflows/run_e2e.yml
+++ b/.github/workflows/run_e2e.yml
@@ -55,8 +55,14 @@ jobs:
               core.setFailed(`No kubeconfig found for user with ID ${userId}.`);
             } else {
               core.info(`Found kubeconfig for user with ID ${userId}`);
-              core.setSecret(kubeconfig);
-              core.setOutput('kubeconfig_data', kubeconfig);
+              const runnerTempDir = process.env['RUNNER_TEMP'];
+              const kubeconfigFile: string = path.join(runnerTempDir, `kubeconfig_${Date.now()}`);
+              fs.writeFileSync(kubeconfigFile, kubeconfig);
+              fs.chmodSync(kubeconfigFile, '600');
+              core.exportVariable('KUBECONFIG', kubeconfigFile)
+
+              // core.setSecret(kubeconfig);
+              // core.setOutput('kubeconfig_data', kubeconfig);
             }
       
       - name: Install Deckhouse-cli
@@ -84,10 +90,10 @@ jobs:
       #   run: |
       #     go install github.com/google/go-containerregistry/cmd/crane@latest
 
-      - uses: azure/k8s-set-context@v4
-        with:
-          method: kubeconfig
-          kubeconfig: ${{ steps.select_user.outputs.kubeconfig_data }}
+      # - uses: azure/k8s-set-context@v4
+      #   with:
+      #     method: kubeconfig
+      #     kubeconfig: ${{ steps.select_user.outputs.kubeconfig_data }}
 
       - uses: deckhouse/modules-actions/setup@v2
         with:

From 830ecd0d2f167132bc297677c02067496525b183 Mon Sep 17 00:00:00 2001
From: Maksim Fedotov <maksim.fedotov@flant.com>
Date: Fri, 4 Apr 2025 10:21:16 +0300
Subject: [PATCH 15/48] require fs

Signed-off-by: Maksim Fedotov <maksim.fedotov@flant.com>
---
 .github/workflows/run_e2e.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/run_e2e.yml b/.github/workflows/run_e2e.yml
index f81627a469..11cb68165e 100644
--- a/.github/workflows/run_e2e.yml
+++ b/.github/workflows/run_e2e.yml
@@ -46,7 +46,8 @@ jobs:
         with:
           script: |
             const ci = require('./.github/scripts/js/ci');
-            const userId = await ci.getClusterUser({context, core})
+            const userId = await ci.getClusterUser({context, core});
+            const fs = require('fs');
             // core.setOutput("user_id", user_id)
             // core.exportVariable("user_id", user_id);
             const kubeconfigs = JSON.parse(process.env.KUBECONFIGS);

From 1debbf14f540591702a3d989bc6992b242043f2f Mon Sep 17 00:00:00 2001
From: Maksim Fedotov <maksim.fedotov@flant.com>
Date: Fri, 4 Apr 2025 10:27:21 +0300
Subject: [PATCH 16/48] remove string

Signed-off-by: Maksim Fedotov <maksim.fedotov@flant.com>
---
 .github/workflows/run_e2e.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/run_e2e.yml b/.github/workflows/run_e2e.yml
index 11cb68165e..49ababefad 100644
--- a/.github/workflows/run_e2e.yml
+++ b/.github/workflows/run_e2e.yml
@@ -57,7 +57,7 @@ jobs:
             } else {
               core.info(`Found kubeconfig for user with ID ${userId}`);
               const runnerTempDir = process.env['RUNNER_TEMP'];
-              const kubeconfigFile: string = path.join(runnerTempDir, `kubeconfig_${Date.now()}`);
+              const kubeconfigFile = path.join(runnerTempDir, `kubeconfig_${Date.now()}`);
               fs.writeFileSync(kubeconfigFile, kubeconfig);
               fs.chmodSync(kubeconfigFile, '600');
               core.exportVariable('KUBECONFIG', kubeconfigFile)

From 401665362839967a69f8c47b7df77b0bd8878818 Mon Sep 17 00:00:00 2001
From: Maksim Fedotov <maksim.fedotov@flant.com>
Date: Fri, 4 Apr 2025 10:29:03 +0300
Subject: [PATCH 17/48] add path

Signed-off-by: Maksim Fedotov <maksim.fedotov@flant.com>
---
 .github/workflows/run_e2e.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/run_e2e.yml b/.github/workflows/run_e2e.yml
index 49ababefad..74edf824c0 100644
--- a/.github/workflows/run_e2e.yml
+++ b/.github/workflows/run_e2e.yml
@@ -48,6 +48,7 @@ jobs:
             const ci = require('./.github/scripts/js/ci');
             const userId = await ci.getClusterUser({context, core});
             const fs = require('fs');
+            const path = require('path');
             // core.setOutput("user_id", user_id)
             // core.exportVariable("user_id", user_id);
             const kubeconfigs = JSON.parse(process.env.KUBECONFIGS);

From ed2af368876a6087a99c34f2c57e3ede06af94c0 Mon Sep 17 00:00:00 2001
From: Maksim Fedotov <maksim.fedotov@flant.com>
Date: Sun, 6 Apr 2025 21:02:35 +0300
Subject: [PATCH 18/48] change task

Signed-off-by: Maksim Fedotov <maksim.fedotov@flant.com>
---
 .github/workflows/run_e2e.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/run_e2e.yml b/.github/workflows/run_e2e.yml
index 74edf824c0..c4933bd293 100644
--- a/.github/workflows/run_e2e.yml
+++ b/.github/workflows/run_e2e.yml
@@ -165,7 +165,7 @@ jobs:
         id: e2e-tests
         working-directory: ./tests/e2e/
         run: |
-          task run:ci -v
+          task run -v
 
   last_comment:
     name: Update comment on finish

From 0129476b4d15991b948b17a7036d3f1c2f57ba8c Mon Sep 17 00:00:00 2001
From: Maksim Fedotov <maksim.fedotov@flant.com>
Date: Mon, 7 Apr 2025 10:40:13 +0300
Subject: [PATCH 19/48] move e2e to build stage

Signed-off-by: Maksim Fedotov <maksim.fedotov@flant.com>
---
 .github/workflows/dev_module_build.yml | 180 ++++++++++++
 .github/workflows/run_e2e.yml          | 378 ++++++++++++-------------
 2 files changed, 369 insertions(+), 189 deletions(-)

diff --git a/.github/workflows/dev_module_build.yml b/.github/workflows/dev_module_build.yml
index aa3bf1279d..ccf8e8ceb7 100644
--- a/.github/workflows/dev_module_build.yml
+++ b/.github/workflows/dev_module_build.yml
@@ -392,3 +392,183 @@ jobs:
           trivy_registry_user: ${{ vars.PROD_MODULES_REGISTRY_LOGIN }}
           trivy_registry_password: ${{ secrets.PROD_MODULES_REGISTRY_PASSWORD }}
           deckhouse_private_repo: ${{vars.DECKHOUSE_PRIVATE_REPO}}
+
+  skip_e2e:
+    if: ${{ github.event.label.name == 'skip/e2e' }}
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Skip E2E tests
+        id: skip_e2e
+        uses: actions/github-script@v6
+        with:
+          script: |
+            const e2eStatus = require('./.github/scripts/js/e2e-commit-status');
+            e2eStatus.onLabeledForSkip({
+              github,
+              context,
+              core,
+              labeled: true,
+              commitSha: context.payload.pull_request.head.sha
+            })
+
+  run_e2e:
+    if: ${{ github.event.label.name == 'e2e/run' }}
+    name: Run E2E tests
+    runs-on: ubuntu-latest
+    needs: 
+      - dev_setup_build
+      - set_e2e_requirement_status
+    steps:
+      - uses: actions/checkout@v4
+      - name: Select user
+        id: select_user
+        uses: actions/github-script@v6
+        env:
+          KUBECONFIGS: ${{ secrets.K8S_CLUSTER_SECRET }}
+        with:
+          script: |
+            const ci = require('./.github/scripts/js/ci');
+            const userId = await ci.getClusterUser({context, core});
+            const fs = require('fs');
+            const path = require('path');
+            // core.setOutput("user_id", user_id)
+            // core.exportVariable("user_id", user_id);
+            const kubeconfigs = JSON.parse(process.env.KUBECONFIGS);
+            const kubeconfig = kubeconfigs.find(config => config.id === userId)?.kubeconfig;
+            if (!kubeconfig) {
+              core.setFailed(`No kubeconfig found for user with ID ${userId}.`);
+            } else {
+              core.info(`Found kubeconfig for user with ID ${userId}`);
+              const runnerTempDir = process.env['RUNNER_TEMP'];
+              const kubeconfigFile = path.join(runnerTempDir, `kubeconfig_${Date.now()}`);
+              fs.writeFileSync(kubeconfigFile, kubeconfig);
+              fs.chmodSync(kubeconfigFile, '600');
+              core.exportVariable('KUBECONFIG', kubeconfigFile)
+
+              // core.setSecret(kubeconfig);
+              // core.setOutput('kubeconfig_data', kubeconfig);
+            }
+      
+      - name: Install Deckhouse-cli
+        run: |
+          echo "Install d8"
+          curl -fsSL -o d8-install.sh https://raw.githubusercontent.com/deckhouse/deckhouse-cli/main/d8-install.sh
+          bash d8-install.sh
+
+      - name: Set up Go ${{ env.GO_VERSION }}
+        uses: actions/setup-go@v5
+        with:
+          go-version: "${{ env.GO_VERSION }}"
+
+      - name: Install Task
+        uses: arduino/setup-task@v2
+
+      - name: Install ginkgo
+        working-directory: ./tests/e2e/
+        run: |
+          echo "Install ginkgo"
+          GINKGO_VERSION=$(go list -f '{{.Version}}' -m github.com/onsi/ginkgo/v2)
+          go install "github.com/onsi/ginkgo/v2/ginkgo@${GINKGO_VERSION}"
+
+      # - name: Install crane
+      #   run: |
+      #     go install github.com/google/go-containerregistry/cmd/crane@latest
+
+      # - uses: azure/k8s-set-context@v4
+      #   with:
+      #     method: kubeconfig
+      #     kubeconfig: ${{ steps.select_user.outputs.kubeconfig_data }}
+
+      - uses: deckhouse/modules-actions/setup@v2
+        with:
+          registry: ${{ vars.DEV_REGISTRY }}
+          registry_login: ${{ vars.DEV_MODULES_REGISTRY_LOGIN }}
+          registry_password: ${{ secrets.DEV_MODULES_REGISTRY_PASSWORD }}
+          
+      - name: Checkout cluster to revision
+        env:
+          v12n_tag: pr${{ github.event.pull_request.number }}
+        run: |
+          d8 k patch mpo virtualization --type merge -p "{\"spec\":{\"imageTag\":\"$v12n_tag\"}}" 
+          images_hash=$(crane export "dev-registry.deckhouse.io/sys/deckhouse-oss/modules/virtualization:$v12n_tag" - | tar -Oxf - images_digests.json)
+          v12n_pods=$(kubectl -n d8-virtualization get pods -o json | jq -c)
+          retry_count=0
+          max_retries=120
+          sleep_interval=5 
+
+          while true; do
+            all_hashes_found=true
+
+            # Fetch current pods information
+            v12n_pods=$(kubectl -n d8-virtualization get pods -o json | jq -c)
+
+            # Process each image entry
+            while IFS= read -r image_entry; do
+              image=$(echo "$image_entry" | jq -r '.key')
+              hash=$(echo "$image_entry" | jq -r '.value')
+
+              if [[ "${image,,}" =~ (libguestfs|predeletehook) ]]; then
+                continue
+              fi
+
+              if echo "$v12n_pods" | grep -q "$hash"; then
+                echo "- ✅ $image $hash"
+              else
+                echo "- 🟥 $image $hash"
+                all_hashes_found=false
+              fi
+            done < <(echo "$images_hash" | jq -c '. | to_entries | sort_by(.key)[]')
+
+            # If all hashes are found, break the loop
+            if [ "$all_hashes_found" = true ]; then
+              echo "All image hashes found in pods."
+              break
+            fi 
+
+            retry_count=$((retry_count + 1))
+            echo "Some hashes are missing, rechecking... Attempt: $retry_count"
+
+            # Check if the retry limit has been reached
+            if [ "$retry_count" -ge "$max_retries" ]; then
+              echo "Error: Timeout reached after $((retry_count * sleep_interval)) seconds. Some image hashes are still missing."
+              exit 1
+            fi
+            # Wait for the specified interval before retrying
+            sleep "$sleep_interval"
+          done
+
+      - name: Download dependencies
+        working-directory: ./tests/e2e/
+        run: |
+          echo "Download dependencies"
+          go mod download
+
+      - name: Run E2E
+        id: e2e-tests
+        working-directory: ./tests/e2e/
+        run: |
+          task run -v
+
+  last_comment:
+    name: Update comment on finish
+    needs:
+      - run_e2e
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout sources
+        uses: actions/checkout@v3.5.2
+
+      - name: Set commit status after e2e run
+        id: set_e2e_requirement_status
+        if: ${{ always() }}
+        uses: actions/github-script@v6.4.1
+        env:
+          JOB_STATUS: ${{ job.status }}
+          STATUS_TARGET_COMMIT: ${{ github.event.pull_request.head.sha }}
+        with:
+          github-token: ${{secrets.RELEASE_PLEASE_TOKEN}}
+          script: |
+            const e2eStatus = require('./.github/scripts/js/e2e-commit-status');
+
+            await e2eStatus.setStatusAfterE2eRun({github, context, core});
\ No newline at end of file
diff --git a/.github/workflows/run_e2e.yml b/.github/workflows/run_e2e.yml
index c4933bd293..b90dbc26f3 100644
--- a/.github/workflows/run_e2e.yml
+++ b/.github/workflows/run_e2e.yml
@@ -1,191 +1,191 @@
-name: Run e2e
-on:
-  # pull_request_target:
-  pull_request:
-    types: 
-      - labeled
-
-env:
-  GO_VERSION: "1.22.7"
-
-defaults:
-  run:
-    shell: bash
-
-jobs:
-  skip_e2e:
-    if: ${{ github.event.label.name == 'skip/e2e' }}
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - name: Skip E2E tests
-        id: skip_e2e
-        uses: actions/github-script@v6
-        with:
-          script: |
-            const e2eStatus = require('./.github/scripts/js/e2e-commit-status');
-            e2eStatus.onLabeledForSkip({
-              github,
-              context,
-              core,
-              labeled: true,
-              commitSha: context.payload.pull_request.head.sha
-            })
-
-  run_e2e:
-    if: ${{ github.event.label.name == 'e2e/run' }}
-    name: Run E2E tests
-    runs-on: ubuntu-latest
-    steps:
-      - uses: actions/checkout@v4
-      - name: Select user
-        id: select_user
-        uses: actions/github-script@v6
-        env:
-          KUBECONFIGS: ${{ secrets.K8S_CLUSTER_SECRET }}
-        with:
-          script: |
-            const ci = require('./.github/scripts/js/ci');
-            const userId = await ci.getClusterUser({context, core});
-            const fs = require('fs');
-            const path = require('path');
-            // core.setOutput("user_id", user_id)
-            // core.exportVariable("user_id", user_id);
-            const kubeconfigs = JSON.parse(process.env.KUBECONFIGS);
-            const kubeconfig = kubeconfigs.find(config => config.id === userId)?.kubeconfig;
-            if (!kubeconfig) {
-              core.setFailed(`No kubeconfig found for user with ID ${userId}.`);
-            } else {
-              core.info(`Found kubeconfig for user with ID ${userId}`);
-              const runnerTempDir = process.env['RUNNER_TEMP'];
-              const kubeconfigFile = path.join(runnerTempDir, `kubeconfig_${Date.now()}`);
-              fs.writeFileSync(kubeconfigFile, kubeconfig);
-              fs.chmodSync(kubeconfigFile, '600');
-              core.exportVariable('KUBECONFIG', kubeconfigFile)
-
-              // core.setSecret(kubeconfig);
-              // core.setOutput('kubeconfig_data', kubeconfig);
-            }
+# name: Run e2e
+# on:
+#   # pull_request_target:
+#   pull_request:
+#     types: 
+#       - labeled
+
+# env:
+#   GO_VERSION: "1.22.7"
+
+# defaults:
+#   run:
+#     shell: bash
+
+# jobs:
+#   skip_e2e:
+#     if: ${{ github.event.label.name == 'skip/e2e' }}
+#     runs-on: ubuntu-latest
+#     steps:
+#       - uses: actions/checkout@v4
+#       - name: Skip E2E tests
+#         id: skip_e2e
+#         uses: actions/github-script@v6
+#         with:
+#           script: |
+#             const e2eStatus = require('./.github/scripts/js/e2e-commit-status');
+#             e2eStatus.onLabeledForSkip({
+#               github,
+#               context,
+#               core,
+#               labeled: true,
+#               commitSha: context.payload.pull_request.head.sha
+#             })
+
+#   run_e2e:
+#     if: ${{ github.event.label.name == 'e2e/run' }}
+#     name: Run E2E tests
+#     runs-on: ubuntu-latest
+#     steps:
+#       - uses: actions/checkout@v4
+#       - name: Select user
+#         id: select_user
+#         uses: actions/github-script@v6
+#         env:
+#           KUBECONFIGS: ${{ secrets.K8S_CLUSTER_SECRET }}
+#         with:
+#           script: |
+#             const ci = require('./.github/scripts/js/ci');
+#             const userId = await ci.getClusterUser({context, core});
+#             const fs = require('fs');
+#             const path = require('path');
+#             // core.setOutput("user_id", user_id)
+#             // core.exportVariable("user_id", user_id);
+#             const kubeconfigs = JSON.parse(process.env.KUBECONFIGS);
+#             const kubeconfig = kubeconfigs.find(config => config.id === userId)?.kubeconfig;
+#             if (!kubeconfig) {
+#               core.setFailed(`No kubeconfig found for user with ID ${userId}.`);
+#             } else {
+#               core.info(`Found kubeconfig for user with ID ${userId}`);
+#               const runnerTempDir = process.env['RUNNER_TEMP'];
+#               const kubeconfigFile = path.join(runnerTempDir, `kubeconfig_${Date.now()}`);
+#               fs.writeFileSync(kubeconfigFile, kubeconfig);
+#               fs.chmodSync(kubeconfigFile, '600');
+#               core.exportVariable('KUBECONFIG', kubeconfigFile)
+
+#               // core.setSecret(kubeconfig);
+#               // core.setOutput('kubeconfig_data', kubeconfig);
+#             }
       
-      - name: Install Deckhouse-cli
-        run: |
-          echo "Install d8"
-          curl -fsSL -o d8-install.sh https://raw.githubusercontent.com/deckhouse/deckhouse-cli/main/d8-install.sh
-          bash d8-install.sh
-
-      - name: Set up Go ${{ env.GO_VERSION }}
-        uses: actions/setup-go@v5
-        with:
-          go-version: "${{ env.GO_VERSION }}"
-
-      - name: Install Task
-        uses: arduino/setup-task@v2
-
-      - name: Install ginkgo
-        working-directory: ./tests/e2e/
-        run: |
-          echo "Install ginkgo"
-          GINKGO_VERSION=$(go list -f '{{.Version}}' -m github.com/onsi/ginkgo/v2)
-          go install "github.com/onsi/ginkgo/v2/ginkgo@${GINKGO_VERSION}"
-
-      # - name: Install crane
-      #   run: |
-      #     go install github.com/google/go-containerregistry/cmd/crane@latest
-
-      # - uses: azure/k8s-set-context@v4
-      #   with:
-      #     method: kubeconfig
-      #     kubeconfig: ${{ steps.select_user.outputs.kubeconfig_data }}
-
-      - uses: deckhouse/modules-actions/setup@v2
-        with:
-          registry: ${{ vars.DEV_REGISTRY }}
-          registry_login: ${{ vars.DEV_MODULES_REGISTRY_LOGIN }}
-          registry_password: ${{ secrets.DEV_MODULES_REGISTRY_PASSWORD }}
+#       - name: Install Deckhouse-cli
+#         run: |
+#           echo "Install d8"
+#           curl -fsSL -o d8-install.sh https://raw.githubusercontent.com/deckhouse/deckhouse-cli/main/d8-install.sh
+#           bash d8-install.sh
+
+#       - name: Set up Go ${{ env.GO_VERSION }}
+#         uses: actions/setup-go@v5
+#         with:
+#           go-version: "${{ env.GO_VERSION }}"
+
+#       - name: Install Task
+#         uses: arduino/setup-task@v2
+
+#       - name: Install ginkgo
+#         working-directory: ./tests/e2e/
+#         run: |
+#           echo "Install ginkgo"
+#           GINKGO_VERSION=$(go list -f '{{.Version}}' -m github.com/onsi/ginkgo/v2)
+#           go install "github.com/onsi/ginkgo/v2/ginkgo@${GINKGO_VERSION}"
+
+#       # - name: Install crane
+#       #   run: |
+#       #     go install github.com/google/go-containerregistry/cmd/crane@latest
+
+#       # - uses: azure/k8s-set-context@v4
+#       #   with:
+#       #     method: kubeconfig
+#       #     kubeconfig: ${{ steps.select_user.outputs.kubeconfig_data }}
+
+#       - uses: deckhouse/modules-actions/setup@v2
+#         with:
+#           registry: ${{ vars.DEV_REGISTRY }}
+#           registry_login: ${{ vars.DEV_MODULES_REGISTRY_LOGIN }}
+#           registry_password: ${{ secrets.DEV_MODULES_REGISTRY_PASSWORD }}
           
-      - name: Checkout cluster to revision
-        env:
-          v12n_tag: pr${{ github.event.pull_request.number }}
-        run: |
-          d8 k patch mpo virtualization --type merge -p "{\"spec\":{\"imageTag\":\"$v12n_tag\"}}" 
-          images_hash=$(crane export "dev-registry.deckhouse.io/sys/deckhouse-oss/modules/virtualization:$v12n_tag" - | tar -Oxf - images_digests.json)
-          v12n_pods=$(kubectl -n d8-virtualization get pods -o json | jq -c)
-          retry_count=0
-          max_retries=120
-          sleep_interval=5 
-
-          while true; do
-            all_hashes_found=true
-
-            # Fetch current pods information
-            v12n_pods=$(kubectl -n d8-virtualization get pods -o json | jq -c)
-
-            # Process each image entry
-            while IFS= read -r image_entry; do
-              image=$(echo "$image_entry" | jq -r '.key')
-              hash=$(echo "$image_entry" | jq -r '.value')
-
-              if [[ "${image,,}" =~ (libguestfs|predeletehook) ]]; then
-                continue
-              fi
-
-              if echo "$v12n_pods" | grep -q "$hash"; then
-                echo "- ✅ $image $hash"
-              else
-                echo "- 🟥 $image $hash"
-                all_hashes_found=false
-              fi
-            done < <(echo "$images_hash" | jq -c '. | to_entries | sort_by(.key)[]')
-
-            # If all hashes are found, break the loop
-            if [ "$all_hashes_found" = true ]; then
-              echo "All image hashes found in pods."
-              break
-            fi 
-
-            retry_count=$((retry_count + 1))
-            echo "Some hashes are missing, rechecking... Attempt: $retry_count"
-
-            # Check if the retry limit has been reached
-            if [ "$retry_count" -ge "$max_retries" ]; then
-              echo "Error: Timeout reached after $((retry_count * sleep_interval)) seconds. Some image hashes are still missing."
-              exit 1
-            fi
-            # Wait for the specified interval before retrying
-            sleep "$sleep_interval"
-          done
-
-      - name: Download dependencies
-        working-directory: ./tests/e2e/
-        run: |
-          echo "Download dependencies"
-          go mod download
-
-      - name: Run E2E
-        id: e2e-tests
-        working-directory: ./tests/e2e/
-        run: |
-          task run -v
-
-  last_comment:
-    name: Update comment on finish
-    needs:
-      - run_e2e
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout sources
-        uses: actions/checkout@v3.5.2
-
-      - name: Set commit status after e2e run
-        id: set_e2e_requirement_status
-        if: ${{ always() }}
-        uses: actions/github-script@v6.4.1
-        env:
-          JOB_STATUS: ${{ job.status }}
-          STATUS_TARGET_COMMIT: ${{ github.event.pull_request.head.sha }}
-        with:
-          github-token: ${{secrets.RELEASE_PLEASE_TOKEN}}
-          script: |
-            const e2eStatus = require('./.github/scripts/js/e2e-commit-status');
-
-            await e2eStatus.setStatusAfterE2eRun({github, context, core});
\ No newline at end of file
+#       - name: Checkout cluster to revision
+#         env:
+#           v12n_tag: pr${{ github.event.pull_request.number }}
+#         run: |
+#           d8 k patch mpo virtualization --type merge -p "{\"spec\":{\"imageTag\":\"$v12n_tag\"}}" 
+#           images_hash=$(crane export "dev-registry.deckhouse.io/sys/deckhouse-oss/modules/virtualization:$v12n_tag" - | tar -Oxf - images_digests.json)
+#           v12n_pods=$(kubectl -n d8-virtualization get pods -o json | jq -c)
+#           retry_count=0
+#           max_retries=120
+#           sleep_interval=5 
+
+#           while true; do
+#             all_hashes_found=true
+
+#             # Fetch current pods information
+#             v12n_pods=$(kubectl -n d8-virtualization get pods -o json | jq -c)
+
+#             # Process each image entry
+#             while IFS= read -r image_entry; do
+#               image=$(echo "$image_entry" | jq -r '.key')
+#               hash=$(echo "$image_entry" | jq -r '.value')
+
+#               if [[ "${image,,}" =~ (libguestfs|predeletehook) ]]; then
+#                 continue
+#               fi
+
+#               if echo "$v12n_pods" | grep -q "$hash"; then
+#                 echo "- ✅ $image $hash"
+#               else
+#                 echo "- 🟥 $image $hash"
+#                 all_hashes_found=false
+#               fi
+#             done < <(echo "$images_hash" | jq -c '. | to_entries | sort_by(.key)[]')
+
+#             # If all hashes are found, break the loop
+#             if [ "$all_hashes_found" = true ]; then
+#               echo "All image hashes found in pods."
+#               break
+#             fi 
+
+#             retry_count=$((retry_count + 1))
+#             echo "Some hashes are missing, rechecking... Attempt: $retry_count"
+
+#             # Check if the retry limit has been reached
+#             if [ "$retry_count" -ge "$max_retries" ]; then
+#               echo "Error: Timeout reached after $((retry_count * sleep_interval)) seconds. Some image hashes are still missing."
+#               exit 1
+#             fi
+#             # Wait for the specified interval before retrying
+#             sleep "$sleep_interval"
+#           done
+
+#       - name: Download dependencies
+#         working-directory: ./tests/e2e/
+#         run: |
+#           echo "Download dependencies"
+#           go mod download
+
+#       - name: Run E2E
+#         id: e2e-tests
+#         working-directory: ./tests/e2e/
+#         run: |
+#           task run -v
+
+#   last_comment:
+#     name: Update comment on finish
+#     needs:
+#       - run_e2e
+#     runs-on: ubuntu-latest
+#     steps:
+#       - name: Checkout sources
+#         uses: actions/checkout@v3.5.2
+
+#       - name: Set commit status after e2e run
+#         id: set_e2e_requirement_status
+#         if: ${{ always() }}
+#         uses: actions/github-script@v6.4.1
+#         env:
+#           JOB_STATUS: ${{ job.status }}
+#           STATUS_TARGET_COMMIT: ${{ github.event.pull_request.head.sha }}
+#         with:
+#           github-token: ${{secrets.RELEASE_PLEASE_TOKEN}}
+#           script: |
+#             const e2eStatus = require('./.github/scripts/js/e2e-commit-status');
+
+#             await e2eStatus.setStatusAfterE2eRun({github, context, core});
\ No newline at end of file

From c9beb6888a158df4658736503679945f4f85cc3b Mon Sep 17 00:00:00 2001
From: Maksim Fedotov <maksim.fedotov@flant.com>
Date: Mon, 7 Apr 2025 11:21:56 +0300
Subject: [PATCH 20/48] format yaml

Signed-off-by: Maksim Fedotov <maksim.fedotov@flant.com>
---
 .github/workflows/dev_module_build.yml |  8 ++++----
 .github/workflows/run_e2e.yml          | 14 +++++++-------
 2 files changed, 11 insertions(+), 11 deletions(-)

diff --git a/.github/workflows/dev_module_build.yml b/.github/workflows/dev_module_build.yml
index ccf8e8ceb7..beeda0c1ec 100644
--- a/.github/workflows/dev_module_build.yml
+++ b/.github/workflows/dev_module_build.yml
@@ -416,7 +416,7 @@ jobs:
     if: ${{ github.event.label.name == 'e2e/run' }}
     name: Run E2E tests
     runs-on: ubuntu-latest
-    needs: 
+    needs:
       - dev_setup_build
       - set_e2e_requirement_status
     steps:
@@ -449,7 +449,7 @@ jobs:
               // core.setSecret(kubeconfig);
               // core.setOutput('kubeconfig_data', kubeconfig);
             }
-      
+
       - name: Install Deckhouse-cli
         run: |
           echo "Install d8"
@@ -485,7 +485,7 @@ jobs:
           registry: ${{ vars.DEV_REGISTRY }}
           registry_login: ${{ vars.DEV_MODULES_REGISTRY_LOGIN }}
           registry_password: ${{ secrets.DEV_MODULES_REGISTRY_PASSWORD }}
-          
+
       - name: Checkout cluster to revision
         env:
           v12n_tag: pr${{ github.event.pull_request.number }}
@@ -571,4 +571,4 @@ jobs:
           script: |
             const e2eStatus = require('./.github/scripts/js/e2e-commit-status');
 
-            await e2eStatus.setStatusAfterE2eRun({github, context, core});
\ No newline at end of file
+            await e2eStatus.setStatusAfterE2eRun({github, context, core});
diff --git a/.github/workflows/run_e2e.yml b/.github/workflows/run_e2e.yml
index b90dbc26f3..2584b89541 100644
--- a/.github/workflows/run_e2e.yml
+++ b/.github/workflows/run_e2e.yml
@@ -2,7 +2,7 @@
 # on:
 #   # pull_request_target:
 #   pull_request:
-#     types: 
+#     types:
 #       - labeled
 
 # env:
@@ -66,7 +66,7 @@
 #               // core.setSecret(kubeconfig);
 #               // core.setOutput('kubeconfig_data', kubeconfig);
 #             }
-      
+
 #       - name: Install Deckhouse-cli
 #         run: |
 #           echo "Install d8"
@@ -102,17 +102,17 @@
 #           registry: ${{ vars.DEV_REGISTRY }}
 #           registry_login: ${{ vars.DEV_MODULES_REGISTRY_LOGIN }}
 #           registry_password: ${{ secrets.DEV_MODULES_REGISTRY_PASSWORD }}
-          
+
 #       - name: Checkout cluster to revision
 #         env:
 #           v12n_tag: pr${{ github.event.pull_request.number }}
 #         run: |
-#           d8 k patch mpo virtualization --type merge -p "{\"spec\":{\"imageTag\":\"$v12n_tag\"}}" 
+#           d8 k patch mpo virtualization --type merge -p "{\"spec\":{\"imageTag\":\"$v12n_tag\"}}"
 #           images_hash=$(crane export "dev-registry.deckhouse.io/sys/deckhouse-oss/modules/virtualization:$v12n_tag" - | tar -Oxf - images_digests.json)
 #           v12n_pods=$(kubectl -n d8-virtualization get pods -o json | jq -c)
 #           retry_count=0
 #           max_retries=120
-#           sleep_interval=5 
+#           sleep_interval=5
 
 #           while true; do
 #             all_hashes_found=true
@@ -141,7 +141,7 @@
 #             if [ "$all_hashes_found" = true ]; then
 #               echo "All image hashes found in pods."
 #               break
-#             fi 
+#             fi
 
 #             retry_count=$((retry_count + 1))
 #             echo "Some hashes are missing, rechecking... Attempt: $retry_count"
@@ -188,4 +188,4 @@
 #           script: |
 #             const e2eStatus = require('./.github/scripts/js/e2e-commit-status');
 
-#             await e2eStatus.setStatusAfterE2eRun({github, context, core});
\ No newline at end of file
+#             await e2eStatus.setStatusAfterE2eRun({github, context, core});

From bc072a5fae5a8b441a4a61009dfc37b7cf871ca6 Mon Sep 17 00:00:00 2001
From: Maksim Fedotov <maksim.fedotov@flant.com>
Date: Mon, 7 Apr 2025 14:04:14 +0300
Subject: [PATCH 21/48] change runner

Signed-off-by: Maksim Fedotov <maksim.fedotov@flant.com>
---
 .github/workflows/dev_module_build.yml | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/dev_module_build.yml b/.github/workflows/dev_module_build.yml
index beeda0c1ec..df149326a6 100644
--- a/.github/workflows/dev_module_build.yml
+++ b/.github/workflows/dev_module_build.yml
@@ -415,10 +415,11 @@ jobs:
   run_e2e:
     if: ${{ github.event.label.name == 'e2e/run' }}
     name: Run E2E tests
-    runs-on: ubuntu-latest
+    runs-on: ${{ fromJSON(needs.set_vars.outputs.runner_type)}}
     needs:
       - dev_setup_build
       - set_e2e_requirement_status
+      - set_vars
     steps:
       - uses: actions/checkout@v4
       - name: Select user

From 35dbae6e9dd2ccf29ac09903972b3dc45ddc6544 Mon Sep 17 00:00:00 2001
From: Maksim Fedotov <maksim.fedotov@flant.com>
Date: Mon, 7 Apr 2025 23:49:11 +0300
Subject: [PATCH 22/48] add cleanup

Signed-off-by: Maksim Fedotov <maksim.fedotov@flant.com>
---
 .github/workflows/dev_module_build.yml |  7 ++++++
 tests/e2e/Taskfile.yaml                | 13 ++++++++++
 tests/e2e/config/config.go             |  1 +
 tests/e2e/default_config.yaml          |  7 +++++-
 tests/e2e/tests_suite_test.go          | 33 ++++++++++----------------
 5 files changed, 40 insertions(+), 21 deletions(-)

diff --git a/.github/workflows/dev_module_build.yml b/.github/workflows/dev_module_build.yml
index df149326a6..5c8ec98e2a 100644
--- a/.github/workflows/dev_module_build.yml
+++ b/.github/workflows/dev_module_build.yml
@@ -551,6 +551,13 @@ jobs:
         run: |
           task run -v
 
+      - name: Cleanup E2E resources on cancel
+        if: always() && (steps.e2e-tests.outcome == 'cancelled' || steps.e2e-tests.outcome == 'failure')
+        id: e2e-tests-cleanup
+        working-directory: ./tests/e2e/
+        run: |
+          task cleanup
+
   last_comment:
     name: Update comment on finish
     needs:
diff --git a/tests/e2e/Taskfile.yaml b/tests/e2e/Taskfile.yaml
index 0f6aa48665..8ea8995b03 100644
--- a/tests/e2e/Taskfile.yaml
+++ b/tests/e2e/Taskfile.yaml
@@ -110,3 +110,16 @@ tasks:
           {{if .FOCUS -}}
           --focus "{{ .FOCUS }}"
           {{end -}}
+  cleanup:
+    desc: "Cleanup namespaces & resources left from e2e tests"
+    deps:
+      - kubectl
+      - d8
+    cmds: 
+      - |
+        E2E_PREFIX=head-$(git rev-parse --short HEAD)
+        readarray -t CLEANUP_RESOURCES < <(yq .cleanupResources[] default_config.yaml)
+        for RESOURCE in "${CLEANUP_RESOURCES[@]}"; do 
+            kubectl get $RESOURCE --no-headers | awk "/$E2E_PREFIX/{print \$1}" | xargs kubectl delete $RESOURCE 
+        done
+        kubectl get namespaces --no-headers | awk "/$E2E_PREFIX/{print \$1}" | xargs kubectl delete namespaces
diff --git a/tests/e2e/config/config.go b/tests/e2e/config/config.go
index 6d335b117d..88fde62036 100644
--- a/tests/e2e/config/config.go
+++ b/tests/e2e/config/config.go
@@ -131,6 +131,7 @@ type Config struct {
 	Namespace        string           `yaml:"namespaceSuffix"`
 	TestData         TestData         `yaml:"testData"`
 	LogFilter        []string         `yaml:"logFilter"`
+	CleanupResources []string         `yaml:"cleanupResources"`
 	RegexpLogFilter  []regexp.Regexp  `yaml:"regexpLogFilter"`
 	StorageClass     StorageClass
 }
diff --git a/tests/e2e/default_config.yaml b/tests/e2e/default_config.yaml
index adaafe08e3..0f4c6db057 100644
--- a/tests/e2e/default_config.yaml
+++ b/tests/e2e/default_config.yaml
@@ -52,6 +52,11 @@ logFilter:
   - "get storage class specified in spec: storage class not found" # Err.
   - "lastTransitionTime: Required value" # Err.
   - "virtualmachineipaddressleases.virtualization.deckhouse.io "
-regexpLogFilter:
   - "failed to detach: .* not found" # "err" "failed to detach: virtualmachine.kubevirt.io \"head-497d17b-vm-automatic-with-hotplug\" not found",
   - "error patching .* not found" # "err" "error patching *** virtualimages.virtualization.deckhouse.io \"head-497d17b-vi-pvc-oref-vi-oref-vd\" not found",
+
+cleanupResources:
+  - clustervirtualimages.virtualization.deckhouse.io
+  - virtualmachineclasses.virtualization.deckhouse.io
+  - replicatedstorageclasses.storage.deckhouse.io
+  - virtualmachineipaddressleases.virtualization.deckhouse.io
diff --git a/tests/e2e/tests_suite_test.go b/tests/e2e/tests_suite_test.go
index 80f83ccfa5..98360af7d5 100644
--- a/tests/e2e/tests_suite_test.go
+++ b/tests/e2e/tests_suite_test.go
@@ -210,26 +210,19 @@ func Cleanup() []error {
 			continue
 		}
 	}
-
-	res = kubectl.Delete(kc.DeleteOptions{
-		IgnoreNotFound: true,
-		Labels:         map[string]string{"id": namePrefix},
-		Resource:       kc.ResourceCVI,
-	})
-	if res.Error() != nil {
-		cleanupErrs = append(
-			cleanupErrs, fmt.Errorf("cmd: %s\nstderr: %s", res.GetCmd(), res.StdErr()),
-		)
-	}
-	res = kubectl.Delete(kc.DeleteOptions{
-		IgnoreNotFound: true,
-		Labels:         map[string]string{"id": namePrefix},
-		Resource:       kc.ResourceVMClass,
-	})
-	if res.Error() != nil {
-		cleanupErrs = append(
-			cleanupErrs, fmt.Errorf("cmd: %s\nstderr: %s", res.GetCmd(), res.StdErr()),
-		)
+	
+	for _, r := range conf.CleanupResources {
+		res = kubectl.Delete(kc.DeleteOptions{
+			IgnoreNotFound: true,
+			Labels:         map[string]string{"id": namePrefix},
+			Resource: 		kc.Resource(r),
+		})
+		if res.Error() != nil {
+			cleanupErrs = append(
+				cleanupErrs, fmt.Errorf("cmd: %s\nstderr: %s", res.GetCmd(), res.StdErr()),
+			)
+			continue
+		}
 	}
 
 	return cleanupErrs

From 34f308a57373be7b67e2e48860217b964c45f543 Mon Sep 17 00:00:00 2001
From: Maksim Fedotov <maksim.fedotov@flant.com>
Date: Mon, 7 Apr 2025 23:56:13 +0300
Subject: [PATCH 23/48] return regexpLogFilter

Signed-off-by: Maksim Fedotov <maksim.fedotov@flant.com>
---
 tests/e2e/default_config.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/e2e/default_config.yaml b/tests/e2e/default_config.yaml
index 0f4c6db057..f0e32fa5c3 100644
--- a/tests/e2e/default_config.yaml
+++ b/tests/e2e/default_config.yaml
@@ -52,6 +52,7 @@ logFilter:
   - "get storage class specified in spec: storage class not found" # Err.
   - "lastTransitionTime: Required value" # Err.
   - "virtualmachineipaddressleases.virtualization.deckhouse.io "
+regexpLogFilter:
   - "failed to detach: .* not found" # "err" "failed to detach: virtualmachine.kubevirt.io \"head-497d17b-vm-automatic-with-hotplug\" not found",
   - "error patching .* not found" # "err" "error patching *** virtualimages.virtualization.deckhouse.io \"head-497d17b-vi-pvc-oref-vi-oref-vd\" not found",
 

From 6673ab1a59bd00a95e71685bd9b82ce08590af1b Mon Sep 17 00:00:00 2001
From: Maksim Fedotov <maksim.fedotov@flant.com>
Date: Tue, 8 Apr 2025 00:00:44 +0300
Subject: [PATCH 24/48] set runs-on: ubuntu-latest & remove comments

Signed-off-by: Maksim Fedotov <maksim.fedotov@flant.com>
---
 .github/workflows/dev_module_build.yml | 14 ++++----------
 1 file changed, 4 insertions(+), 10 deletions(-)

diff --git a/.github/workflows/dev_module_build.yml b/.github/workflows/dev_module_build.yml
index 5c8ec98e2a..6980be4441 100644
--- a/.github/workflows/dev_module_build.yml
+++ b/.github/workflows/dev_module_build.yml
@@ -336,6 +336,7 @@ jobs:
           module_source: ${{ vars.DEV_MODULE_SOURCE}}
           module_name: ${{ vars.MODULE_NAME }}
           module_tag: "$MODULES_MODULE_TAG"
+
   pull_request_info:
     name: Get PR info
     runs-on: ubuntu-latest
@@ -354,6 +355,7 @@ jobs:
               issue_number: prNumber
             });
             core.setOutput('labels', JSON.stringify(labels));
+
   set_e2e_requirement_status:
     name: Set 'waiting for e2e' commit status
     needs:
@@ -373,6 +375,7 @@ jobs:
           script: |
             const e2eStatus = require('./.github/scripts/js/e2e-commit-status');
             await e2eStatus.setInitialStatus({github, context, core});
+
   cve_scan_on_pr:
     name: Trivy images check
     runs-on: ${{ fromJSON(needs.set_vars.outputs.runner_type)}}
@@ -415,7 +418,7 @@ jobs:
   run_e2e:
     if: ${{ github.event.label.name == 'e2e/run' }}
     name: Run E2E tests
-    runs-on: ${{ fromJSON(needs.set_vars.outputs.runner_type)}}
+    runs-on: ubuntu-latest
     needs:
       - dev_setup_build
       - set_e2e_requirement_status
@@ -472,15 +475,6 @@ jobs:
           GINKGO_VERSION=$(go list -f '{{.Version}}' -m github.com/onsi/ginkgo/v2)
           go install "github.com/onsi/ginkgo/v2/ginkgo@${GINKGO_VERSION}"
 
-      # - name: Install crane
-      #   run: |
-      #     go install github.com/google/go-containerregistry/cmd/crane@latest
-
-      # - uses: azure/k8s-set-context@v4
-      #   with:
-      #     method: kubeconfig
-      #     kubeconfig: ${{ steps.select_user.outputs.kubeconfig_data }}
-
       - uses: deckhouse/modules-actions/setup@v2
         with:
           registry: ${{ vars.DEV_REGISTRY }}

From eea9cc82b9663593c21ffe548d86600e526b2e2f Mon Sep 17 00:00:00 2001
From: Maksim Fedotov <maksim.fedotov@flant.com>
Date: Tue, 8 Apr 2025 00:11:01 +0300
Subject: [PATCH 25/48] change order of resource cleanup

Signed-off-by: Maksim Fedotov <maksim.fedotov@flant.com>
---
 tests/e2e/Taskfile.yaml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/tests/e2e/Taskfile.yaml b/tests/e2e/Taskfile.yaml
index 8ea8995b03..a763002b8b 100644
--- a/tests/e2e/Taskfile.yaml
+++ b/tests/e2e/Taskfile.yaml
@@ -118,8 +118,8 @@ tasks:
     cmds: 
       - |
         E2E_PREFIX=head-$(git rev-parse --short HEAD)
+        kubectl get namespaces --no-headers | awk "/$E2E_PREFIX/{print \$1}" | xargs kubectl delete namespaces
         readarray -t CLEANUP_RESOURCES < <(yq .cleanupResources[] default_config.yaml)
         for RESOURCE in "${CLEANUP_RESOURCES[@]}"; do 
             kubectl get $RESOURCE --no-headers | awk "/$E2E_PREFIX/{print \$1}" | xargs kubectl delete $RESOURCE 
         done
-        kubectl get namespaces --no-headers | awk "/$E2E_PREFIX/{print \$1}" | xargs kubectl delete namespaces

From b0d79a5be56139d462db267c17c601cfb77a3d87 Mon Sep 17 00:00:00 2001
From: Maksim Fedotov <maksim.fedotov@flant.com>
Date: Tue, 8 Apr 2025 00:14:37 +0300
Subject: [PATCH 26/48] chore

Signed-off-by: Maksim Fedotov <maksim.fedotov@flant.com>
---
 .github/workflows/run_e2e.yml | 191 ----------------------------------
 tests/e2e/Taskfile.yaml       |   2 +-
 2 files changed, 1 insertion(+), 192 deletions(-)
 delete mode 100644 .github/workflows/run_e2e.yml

diff --git a/.github/workflows/run_e2e.yml b/.github/workflows/run_e2e.yml
deleted file mode 100644
index 2584b89541..0000000000
--- a/.github/workflows/run_e2e.yml
+++ /dev/null
@@ -1,191 +0,0 @@
-# name: Run e2e
-# on:
-#   # pull_request_target:
-#   pull_request:
-#     types:
-#       - labeled
-
-# env:
-#   GO_VERSION: "1.22.7"
-
-# defaults:
-#   run:
-#     shell: bash
-
-# jobs:
-#   skip_e2e:
-#     if: ${{ github.event.label.name == 'skip/e2e' }}
-#     runs-on: ubuntu-latest
-#     steps:
-#       - uses: actions/checkout@v4
-#       - name: Skip E2E tests
-#         id: skip_e2e
-#         uses: actions/github-script@v6
-#         with:
-#           script: |
-#             const e2eStatus = require('./.github/scripts/js/e2e-commit-status');
-#             e2eStatus.onLabeledForSkip({
-#               github,
-#               context,
-#               core,
-#               labeled: true,
-#               commitSha: context.payload.pull_request.head.sha
-#             })
-
-#   run_e2e:
-#     if: ${{ github.event.label.name == 'e2e/run' }}
-#     name: Run E2E tests
-#     runs-on: ubuntu-latest
-#     steps:
-#       - uses: actions/checkout@v4
-#       - name: Select user
-#         id: select_user
-#         uses: actions/github-script@v6
-#         env:
-#           KUBECONFIGS: ${{ secrets.K8S_CLUSTER_SECRET }}
-#         with:
-#           script: |
-#             const ci = require('./.github/scripts/js/ci');
-#             const userId = await ci.getClusterUser({context, core});
-#             const fs = require('fs');
-#             const path = require('path');
-#             // core.setOutput("user_id", user_id)
-#             // core.exportVariable("user_id", user_id);
-#             const kubeconfigs = JSON.parse(process.env.KUBECONFIGS);
-#             const kubeconfig = kubeconfigs.find(config => config.id === userId)?.kubeconfig;
-#             if (!kubeconfig) {
-#               core.setFailed(`No kubeconfig found for user with ID ${userId}.`);
-#             } else {
-#               core.info(`Found kubeconfig for user with ID ${userId}`);
-#               const runnerTempDir = process.env['RUNNER_TEMP'];
-#               const kubeconfigFile = path.join(runnerTempDir, `kubeconfig_${Date.now()}`);
-#               fs.writeFileSync(kubeconfigFile, kubeconfig);
-#               fs.chmodSync(kubeconfigFile, '600');
-#               core.exportVariable('KUBECONFIG', kubeconfigFile)
-
-#               // core.setSecret(kubeconfig);
-#               // core.setOutput('kubeconfig_data', kubeconfig);
-#             }
-
-#       - name: Install Deckhouse-cli
-#         run: |
-#           echo "Install d8"
-#           curl -fsSL -o d8-install.sh https://raw.githubusercontent.com/deckhouse/deckhouse-cli/main/d8-install.sh
-#           bash d8-install.sh
-
-#       - name: Set up Go ${{ env.GO_VERSION }}
-#         uses: actions/setup-go@v5
-#         with:
-#           go-version: "${{ env.GO_VERSION }}"
-
-#       - name: Install Task
-#         uses: arduino/setup-task@v2
-
-#       - name: Install ginkgo
-#         working-directory: ./tests/e2e/
-#         run: |
-#           echo "Install ginkgo"
-#           GINKGO_VERSION=$(go list -f '{{.Version}}' -m github.com/onsi/ginkgo/v2)
-#           go install "github.com/onsi/ginkgo/v2/ginkgo@${GINKGO_VERSION}"
-
-#       # - name: Install crane
-#       #   run: |
-#       #     go install github.com/google/go-containerregistry/cmd/crane@latest
-
-#       # - uses: azure/k8s-set-context@v4
-#       #   with:
-#       #     method: kubeconfig
-#       #     kubeconfig: ${{ steps.select_user.outputs.kubeconfig_data }}
-
-#       - uses: deckhouse/modules-actions/setup@v2
-#         with:
-#           registry: ${{ vars.DEV_REGISTRY }}
-#           registry_login: ${{ vars.DEV_MODULES_REGISTRY_LOGIN }}
-#           registry_password: ${{ secrets.DEV_MODULES_REGISTRY_PASSWORD }}
-
-#       - name: Checkout cluster to revision
-#         env:
-#           v12n_tag: pr${{ github.event.pull_request.number }}
-#         run: |
-#           d8 k patch mpo virtualization --type merge -p "{\"spec\":{\"imageTag\":\"$v12n_tag\"}}"
-#           images_hash=$(crane export "dev-registry.deckhouse.io/sys/deckhouse-oss/modules/virtualization:$v12n_tag" - | tar -Oxf - images_digests.json)
-#           v12n_pods=$(kubectl -n d8-virtualization get pods -o json | jq -c)
-#           retry_count=0
-#           max_retries=120
-#           sleep_interval=5
-
-#           while true; do
-#             all_hashes_found=true
-
-#             # Fetch current pods information
-#             v12n_pods=$(kubectl -n d8-virtualization get pods -o json | jq -c)
-
-#             # Process each image entry
-#             while IFS= read -r image_entry; do
-#               image=$(echo "$image_entry" | jq -r '.key')
-#               hash=$(echo "$image_entry" | jq -r '.value')
-
-#               if [[ "${image,,}" =~ (libguestfs|predeletehook) ]]; then
-#                 continue
-#               fi
-
-#               if echo "$v12n_pods" | grep -q "$hash"; then
-#                 echo "- ✅ $image $hash"
-#               else
-#                 echo "- 🟥 $image $hash"
-#                 all_hashes_found=false
-#               fi
-#             done < <(echo "$images_hash" | jq -c '. | to_entries | sort_by(.key)[]')
-
-#             # If all hashes are found, break the loop
-#             if [ "$all_hashes_found" = true ]; then
-#               echo "All image hashes found in pods."
-#               break
-#             fi
-
-#             retry_count=$((retry_count + 1))
-#             echo "Some hashes are missing, rechecking... Attempt: $retry_count"
-
-#             # Check if the retry limit has been reached
-#             if [ "$retry_count" -ge "$max_retries" ]; then
-#               echo "Error: Timeout reached after $((retry_count * sleep_interval)) seconds. Some image hashes are still missing."
-#               exit 1
-#             fi
-#             # Wait for the specified interval before retrying
-#             sleep "$sleep_interval"
-#           done
-
-#       - name: Download dependencies
-#         working-directory: ./tests/e2e/
-#         run: |
-#           echo "Download dependencies"
-#           go mod download
-
-#       - name: Run E2E
-#         id: e2e-tests
-#         working-directory: ./tests/e2e/
-#         run: |
-#           task run -v
-
-#   last_comment:
-#     name: Update comment on finish
-#     needs:
-#       - run_e2e
-#     runs-on: ubuntu-latest
-#     steps:
-#       - name: Checkout sources
-#         uses: actions/checkout@v3.5.2
-
-#       - name: Set commit status after e2e run
-#         id: set_e2e_requirement_status
-#         if: ${{ always() }}
-#         uses: actions/github-script@v6.4.1
-#         env:
-#           JOB_STATUS: ${{ job.status }}
-#           STATUS_TARGET_COMMIT: ${{ github.event.pull_request.head.sha }}
-#         with:
-#           github-token: ${{secrets.RELEASE_PLEASE_TOKEN}}
-#           script: |
-#             const e2eStatus = require('./.github/scripts/js/e2e-commit-status');
-
-#             await e2eStatus.setStatusAfterE2eRun({github, context, core});
diff --git a/tests/e2e/Taskfile.yaml b/tests/e2e/Taskfile.yaml
index a763002b8b..0b274dc079 100644
--- a/tests/e2e/Taskfile.yaml
+++ b/tests/e2e/Taskfile.yaml
@@ -115,7 +115,7 @@ tasks:
     deps:
       - kubectl
       - d8
-    cmds: 
+    cmds:
       - |
         E2E_PREFIX=head-$(git rev-parse --short HEAD)
         kubectl get namespaces --no-headers | awk "/$E2E_PREFIX/{print \$1}" | xargs kubectl delete namespaces

From 8a44fe3f519f692846b73ff48332b7d447e0ffee Mon Sep 17 00:00:00 2001
From: Maksim Fedotov <maksim.fedotov@flant.com>
Date: Tue, 8 Apr 2025 10:13:12 +0300
Subject: [PATCH 27/48] fix condition

Signed-off-by: Maksim Fedotov <maksim.fedotov@flant.com>
---
 .github/workflows/dev_module_build.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/dev_module_build.yml b/.github/workflows/dev_module_build.yml
index 6980be4441..379f61256f 100644
--- a/.github/workflows/dev_module_build.yml
+++ b/.github/workflows/dev_module_build.yml
@@ -546,7 +546,7 @@ jobs:
           task run -v
 
       - name: Cleanup E2E resources on cancel
-        if: always() && (steps.e2e-tests.outcome == 'cancelled' || steps.e2e-tests.outcome == 'failure')
+        if: always() && steps.e2e-tests.outcome == 'cancelled'
         id: e2e-tests-cleanup
         working-directory: ./tests/e2e/
         run: |

From 7f0eed8d420e026c7488b0934841cec426338ad6 Mon Sep 17 00:00:00 2001
From: Maksim Fedotov <maksim.fedotov@flant.com>
Date: Tue, 8 Apr 2025 10:54:24 +0300
Subject: [PATCH 28/48] add project cleanup

Signed-off-by: Maksim Fedotov <maksim.fedotov@flant.com>
---
 tests/e2e/Taskfile.yaml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tests/e2e/Taskfile.yaml b/tests/e2e/Taskfile.yaml
index 0b274dc079..8843c080eb 100644
--- a/tests/e2e/Taskfile.yaml
+++ b/tests/e2e/Taskfile.yaml
@@ -119,6 +119,7 @@ tasks:
       - |
         E2E_PREFIX=head-$(git rev-parse --short HEAD)
         kubectl get namespaces --no-headers | awk "/$E2E_PREFIX/{print \$1}" | xargs kubectl delete namespaces
+        kubectl get projects --no-headers | awk "/$E2E_PREFIX/{print \$1}" | xargs kubectl delete projects
         readarray -t CLEANUP_RESOURCES < <(yq .cleanupResources[] default_config.yaml)
         for RESOURCE in "${CLEANUP_RESOURCES[@]}"; do 
             kubectl get $RESOURCE --no-headers | awk "/$E2E_PREFIX/{print \$1}" | xargs kubectl delete $RESOURCE 

From 0d16b90d4d47d89082697cc947b05c8f081c5845 Mon Sep 17 00:00:00 2001
From: Maksim Fedotov <maksim.fedotov@flant.com>
Date: Tue, 8 Apr 2025 11:55:34 +0300
Subject: [PATCH 29/48] add user cluster labels

Signed-off-by: Maksim Fedotov <maksim.fedotov@flant.com>
---
 .github/scripts/js/constants.js | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/.github/scripts/js/constants.js b/.github/scripts/js/constants.js
index 4633e50114..2ab8408d3a 100644
--- a/.github/scripts/js/constants.js
+++ b/.github/scripts/js/constants.js
@@ -27,7 +27,16 @@ const labels = {
 module.exports.knownLabels = labels;
 const userClusterLables = {
   'e2e/user/nevermarine' : { type: 'e2e-user', id: '72984806'},
-  'e2e/user/Isteb4k' : { type: 'e2e-user', id: '93128416'}
+  'e2e/user/Isteb4k' : { type: 'e2e-user', id: '93128416'},
+  'e2e/user/fl64' : { type: 'e2e-user', id: '2950658'},
+  'e2e/user/hardcoretime' : { type: 'e2e-user', id: '36233932'},
+  'e2e/user/yaroslavborbat' : { type: 'e2e-user', id: '86148689'},
+  'e2e/user/danilrwx' : { type: 'e2e-user', id: '22664775'},
+  'e2e/user/eofff' : { type: 'e2e-user', id: '7936159'},
+  'e2e/user/LopatinDmitr' : { type: 'e2e-user', id: '93423466'},
+  'e2e/user/universal-itengineer' : { type: 'e2e-user', id: '141920865'},
+  'e2e/user/hayer969' : { type: 'e2e-user', id: '74240854'},
+
 };
 module.exports.userClusterLabels = userClusterLables;
 

From 60a7f2acf22ef15c15adba2820530a505aee23a6 Mon Sep 17 00:00:00 2001
From: Maksim Fedotov <maksim.fedotov@flant.com>
Date: Tue, 8 Apr 2025 12:08:48 +0300
Subject: [PATCH 30/48] run e2e only on pull requests

Signed-off-by: Maksim Fedotov <maksim.fedotov@flant.com>
---
 .github/workflows/dev_module_build.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/dev_module_build.yml b/.github/workflows/dev_module_build.yml
index 379f61256f..099c8d3733 100644
--- a/.github/workflows/dev_module_build.yml
+++ b/.github/workflows/dev_module_build.yml
@@ -416,7 +416,7 @@ jobs:
             })
 
   run_e2e:
-    if: ${{ github.event.label.name == 'e2e/run' }}
+    if: ${{ github.event_name == 'pull_request' && github.event.action == 'labeled' && github.event.label.name == 'e2e/run' }}
     name: Run E2E tests
     runs-on: ubuntu-latest
     needs:

From ad228d04fb30efb3b94265a9548babfb71877ec7 Mon Sep 17 00:00:00 2001
From: Maksim Fedotov <maksim.fedotov@flant.com>
Date: Tue, 8 Apr 2025 14:30:37 +0300
Subject: [PATCH 31/48] fix cleanup failing on missing resources

Signed-off-by: Maksim Fedotov <maksim.fedotov@flant.com>
---
 tests/e2e/Taskfile.yaml | 33 +++++++++++++++++++++++++++------
 1 file changed, 27 insertions(+), 6 deletions(-)

diff --git a/tests/e2e/Taskfile.yaml b/tests/e2e/Taskfile.yaml
index 8843c080eb..651050b9a1 100644
--- a/tests/e2e/Taskfile.yaml
+++ b/tests/e2e/Taskfile.yaml
@@ -117,10 +117,31 @@ tasks:
       - d8
     cmds:
       - |
-        E2E_PREFIX=head-$(git rev-parse --short HEAD)
-        kubectl get namespaces --no-headers | awk "/$E2E_PREFIX/{print \$1}" | xargs kubectl delete namespaces
-        kubectl get projects --no-headers | awk "/$E2E_PREFIX/{print \$1}" | xargs kubectl delete projects
-        readarray -t CLEANUP_RESOURCES < <(yq .cleanupResources[] default_config.yaml)
-        for RESOURCE in "${CLEANUP_RESOURCES[@]}"; do 
-            kubectl get $RESOURCE --no-headers | awk "/$E2E_PREFIX/{print \$1}" | xargs kubectl delete $RESOURCE 
+        E2E_PREFIX="head-$(git rev-parse --short HEAD)"
+
+        # Delete matching namespaces
+        namespaces=$(kubectl get namespaces --no-headers | awk "/$E2E_PREFIX/ {print \$1}")
+        if [[ -n "$namespaces" ]]; then
+          echo "$namespaces" | xargs -r kubectl delete namespaces
+        else
+          echo "No namespaces found with prefix $E2E_PREFIX"
+        fi
+
+        # Delete matching projects
+        projects=$(kubectl get projects --no-headers | awk "/$E2E_PREFIX/ {print \$1}")
+        if [[ -n "$projects" ]]; then
+          echo "$projects" | xargs -r kubectl delete projects
+        else
+          echo "No projects found with prefix $E2E_PREFIX"
+        fi
+
+        # Load cleanup resources from yaml, and delete resources matching the prefix
+        readarray -t CLEANUP_RESOURCES < <(yq '.cleanupResources[]' default_config.yaml)
+        for RESOURCE in "${CLEANUP_RESOURCES[@]}"; do
+          items=$(kubectl get "$RESOURCE" --no-headers 2>/dev/null | awk "/$E2E_PREFIX/ {print \$1}")
+          if [[ -n "$items" ]]; then
+            echo "$items" | xargs -r kubectl delete "$RESOURCE"
+          else
+            echo "No $RESOURCE found with prefix $E2E_PREFIX"
+          fi
         done

From e55b87a61010260810c0a0a71ff1583b3a92f02e Mon Sep 17 00:00:00 2001
From: Maksim Fedotov <maksim.fedotov@flant.com>
Date: Tue, 8 Apr 2025 23:27:56 +0300
Subject: [PATCH 32/48] remove comments

Signed-off-by: Maksim Fedotov <maksim.fedotov@flant.com>
---
 .github/scripts/js/ci.js               | 2 --
 .github/workflows/dev_module_build.yml | 5 -----
 2 files changed, 7 deletions(-)

diff --git a/.github/scripts/js/ci.js b/.github/scripts/js/ci.js
index 67b97e7ed2..15c878d2bb 100644
--- a/.github/scripts/js/ci.js
+++ b/.github/scripts/js/ci.js
@@ -85,8 +85,6 @@ const getClusterUser = async ({context, core}) => {
     const prAuthorId = context.payload.pull_request.user.id;
     core.info(`PR author: ${prAuthorId}`);
     return prAuthorId.toString();
-    // const authorLabel = [{'name': `e2e/user/${prAuthor}`}];
-    // retry check for PR author's cluster label
   } else if (userLabelsInPR.length > 1) {
     return core.setFailed(`Error: PR has multiple user labels: ${userLabelsInPR.join(', ')}`);
   }
diff --git a/.github/workflows/dev_module_build.yml b/.github/workflows/dev_module_build.yml
index 099c8d3733..90a82ee41c 100644
--- a/.github/workflows/dev_module_build.yml
+++ b/.github/workflows/dev_module_build.yml
@@ -436,8 +436,6 @@ jobs:
             const userId = await ci.getClusterUser({context, core});
             const fs = require('fs');
             const path = require('path');
-            // core.setOutput("user_id", user_id)
-            // core.exportVariable("user_id", user_id);
             const kubeconfigs = JSON.parse(process.env.KUBECONFIGS);
             const kubeconfig = kubeconfigs.find(config => config.id === userId)?.kubeconfig;
             if (!kubeconfig) {
@@ -449,9 +447,6 @@ jobs:
               fs.writeFileSync(kubeconfigFile, kubeconfig);
               fs.chmodSync(kubeconfigFile, '600');
               core.exportVariable('KUBECONFIG', kubeconfigFile)
-
-              // core.setSecret(kubeconfig);
-              // core.setOutput('kubeconfig_data', kubeconfig);
             }
 
       - name: Install Deckhouse-cli

From 295ffda75446e9b67bb5aec8fa79f5c48b307e00 Mon Sep 17 00:00:00 2001
From: Maksim Fedotov <maksim.fedotov@flant.com>
Date: Tue, 8 Apr 2025 23:44:56 +0300
Subject: [PATCH 33/48] cleanup constants & move to vars.USER_CLUSTER_LABELS

Signed-off-by: Maksim Fedotov <maksim.fedotov@flant.com>
---
 .github/scripts/js/ci.js               |  9 ++---
 .github/scripts/js/constants.js        | 54 --------------------------
 .github/workflows/dev_module_build.yml |  4 +-
 3 files changed, 6 insertions(+), 61 deletions(-)

diff --git a/.github/scripts/js/ci.js b/.github/scripts/js/ci.js
index 15c878d2bb..261e9eaa03 100644
--- a/.github/scripts/js/ci.js
+++ b/.github/scripts/js/ci.js
@@ -25,9 +25,6 @@
  * @returns {object}
  */
 
-const {
-	userClusterLabels
-  } = require('./constants');
 const { dumpError } = require('./error');
 const extractCommandFromComment = (comment) => {
 	// Split comment to lines.
@@ -69,7 +66,7 @@ const reactToComment = async ({github, context, comment_id, content}) => {
 };
 module.exports.reactToComment = reactToComment;
 
-const checkUserClusterLabel = async ({prLabels}) => {
+const checkUserClusterLabel = async ({prLabels, userClusterLabels}) => {
     const userLabelsInPR = prLabels
         .map(label => label.name)
         .filter(labelName => userClusterLabels[labelName]);
@@ -77,9 +74,9 @@ const checkUserClusterLabel = async ({prLabels}) => {
 };
 module.exports.checkUserClusterLabel = checkUserClusterLabel;
 
-const getClusterUser = async ({context, core}) => {
+const getClusterUser = async ({context, core, userClusterLabels}) => {
   const prLabels = context.payload.pull_request.labels;
-  let userLabelsInPR = await checkUserClusterLabel({prLabels});
+  let userLabelsInPR = await checkUserClusterLabel({prLabels, userClusterLabels});
   if (userLabelsInPR.length === 0) {
     core.info('No user labels found in PR, using PR author\'s cluster');
     const prAuthorId = context.payload.pull_request.user.id;
diff --git a/.github/scripts/js/constants.js b/.github/scripts/js/constants.js
index 2ab8408d3a..66a8f9a819 100644
--- a/.github/scripts/js/constants.js
+++ b/.github/scripts/js/constants.js
@@ -25,20 +25,6 @@ const labels = {
 
 };
 module.exports.knownLabels = labels;
-const userClusterLables = {
-  'e2e/user/nevermarine' : { type: 'e2e-user', id: '72984806'},
-  'e2e/user/Isteb4k' : { type: 'e2e-user', id: '93128416'},
-  'e2e/user/fl64' : { type: 'e2e-user', id: '2950658'},
-  'e2e/user/hardcoretime' : { type: 'e2e-user', id: '36233932'},
-  'e2e/user/yaroslavborbat' : { type: 'e2e-user', id: '86148689'},
-  'e2e/user/danilrwx' : { type: 'e2e-user', id: '22664775'},
-  'e2e/user/eofff' : { type: 'e2e-user', id: '7936159'},
-  'e2e/user/LopatinDmitr' : { type: 'e2e-user', id: '93423466'},
-  'e2e/user/universal-itengineer' : { type: 'e2e-user', id: '141920865'},
-  'e2e/user/hayer969' : { type: 'e2e-user', id: '74240854'},
-
-};
-module.exports.userClusterLabels = userClusterLables;
 
 // Label to detect if issue is a release issue.
 const releaseIssueLabel = 'issue/release';
@@ -77,43 +63,3 @@ module.exports.labelsSrv = {
     }) || [''])[0];
   }
 };
-
-// Providers for e2e tests.
-const providers = Object.entries(labels)
-  .filter(([name, info]) => info.type === 'e2e-run')
-  .map(([name, info]) => info.provider)
-  .sort();
-module.exports.knownProviders = providers;
-
-// Channels available for deploy.
-const channels = [
-  //
-  'alpha',
-  'beta',
-  'early-access',
-  'stable',
-  'rock-solid'
-];
-
-module.exports.knownChannels = channels;
-
-const criNames = Object.entries(labels)
-  .filter(([name, info]) => info.type === 'e2e-use' && !!info.cri)
-  .map(([name, info]) => info.cri);
-module.exports.knownCRINames = criNames;
-
-const kubernetesVersions = Object.entries(labels)
-  .filter(([name, info]) => info.type === 'e2e-use' && !!info.ver)
-  .map(([name, info]) => info.ver)
-  .sort();
-module.exports.knownKubernetesVersions = kubernetesVersions;
-
-module.exports.e2eDefaults = {
-  criName: 'Containerd',
-  edition: 'FE',
-  multimaster: false,
-  cis: false
-};
-
-const editions = ['CE', 'EE', 'FE', 'BE', 'SE', 'SE-plus'];
-module.exports.knownEditions = editions;
diff --git a/.github/workflows/dev_module_build.yml b/.github/workflows/dev_module_build.yml
index 90a82ee41c..a203037c04 100644
--- a/.github/workflows/dev_module_build.yml
+++ b/.github/workflows/dev_module_build.yml
@@ -430,10 +430,12 @@ jobs:
         uses: actions/github-script@v6
         env:
           KUBECONFIGS: ${{ secrets.K8S_CLUSTER_SECRET }}
+          USER_CLUSTER_LABELS: ${{ vars.USER_CLUSTER_LABELS }}
         with:
           script: |
             const ci = require('./.github/scripts/js/ci');
-            const userId = await ci.getClusterUser({context, core});
+            const userClusterLabels = JSON.parse(process.env.USER_CLUSTER_LABELS);
+            const userId = await ci.getClusterUser({context, core, userClusterLabels});
             const fs = require('fs');
             const path = require('path');
             const kubeconfigs = JSON.parse(process.env.KUBECONFIGS);

From aa57d4b8f01d2472794313b149301cac07dea83c Mon Sep 17 00:00:00 2001
From: Maksim Fedotov <maksim.fedotov@flant.com>
Date: Wed, 9 Apr 2025 13:49:26 +0300
Subject: [PATCH 34/48] add error handling for user cluster labels

Signed-off-by: Maksim Fedotov <maksim.fedotov@flant.com>
---
 .github/workflows/dev_module_build.yml | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/dev_module_build.yml b/.github/workflows/dev_module_build.yml
index a203037c04..550a90cd5d 100644
--- a/.github/workflows/dev_module_build.yml
+++ b/.github/workflows/dev_module_build.yml
@@ -434,7 +434,16 @@ jobs:
         with:
           script: |
             const ci = require('./.github/scripts/js/ci');
-            const userClusterLabels = JSON.parse(process.env.USER_CLUSTER_LABELS);
+            let userClusterLabels;
+            try {
+              if (!process.env.USER_CLUSTER_LABELS || process.env.USER_CLUSTER_LABELS.trim() === '') {
+                throw new Error('USER_CLUSTER_LABELS is empty or not provided.');
+              }
+              userClusterLabels = JSON.parse(process.env.USER_CLUSTER_LABELS);
+            } catch (error) {
+              core.setFailed(`Failed to parse USER_CLUSTER_LABELS: ${error.message}`);
+              return;
+            }
             const userId = await ci.getClusterUser({context, core, userClusterLabels});
             const fs = require('fs');
             const path = require('path');

From 5c2c94b69a2a0663176cd4f8454d48ae757a9b66 Mon Sep 17 00:00:00 2001
From: Maksim Fedotov <maksim.fedotov@flant.com>
Date: Thu, 10 Apr 2025 11:49:26 +0300
Subject: [PATCH 35/48] remove old code

Signed-off-by: Maksim Fedotov <maksim.fedotov@flant.com>
---
 .github/scripts/js/constants.js | 40 ---------------------------------
 1 file changed, 40 deletions(-)

diff --git a/.github/scripts/js/constants.js b/.github/scripts/js/constants.js
index 66a8f9a819..7515864230 100644
--- a/.github/scripts/js/constants.js
+++ b/.github/scripts/js/constants.js
@@ -12,9 +12,7 @@
 //@ts-check
 
 const skipE2eLabel = 'skip/e2e';
-const abortFailedE2eCommand = '/e2e/abort';
 module.exports.skipE2eLabel = skipE2eLabel;
-module.exports.abortFailedE2eCommand = abortFailedE2eCommand;
 
 // Labels available for pull requests.
 const labels = {
@@ -25,41 +23,3 @@ const labels = {
 
 };
 module.exports.knownLabels = labels;
-
-// Label to detect if issue is a release issue.
-const releaseIssueLabel = 'issue/release';
-module.exports.releaseIssueLabel = releaseIssueLabel;
-
-const slashCommands = {
-  deploy: ['deploy/alpha', 'deploy/beta', 'deploy/early-access', 'deploy/stable', 'deploy/rock-solid'],
-  suspend: ['suspend/alpha', 'suspend/beta', 'suspend/early-access', 'suspend/stable', 'suspend/rock-solid']
-};
-module.exports.knownSlashCommands = slashCommands;
-
-module.exports.labelsSrv = {
-  /**
-   * Search for known label name using its type and property:
-   * - search by provider property for e2e-run labels
-   * - search by env property for deploy-web labels
-   *
-   * @param {object} inputs
-   * @param {string} inputs.labelType
-   * @param {string} inputs.labelSubject
-   * @returns {string}
-   */
-  findLabel: ({ labelType, labelSubject }) => {
-    return (Object.entries(labels).find(([name, info]) => {
-      if (info.type === labelType) {
-        if (labelType === 'e2e-run') {
-          return info.provider === labelSubject;
-        }
-        if (labelType === 'deploy-web') {
-          return info.env === labelSubject;
-        }
-
-        return true;
-      }
-      return false;
-    }) || [''])[0];
-  }
-};

From 98118301d3fdb1db44b4a80c6d437427529e492b Mon Sep 17 00:00:00 2001
From: Maksim Fedotov <maksim.fedotov@flant.com>
Date: Thu, 10 Apr 2025 11:57:00 +0300
Subject: [PATCH 36/48] refactor cleanup

Signed-off-by: Maksim Fedotov <maksim.fedotov@flant.com>
---
 tests/e2e/Taskfile.yaml | 37 +++++++++++++++----------------------
 1 file changed, 15 insertions(+), 22 deletions(-)

diff --git a/tests/e2e/Taskfile.yaml b/tests/e2e/Taskfile.yaml
index 651050b9a1..a799e13875 100644
--- a/tests/e2e/Taskfile.yaml
+++ b/tests/e2e/Taskfile.yaml
@@ -119,29 +119,22 @@ tasks:
       - |
         E2E_PREFIX="head-$(git rev-parse --short HEAD)"
 
-        # Delete matching namespaces
-        namespaces=$(kubectl get namespaces --no-headers | awk "/$E2E_PREFIX/ {print \$1}")
-        if [[ -n "$namespaces" ]]; then
-          echo "$namespaces" | xargs -r kubectl delete namespaces
-        else
-          echo "No namespaces found with prefix $E2E_PREFIX"
-        fi
+        delete_resources_with_prefix() {
+            local RESOURCE_TYPE=$1
+            local RESOURCES=$(kubectl get "$RESOURCE_TYPE" --no-headers 2>/dev/null | awk "/$E2E_PREFIX/ {print \$1}")
+            
+            if [[ -n "$RESOURCES" ]]; then
+                echo "$RESOURCES" | xargs -r kubectl delete "$RESOURCE_TYPE"
+            else
+                echo "No $RESOURCE_TYPE found with prefix $E2E_PREFIX"
+            fi
+        }
 
-        # Delete matching projects
-        projects=$(kubectl get projects --no-headers | awk "/$E2E_PREFIX/ {print \$1}")
-        if [[ -n "$projects" ]]; then
-          echo "$projects" | xargs -r kubectl delete projects
-        else
-          echo "No projects found with prefix $E2E_PREFIX"
-        fi
+        delete_resources_with_prefix "namespaces"
+
+        delete_resources_with_prefix "projects"
 
-        # Load cleanup resources from yaml, and delete resources matching the prefix
         readarray -t CLEANUP_RESOURCES < <(yq '.cleanupResources[]' default_config.yaml)
         for RESOURCE in "${CLEANUP_RESOURCES[@]}"; do
-          items=$(kubectl get "$RESOURCE" --no-headers 2>/dev/null | awk "/$E2E_PREFIX/ {print \$1}")
-          if [[ -n "$items" ]]; then
-            echo "$items" | xargs -r kubectl delete "$RESOURCE"
-          else
-            echo "No $RESOURCE found with prefix $E2E_PREFIX"
-          fi
-        done
+            delete_resources_with_prefix "$RESOURCE"
+        done
\ No newline at end of file

From fef17e22ead4435c42c82d9c710d8b46e8aa4710 Mon Sep 17 00:00:00 2001
From: Maksim Fedotov <maksim.fedotov@flant.com>
Date: Thu, 10 Apr 2025 13:46:45 +0300
Subject: [PATCH 37/48] move checkout to mpo to Taskfile

Signed-off-by: Maksim Fedotov <maksim.fedotov@flant.com>
---
 .github/workflows/dev_module_build.yml | 48 +---------------------
 tests/e2e/Taskfile.yaml                | 57 ++++++++++++++++++++++++++
 2 files changed, 58 insertions(+), 47 deletions(-)

diff --git a/.github/workflows/dev_module_build.yml b/.github/workflows/dev_module_build.yml
index 550a90cd5d..531fc6a11e 100644
--- a/.github/workflows/dev_module_build.yml
+++ b/.github/workflows/dev_module_build.yml
@@ -491,53 +491,7 @@ jobs:
         env:
           v12n_tag: pr${{ github.event.pull_request.number }}
         run: |
-          d8 k patch mpo virtualization --type merge -p "{\"spec\":{\"imageTag\":\"$v12n_tag\"}}" 
-          images_hash=$(crane export "dev-registry.deckhouse.io/sys/deckhouse-oss/modules/virtualization:$v12n_tag" - | tar -Oxf - images_digests.json)
-          v12n_pods=$(kubectl -n d8-virtualization get pods -o json | jq -c)
-          retry_count=0
-          max_retries=120
-          sleep_interval=5 
-
-          while true; do
-            all_hashes_found=true
-
-            # Fetch current pods information
-            v12n_pods=$(kubectl -n d8-virtualization get pods -o json | jq -c)
-
-            # Process each image entry
-            while IFS= read -r image_entry; do
-              image=$(echo "$image_entry" | jq -r '.key')
-              hash=$(echo "$image_entry" | jq -r '.value')
-
-              if [[ "${image,,}" =~ (libguestfs|predeletehook) ]]; then
-                continue
-              fi
-
-              if echo "$v12n_pods" | grep -q "$hash"; then
-                echo "- ✅ $image $hash"
-              else
-                echo "- 🟥 $image $hash"
-                all_hashes_found=false
-              fi
-            done < <(echo "$images_hash" | jq -c '. | to_entries | sort_by(.key)[]')
-
-            # If all hashes are found, break the loop
-            if [ "$all_hashes_found" = true ]; then
-              echo "All image hashes found in pods."
-              break
-            fi 
-
-            retry_count=$((retry_count + 1))
-            echo "Some hashes are missing, rechecking... Attempt: $retry_count"
-
-            # Check if the retry limit has been reached
-            if [ "$retry_count" -ge "$max_retries" ]; then
-              echo "Error: Timeout reached after $((retry_count * sleep_interval)) seconds. Some image hashes are still missing."
-              exit 1
-            fi
-            # Wait for the specified interval before retrying
-            sleep "$sleep_interval"
-          done
+          task checkout-to-mpo
 
       - name: Download dependencies
         working-directory: ./tests/e2e/
diff --git a/tests/e2e/Taskfile.yaml b/tests/e2e/Taskfile.yaml
index a799e13875..3201ba759d 100644
--- a/tests/e2e/Taskfile.yaml
+++ b/tests/e2e/Taskfile.yaml
@@ -137,4 +137,61 @@ tasks:
         readarray -t CLEANUP_RESOURCES < <(yq '.cleanupResources[]' default_config.yaml)
         for RESOURCE in "${CLEANUP_RESOURCES[@]}"; do
             delete_resources_with_prefix "$RESOURCE"
+        done
+  
+  checkout-to-mpo:
+    deps:
+      - d8
+    cmds:
+      - |
+        if [ -z "$v12n_tag" ]; then
+          echo "Error: v12n_tag is not set."
+          exit 1
+        fi
+        d8 k patch mpo virtualization --type merge -p "{\"spec\":{\"imageTag\":\"$v12n_tag\"}}" 
+        images_hash=$(crane export "dev-registry.deckhouse.io/sys/deckhouse-oss/modules/virtualization:$v12n_tag" - | tar -Oxf - images_digests.json)
+        v12n_pods=$(kubectl -n d8-virtualization get pods -o json | jq -c)
+        retry_count=0
+        max_retries=120
+        sleep_interval=5 
+
+        while true; do
+          all_hashes_found=true
+
+          # Fetch current pods information
+          v12n_pods=$(kubectl -n d8-virtualization get pods -o json | jq -c)
+
+          # Process each image entry
+          while IFS= read -r image_entry; do
+            image=$(echo "$image_entry" | jq -r '.key')
+            hash=$(echo "$image_entry" | jq -r '.value')
+
+            if [[ "${image,,}" =~ (libguestfs|predeletehook) ]]; then
+              continue
+            fi
+
+            if echo "$v12n_pods" | grep -q "$hash"; then
+              echo "- ✅ $image $hash"
+            else
+              echo "- 🟥 $image $hash"
+              all_hashes_found=false
+            fi
+          done < <(echo "$images_hash" | jq -c '. | to_entries | sort_by(.key)[]')
+
+          # If all hashes are found, break the loop
+          if [ "$all_hashes_found" = true ]; then
+            echo "All image hashes found in pods."
+            break
+          fi 
+
+          retry_count=$((retry_count + 1))
+          echo "Some hashes are missing, rechecking... Attempt: $retry_count"
+
+          # Check if the retry limit has been reached
+          if [ "$retry_count" -ge "$max_retries" ]; then
+            echo "Error: Timeout reached after $((retry_count * sleep_interval)) seconds. Some image hashes are still missing."
+            exit 1
+          fi
+          # Wait for the specified interval before retrying
+          sleep "$sleep_interval"
         done
\ No newline at end of file

From ebebdcdc4a59d572172b302e2a1d475660064550 Mon Sep 17 00:00:00 2001
From: Maksim Fedotov <maksim.fedotov@flant.com>
Date: Thu, 10 Apr 2025 13:51:31 +0300
Subject: [PATCH 38/48] remove unnecessary continue

Signed-off-by: Maksim Fedotov <maksim.fedotov@flant.com>
---
 tests/e2e/tests_suite_test.go | 1 -
 1 file changed, 1 deletion(-)

diff --git a/tests/e2e/tests_suite_test.go b/tests/e2e/tests_suite_test.go
index 98360af7d5..70b43af3e3 100644
--- a/tests/e2e/tests_suite_test.go
+++ b/tests/e2e/tests_suite_test.go
@@ -221,7 +221,6 @@ func Cleanup() []error {
 			cleanupErrs = append(
 				cleanupErrs, fmt.Errorf("cmd: %s\nstderr: %s", res.GetCmd(), res.StdErr()),
 			)
-			continue
 		}
 	}
 

From 897cff07f5cd0cb0c6c2a43baf4804b8726e04de Mon Sep 17 00:00:00 2001
From: Maksim Fedotov <maksim.fedotov@flant.com>
Date: Thu, 10 Apr 2025 16:58:05 +0300
Subject: [PATCH 39/48] add label removal

Signed-off-by: Maksim Fedotov <maksim.fedotov@flant.com>
---
 .github/workflows/dev_module_build.yml | 24 +++++++++++++++++++++++-
 1 file changed, 23 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/dev_module_build.yml b/.github/workflows/dev_module_build.yml
index 531fc6a11e..1d9449199c 100644
--- a/.github/workflows/dev_module_build.yml
+++ b/.github/workflows/dev_module_build.yml
@@ -521,7 +521,7 @@ jobs:
       - name: Checkout sources
         uses: actions/checkout@v3.5.2
 
-      - name: Set commit status after e2e run
+      - name: Set commit status after e2e run and remove label
         id: set_e2e_requirement_status
         if: ${{ always() }}
         uses: actions/github-script@v6.4.1
@@ -534,3 +534,25 @@ jobs:
             const e2eStatus = require('./.github/scripts/js/e2e-commit-status');
 
             await e2eStatus.setStatusAfterE2eRun({github, context, core});
+            const issueNumber = context.issue.number;
+            const owner = context.repo.owner;
+            const repo = context.repo.repo;
+            const labelToRemove = 'e2e/run';
+
+            const { data: labels } = await github.rest.issues.listLabelsOnIssue({
+              owner,
+              repo,
+              issue_number: issueNumber,
+            });
+
+            if (labels.some(label => label.name === labelToRemove)) {
+              await github.rest.issues.removeLabel({
+                owner,
+                repo,
+                issue_number: issueNumber,
+                name: labelToRemove,
+              });
+              console.log(`Removed label '${labelToRemove}' from PR #${issueNumber}`);
+            } else {
+              console.log(`Label '${labelToRemove}' not found on PR #${issueNumber}`);
+            } 
\ No newline at end of file

From 66a1d0bb6c107e71a54eac334a4d74bdecbf0214 Mon Sep 17 00:00:00 2001
From: Maksim Fedotov <maksim.fedotov@flant.com>
Date: Thu, 10 Apr 2025 17:02:52 +0300
Subject: [PATCH 40/48] format yaml

Signed-off-by: Maksim Fedotov <maksim.fedotov@flant.com>
---
 tests/e2e/Taskfile.yaml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tests/e2e/Taskfile.yaml b/tests/e2e/Taskfile.yaml
index 3201ba759d..718874cb47 100644
--- a/tests/e2e/Taskfile.yaml
+++ b/tests/e2e/Taskfile.yaml
@@ -138,7 +138,7 @@ tasks:
         for RESOURCE in "${CLEANUP_RESOURCES[@]}"; do
             delete_resources_with_prefix "$RESOURCE"
         done
-  
+
   checkout-to-mpo:
     deps:
       - d8
@@ -194,4 +194,4 @@ tasks:
           fi
           # Wait for the specified interval before retrying
           sleep "$sleep_interval"
-        done
\ No newline at end of file
+        done

From 19f66d1cdf395cac9f0eaffe88ca9b3ac1d4827a Mon Sep 17 00:00:00 2001
From: Maksim Fedotov <maksim.fedotov@flant.com>
Date: Thu, 10 Apr 2025 17:03:03 +0300
Subject: [PATCH 41/48] format yaml 2

Signed-off-by: Maksim Fedotov <maksim.fedotov@flant.com>
---
 .github/workflows/dev_module_build.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/dev_module_build.yml b/.github/workflows/dev_module_build.yml
index 1d9449199c..70b84bb168 100644
--- a/.github/workflows/dev_module_build.yml
+++ b/.github/workflows/dev_module_build.yml
@@ -555,4 +555,4 @@ jobs:
               console.log(`Removed label '${labelToRemove}' from PR #${issueNumber}`);
             } else {
               console.log(`Label '${labelToRemove}' not found on PR #${issueNumber}`);
-            } 
\ No newline at end of file
+            }

From b69ea0ed23cbf9328642b56269b9b11f2b75a285 Mon Sep 17 00:00:00 2001
From: Maksim Fedotov <maksim.fedotov@flant.com>
Date: Thu, 10 Apr 2025 17:16:50 +0300
Subject: [PATCH 42/48] add deckhouse_ready_status check

Signed-off-by: Maksim Fedotov <maksim.fedotov@flant.com>
---
 tests/e2e/Taskfile.yaml | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/tests/e2e/Taskfile.yaml b/tests/e2e/Taskfile.yaml
index 718874cb47..c95334ff23 100644
--- a/tests/e2e/Taskfile.yaml
+++ b/tests/e2e/Taskfile.yaml
@@ -148,6 +148,11 @@ tasks:
           echo "Error: v12n_tag is not set."
           exit 1
         fi
+        DECKHOUSE_READY_STATUS=$(kubectl get po -n d8-system -l app=deckhouse -o json | jq -r '.items[0].status.conditions[] | select(.type=="ContainersReady") | .status')
+        if [ "$DECKHOUSE_READY_STATUS" != "True" ]; then
+          echo "Error: Deckhouse is not ready."
+          exit 1
+        fi
         d8 k patch mpo virtualization --type merge -p "{\"spec\":{\"imageTag\":\"$v12n_tag\"}}" 
         images_hash=$(crane export "dev-registry.deckhouse.io/sys/deckhouse-oss/modules/virtualization:$v12n_tag" - | tar -Oxf - images_digests.json)
         v12n_pods=$(kubectl -n d8-virtualization get pods -o json | jq -c)

From 89ce20599356cdbe7b36d6689104cf5d534f82b4 Mon Sep 17 00:00:00 2001
From: Maksim Fedotov <maksim.fedotov@flant.com>
Date: Thu, 10 Apr 2025 17:17:24 +0300
Subject: [PATCH 43/48] add working dir to checkout-to-mpo

Signed-off-by: Maksim Fedotov <maksim.fedotov@flant.com>
---
 .github/workflows/dev_module_build.yml | 1 +
 1 file changed, 1 insertion(+)

diff --git a/.github/workflows/dev_module_build.yml b/.github/workflows/dev_module_build.yml
index 70b84bb168..09c4e3a930 100644
--- a/.github/workflows/dev_module_build.yml
+++ b/.github/workflows/dev_module_build.yml
@@ -490,6 +490,7 @@ jobs:
       - name: Checkout cluster to revision
         env:
           v12n_tag: pr${{ github.event.pull_request.number }}
+        working-directory: ./tests/e2e/
         run: |
           task checkout-to-mpo
 

From 00e065192c75f7731ad3265cf71b25e48826c016 Mon Sep 17 00:00:00 2001
From: Maksim Fedotov <maksim.fedotov@flant.com>
Date: Thu, 10 Apr 2025 17:18:51 +0300
Subject: [PATCH 44/48] modify `update comment on finish` task

Signed-off-by: Maksim Fedotov <maksim.fedotov@flant.com>
---
 .github/workflows/dev_module_build.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/dev_module_build.yml b/.github/workflows/dev_module_build.yml
index 09c4e3a930..b51dc6180e 100644
--- a/.github/workflows/dev_module_build.yml
+++ b/.github/workflows/dev_module_build.yml
@@ -513,8 +513,9 @@ jobs:
         run: |
           task cleanup
 
-  last_comment:
+  update_comment_on_finish:
     name: Update comment on finish
+    if: ${{ always() }}
     needs:
       - run_e2e
     runs-on: ubuntu-latest
@@ -524,7 +525,6 @@ jobs:
 
       - name: Set commit status after e2e run and remove label
         id: set_e2e_requirement_status
-        if: ${{ always() }}
         uses: actions/github-script@v6.4.1
         env:
           JOB_STATUS: ${{ job.status }}

From dad836ec3a5aba60dabeb084be7c37673ae3911e Mon Sep 17 00:00:00 2001
From: Maksim Fedotov <maksim.fedotov@flant.com>
Date: Thu, 10 Apr 2025 17:31:08 +0300
Subject: [PATCH 45/48] move if back

Signed-off-by: Maksim Fedotov <maksim.fedotov@flant.com>
---
 .github/workflows/dev_module_build.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/dev_module_build.yml b/.github/workflows/dev_module_build.yml
index b51dc6180e..11655663b4 100644
--- a/.github/workflows/dev_module_build.yml
+++ b/.github/workflows/dev_module_build.yml
@@ -515,7 +515,6 @@ jobs:
 
   update_comment_on_finish:
     name: Update comment on finish
-    if: ${{ always() }}
     needs:
       - run_e2e
     runs-on: ubuntu-latest
@@ -524,6 +523,7 @@ jobs:
         uses: actions/checkout@v3.5.2
 
       - name: Set commit status after e2e run and remove label
+        if: ${{ always() }}
         id: set_e2e_requirement_status
         uses: actions/github-script@v6.4.1
         env:

From 89175174922968a60d8d530d24a0a337ac849535 Mon Sep 17 00:00:00 2001
From: Maksim Fedotov <maksim.fedotov@flant.com>
Date: Thu, 10 Apr 2025 22:42:20 +0300
Subject: [PATCH 46/48] remove label

Signed-off-by: Maksim Fedotov <maksim.fedotov@flant.com>
---
 .github/workflows/dev_module_build.yml | 30 ++++++++++++++++++++++++++
 1 file changed, 30 insertions(+)

diff --git a/.github/workflows/dev_module_build.yml b/.github/workflows/dev_module_build.yml
index 11655663b4..1fdaea9002 100644
--- a/.github/workflows/dev_module_build.yml
+++ b/.github/workflows/dev_module_build.yml
@@ -506,6 +506,36 @@ jobs:
         run: |
           task run -v
 
+      - name: Remove label
+        if: ${{ always() }}
+        id: remove-label
+        uses: actions/github-script@v6
+        with:
+          github-token: ${{secrets.RELEASE_PLEASE_TOKEN}}
+          script: |
+            const issueNumber = context.issue.number;
+            const owner = context.repo.owner;
+            const repo = context.repo.repo;
+            const labelToRemove = 'e2e/run';
+
+            const { data: labels } = await github.rest.issues.listLabelsOnIssue({
+              owner,
+              repo,
+              issue_number: issueNumber,
+            });
+
+            if (labels.some(label => label.name === labelToRemove)) {
+              await github.rest.issues.removeLabel({
+                owner,
+                repo,
+                issue_number: issueNumber,
+                name: labelToRemove,
+              });
+              console.log(`Removed label '${labelToRemove}' from PR #${issueNumber}`);
+            } else {
+              console.log(`Label '${labelToRemove}' not found on PR #${issueNumber}`);
+            }
+
       - name: Cleanup E2E resources on cancel
         if: always() && steps.e2e-tests.outcome == 'cancelled'
         id: e2e-tests-cleanup

From 271c1b09dee5e3b4cdba251d6db2ab8806a183a6 Mon Sep 17 00:00:00 2001
From: Maksim Fedotov <maksim.fedotov@flant.com>
Date: Thu, 10 Apr 2025 22:42:58 +0300
Subject: [PATCH 47/48] remove unnecessary code

Signed-off-by: Maksim Fedotov <maksim.fedotov@flant.com>
---
 .github/workflows/dev_module_build.yml | 22 ----------------------
 1 file changed, 22 deletions(-)

diff --git a/.github/workflows/dev_module_build.yml b/.github/workflows/dev_module_build.yml
index 1fdaea9002..b64f72ef38 100644
--- a/.github/workflows/dev_module_build.yml
+++ b/.github/workflows/dev_module_build.yml
@@ -565,25 +565,3 @@ jobs:
             const e2eStatus = require('./.github/scripts/js/e2e-commit-status');
 
             await e2eStatus.setStatusAfterE2eRun({github, context, core});
-            const issueNumber = context.issue.number;
-            const owner = context.repo.owner;
-            const repo = context.repo.repo;
-            const labelToRemove = 'e2e/run';
-
-            const { data: labels } = await github.rest.issues.listLabelsOnIssue({
-              owner,
-              repo,
-              issue_number: issueNumber,
-            });
-
-            if (labels.some(label => label.name === labelToRemove)) {
-              await github.rest.issues.removeLabel({
-                owner,
-                repo,
-                issue_number: issueNumber,
-                name: labelToRemove,
-              });
-              console.log(`Removed label '${labelToRemove}' from PR #${issueNumber}`);
-            } else {
-              console.log(`Label '${labelToRemove}' not found on PR #${issueNumber}`);
-            }

From bfc1b0ac6a6e8b79e0758659fd8bf982746d21c6 Mon Sep 17 00:00:00 2001
From: Maksim Fedotov <maksim.fedotov@flant.com>
Date: Thu, 10 Apr 2025 22:49:40 +0300
Subject: [PATCH 48/48] remove run on unlabeled PR

Signed-off-by: Maksim Fedotov <maksim.fedotov@flant.com>
---
 .github/workflows/dev_module_build.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/dev_module_build.yml b/.github/workflows/dev_module_build.yml
index b64f72ef38..175e5ee699 100644
--- a/.github/workflows/dev_module_build.yml
+++ b/.github/workflows/dev_module_build.yml
@@ -35,7 +35,7 @@ on:
         required: false
         type: number
   pull_request:
-    types: [opened, reopened, synchronize, labeled, unlabeled]
+    types: [opened, reopened, synchronize, labeled]
   push:
     branches:
       - main