diff --git a/.github/scripts/e2e-prepare-env.sh b/.github/scripts/e2e-prepare-env.sh new file mode 100755 index 0000000..0e49dd3 --- /dev/null +++ b/.github/scripts/e2e-prepare-env.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash +# Writes non-secret env to $RUNNER_TEMP/e2e-env.sh and materializes secrets into temp files. +# Never echoes secret values. + +set -euo pipefail + +ENV_FILE="${RUNNER_TEMP}/e2e-env.sh" +: >"$ENV_FILE" + +write_env() { + printf 'export %s=%q\n' "$1" "$2" >>"$ENV_FILE" +} + +if [[ -n "${E2E_SSH_PRIVATE_KEY:-}" ]]; then + KEY_FILE="${RUNNER_TEMP}/e2e_ssh_key" + printf '%s\n' "$E2E_SSH_PRIVATE_KEY" >"$KEY_FILE" + chmod 600 "$KEY_FILE" + write_env SSH_PRIVATE_KEY "$KEY_FILE" +fi + +if [[ -n "${E2E_SSH_PUBLIC_KEY:-}" ]]; then + PUB_FILE="${RUNNER_TEMP}/e2e_ssh_pub" + printf '%s\n' "$E2E_SSH_PUBLIC_KEY" >"$PUB_FILE" + chmod 644 "$PUB_FILE" + write_env SSH_PUBLIC_KEY "$PUB_FILE" +fi + +if [[ -n "${E2E_CLUSTER_KUBECONFIG:-}" ]]; then + KC_FILE="${RUNNER_TEMP}/e2e_kubeconfig" + printf '%s' "$E2E_CLUSTER_KUBECONFIG" | base64 -d >"$KC_FILE" + chmod 600 "$KC_FILE" + write_env KUBE_CONFIG_PATH "$KC_FILE" + write_env E2E_BASE_KUBE_CONFIG_PATH "$KC_FILE" +fi + +if [[ -n "${SSH_HOST:-}" ]]; then + write_env SSH_HOST "$SSH_HOST" + write_env E2E_BASE_SSH_HOST "$SSH_HOST" +fi +if [[ -n "${SSH_USER:-}" ]]; then + write_env SSH_USER "$SSH_USER" + write_env E2E_BASE_SSH_USER "$SSH_USER" +fi +if [[ -n "${SSH_VM_USER:-}" ]]; then + write_env SSH_VM_USER "$SSH_VM_USER" +fi +JUMP_HOST="${SSH_JUMP_HOST:-${E2E_TUNNEL_SSH_JUMP_HOST:-}}" +JUMP_USER="${SSH_JUMP_USER:-${E2E_TUNNEL_SSH_JUMP_USER:-}}" +if [[ -n "${JUMP_HOST}" ]]; then + write_env SSH_JUMP_HOST "$JUMP_HOST" + write_env E2E_TUNNEL_SSH_JUMP_HOST "$JUMP_HOST" +fi +if [[ -n "${JUMP_USER}" ]]; then + write_env SSH_JUMP_USER "$JUMP_USER" + write_env E2E_TUNNEL_SSH_JUMP_USER "$JUMP_USER" +fi +if [[ -n "${LOG_LEVEL:-}" ]]; then + write_env LOG_LEVEL "$LOG_LEVEL" +fi +if [[ -n "${E2E_GINKGO_LABEL_FILTER:-}" ]]; then + write_env E2E_GINKGO_LABEL_FILTER "$E2E_GINKGO_LABEL_FILTER" +fi +if [[ -n "${E2E_TEST_TIMEOUT:-}" ]]; then + write_env E2E_TEST_TIMEOUT "$E2E_TEST_TIMEOUT" +fi +if [[ -n "${TEST_CLUSTER_STORAGE_CLASS:-}" ]]; then + write_env TEST_CLUSTER_STORAGE_CLASS "$TEST_CLUSTER_STORAGE_CLASS" +fi +if [[ -n "${TEST_CLUSTER_NAMESPACE:-}" ]]; then + write_env TEST_CLUSTER_NAMESPACE "$TEST_CLUSTER_NAMESPACE" +fi +if [[ -n "${TEST_CLUSTER_CREATE_MODE:-}" ]]; then + write_env TEST_CLUSTER_CREATE_MODE "$TEST_CLUSTER_CREATE_MODE" +fi + +write_env GOMODCACHE "${GOMODCACHE:-${RUNNER_TEMP}/e2e-gomodcache}" +write_env GOCACHE "${GOCACHE:-${RUNNER_TEMP}/e2e-gocache}" +write_env E2E_ARTIFACT_DIR "${E2E_ARTIFACT_DIR:-${RUNNER_TEMP}/e2e-artifacts}" +write_env E2E_TEMP_DIR "${E2E_TEMP_DIR:-${E2E_ARTIFACT_DIR:-${RUNNER_TEMP}/e2e-artifacts}}" + +echo "e2e-env.sh prepared (secrets written to temp files only)" diff --git a/.github/scripts/e2e-prepare-workspace.sh b/.github/scripts/e2e-prepare-workspace.sh new file mode 100755 index 0000000..f0510b3 --- /dev/null +++ b/.github/scripts/e2e-prepare-workspace.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# Self-hosted runners: E2E may leave read-only Go cache trees in the workspace; +# actions/checkout@v4 then fails with EACCES when wiping the directory. + +set -euo pipefail + +WS="${GITHUB_WORKSPACE:?GITHUB_WORKSPACE is not set}" + +prune_dir() { + local p="$1" + [ -e "$p" ] || return 0 + chmod -R u+w "$p" 2>/dev/null || true + if rm -rf "$p" 2>/dev/null; then + return 0 + fi + if command -v sudo >/dev/null 2>&1; then + sudo chmod -R u+w "$p" 2>/dev/null || true + sudo rm -rf "$p" 2>/dev/null || true + fi +} + +for d in \ + .e2e-gomodcache .e2e-gocache .e2e-artifacts \ + e2e/.gomodcache e2e/.gocache e2e/temp; do + prune_dir "${WS}/${d}" +done diff --git a/.github/workflows/e2e-reusable.yml b/.github/workflows/e2e-reusable.yml new file mode 100644 index 0000000..af045a2 --- /dev/null +++ b/.github/workflows/e2e-reusable.yml @@ -0,0 +1,433 @@ +# Reusable E2E pipeline. +# +# pipeline_mode: +# create-and-test — mocked create-cluster + run-tests (full e2e flow from module side) +# teardown-only — mocked teardown-cluster +# noop — all jobs echo "mocked"; used for CI self-testing in storage-e2e itself + +name: Storage E2E (reusable) + +permissions: + contents: read + checks: write + pull-requests: read + +on: + workflow_call: + inputs: + pipeline_mode: + description: "create-and-test | teardown-only | noop" + type: string + required: true + pr_number: + description: "Pull request number (stable session/namespace key)" + type: string + required: true + module_slug: + description: "Module slug for TEST_CLUSTER_NAMESPACE (e.g. sds-node-configurator)" + type: string + required: true + module_path: + description: "Path to the module e2e Go module root" + type: string + required: true + cluster_provider: + description: "Cluster provider: alwaysCreateNew | alwaysUseExisting | commander" + type: string + required: true + cluster_config: + description: "Path to cluster_config.yml relative to repository root" + type: string + required: false + default: "" + test_package: + description: "Go package for run-tests" + type: string + required: false + default: "./tests/" + label_filter: + description: "Ginkgo label filter" + type: string + required: false + default: "!stress-test" + test_timeout: + description: "go test / ginkgo timeout (E2E_TEST_TIMEOUT)" + type: string + required: false + default: "3h30m" + test_suite: + description: "Go test function name to run (passed to -run). Default matches sds-node-configurator." + type: string + required: false + default: "TestSdsNodeConfigurator" + skip_storage_e2e_replace: + description: "Skip checkout + go mod replace of storage-e2e. Set true when the caller IS storage-e2e (avoids self-referencing replace)." + type: boolean + required: false + default: false + storage_e2e_ref: + description: "Git ref of storage-e2e for checkout" + type: string + required: false + default: "main" + runner_labels: + description: "JSON array of runner labels" + type: string + required: false + default: '["self-hosted","regular"]' + skip_create_cluster: + description: "Deprecated — ignored; create-cluster always runs and uses --skip-if-ready" + type: boolean + required: false + default: false + outputs: + run_tests_result: + description: "Result of run-tests job" + value: ${{ jobs.run-tests.result }} + secrets: + E2E_SSH_PRIVATE_KEY: + required: false + E2E_SSH_PUBLIC_KEY: + required: false + E2E_SSH_HOST: + required: false + E2E_SSH_USER: + required: false + E2E_SSH_JUMP_HOST: + required: false + E2E_SSH_JUMP_USER: + required: false + E2E_CLUSTER_KUBECONFIG: + required: false + E2E_TEST_CLUSTER_CREATE_MODE: + required: false + E2E_TEST_CLUSTER_STORAGE_CLASS: + required: false + E2E_TEST_CLUSTER_CLEANUP: + required: false + E2E_DECKHOUSE_LICENSE: + required: false + E2E_REGISTRY_DOCKER_CFG: + required: false + SSH_VM_USER: + required: false + GOPROXY: + required: false + +defaults: + run: + shell: bash + +env: + E2E_PR_NUMBER: ${{ inputs.pr_number }} + # Stable PR-scoped session cache (one entry per PR; concurrency group prevents parallel E2E on same PR). + E2E_CACHE_KEY: e2e-pr-${{ inputs.pr_number }}-session + # Stable PR namespace (no run_id) — required for cluster resume across re-runs. + TEST_CLUSTER_NAMESPACE: e2e-${{ inputs.module_slug }}-pr${{ inputs.pr_number }} + TEST_CLUSTER_CREATE_MODE: ${{ secrets.E2E_TEST_CLUSTER_PROVIDER || inputs.cluster_provider }} + TEST_CLUSTER_STORAGE_CLASS: ${{ secrets.E2E_TEST_CLUSTER_STORAGE_CLASS }} + DKP_LICENSE_KEY: ${{ secrets.E2E_DECKHOUSE_LICENSE }} + REGISTRY_DOCKER_CFG: ${{ secrets.E2E_REGISTRY_DOCKER_CFG }} + SSH_HOST: ${{ secrets.E2E_DVP_BASE_CLUSTER_SSH_HOST }} + SSH_USER: ${{ secrets.E2E_DVP_BASE_CLUSTER_SSH_USER }} + SSH_VM_USER: ${{ secrets.E2E_SSH_VM_USER }} + # Jump host for test cluster nodes (10.10.10.x). Without it the runner connects directly → timeout. + SSH_JUMP_HOST: ${{ secrets.E2E_DVP_BASE_CLUSTER_SSH_JUMP_HOST }} + SSH_JUMP_USER: ${{ secrets.E2E_DVP_BASE_CLUSTER_SSH_JUMP_USER }} + # API SSH tunnel: ProxyJump only when jump secret is set (do not fall back to SSH_HOST). + E2E_TUNNEL_SSH_JUMP_HOST: ${{ secrets.E2E_DVP_BASE_CLUSTER_SSH_JUMP_HOST }} + E2E_TUNNEL_SSH_JUMP_USER: ${{ secrets.E2E_DVP_BASE_CLUSTER_SSH_JUMP_USER }} + LOG_LEVEL: ${{ vars.E2E_LOG_LEVEL || 'info' }} + E2E_GINKGO_LABEL_FILTER: ${{ inputs.label_filter }} + E2E_LABEL_FILTER: ${{ inputs.label_filter }} + E2E_TEST_TIMEOUT: ${{ inputs.test_timeout }} + # Do not set KUBE_CONFIG_PATH here — e2e-prepare-env.sh writes it from E2E_CLUSTER_KUBECONFIG. + +jobs: + create-cluster: + if: inputs.pipeline_mode == 'create-and-test' || inputs.pipeline_mode == 'noop' + name: create-cluster + runs-on: ${{ fromJSON(inputs.runner_labels) }} + steps: + - name: create-cluster (mocked no-op) + run: | + echo "create-cluster is mocked in this pipeline revision" + echo "Pipeline mode: ${{ inputs.pipeline_mode }}" + echo "PR: ${{ inputs.pr_number }}" + + run-tests: + if: (inputs.pipeline_mode == 'create-and-test' && needs.create-cluster.result == 'success') || inputs.pipeline_mode == 'noop' + name: run-tests + needs: create-cluster + timeout-minutes: 240 + runs-on: ${{ fromJSON(inputs.runner_labels) }} + env: + TEST_CLUSTER_CLEANUP: "false" + CI: "true" + E2E_GINKGO_LABEL_FILTER: ${{ inputs.label_filter }} + # Keep stress tests disabled by default unless caller overrides label_filter. + E2E_TEST_TIMEOUT: ${{ inputs.test_timeout }} + permissions: + checks: write + contents: read + steps: + - name: run-tests (noop) + if: inputs.pipeline_mode == 'noop' + run: | + echo "run-tests is mocked (pipeline_mode: noop)" + echo "PR: ${{ inputs.pr_number }}, module: ${{ inputs.module_slug }}" + + - name: Prepare workspace for checkout (self-hosted) + if: inputs.pipeline_mode != 'noop' + run: | + set -euo pipefail + WS="${GITHUB_WORKSPACE}" + prune_dir() { + local p="$1" + [ -e "$p" ] || return 0 + chmod -R u+w "$p" 2>/dev/null || true + rm -rf "$p" 2>/dev/null || { command -v sudo >/dev/null && sudo chmod -R u+w "$p" && sudo rm -rf "$p"; } || true + } + for d in .e2e-gomodcache .e2e-gocache .e2e-artifacts e2e/.gomodcache e2e/.gocache e2e/temp; do + prune_dir "${WS}/${d}" + done + + - name: Checkout repository + if: inputs.pipeline_mode != 'noop' + uses: actions/checkout@v4 + with: + clean: false + + - name: Checkout storage-e2e + if: inputs.pipeline_mode != 'noop' && !inputs.skip_storage_e2e_replace + uses: actions/checkout@v4 + with: + repository: deckhouse/storage-e2e + ref: ${{ inputs.storage_e2e_ref }} + path: storage-e2e + clean: false + + - name: Setup Go + if: inputs.pipeline_mode != 'noop' && !inputs.skip_storage_e2e_replace + uses: actions/setup-go@v5 + with: + go-version-file: ${{ inputs.module_path }}/go.mod + cache-dependency-path: | + ${{ inputs.module_path }}/go.sum + storage-e2e/go.sum + + - name: Setup Go (self-hosted storage-e2e) + if: inputs.pipeline_mode != 'noop' && inputs.skip_storage_e2e_replace + uses: actions/setup-go@v5 + with: + go-version-file: ${{ inputs.module_path }}/go.mod + cache-dependency-path: ${{ inputs.module_path }}/go.sum + + - name: Setup test environment + if: inputs.pipeline_mode != 'noop' + run: | + E2E_SSH_KEY_PATH=$(mktemp /tmp/e2e_ssh_key.XXXXXX) + E2E_SSH_PUB_PATH=$(mktemp /tmp/e2e_ssh_pub.XXXXXX) + E2E_KUBECONFIG_PATH=$(mktemp /tmp/e2e_kubeconfig.XXXXXX) + echo "E2E_SSH_KEY_PATH=${E2E_SSH_KEY_PATH}" >> "$GITHUB_ENV" + echo "E2E_SSH_PUB_PATH=${E2E_SSH_PUB_PATH}" >> "$GITHUB_ENV" + echo "E2E_KUBECONFIG_PATH=${E2E_KUBECONFIG_PATH}" >> "$GITHUB_ENV" + + printf '%s\n' "$E2E_SSH_PRIVATE_KEY" > "${E2E_SSH_KEY_PATH}" + chmod 600 "${E2E_SSH_KEY_PATH}" + echo "SSH_PRIVATE_KEY=${E2E_SSH_KEY_PATH}" >> "$GITHUB_ENV" + + if [ -n "${E2E_SSH_PUBLIC_KEY:-}" ]; then + printf '%s\n' "$E2E_SSH_PUBLIC_KEY" > "${E2E_SSH_PUB_PATH}" + chmod 644 "${E2E_SSH_PUB_PATH}" + echo "SSH_PUBLIC_KEY=${E2E_SSH_PUB_PATH}" >> "$GITHUB_ENV" + fi + + if [ -n "${E2E_CLUSTER_KUBECONFIG:-}" ]; then + printf '%s' "$E2E_CLUSTER_KUBECONFIG" | base64 -d > "${E2E_KUBECONFIG_PATH}" + chmod 600 "${E2E_KUBECONFIG_PATH}" + echo "KUBE_CONFIG_PATH=${E2E_KUBECONFIG_PATH}" >> "$GITHUB_ENV" + fi + + TEST_NAMESPACE="e2e-${{ inputs.module_slug }}-pr${{ inputs.pr_number }}-${{ github.run_id }}" + echo "TEST_NAMESPACE=${TEST_NAMESPACE}" >> "$GITHUB_ENV" + echo "Namespace: ${TEST_NAMESPACE}" + env: + E2E_SSH_PRIVATE_KEY: ${{ secrets.E2E_DVP_BASE_CLUSTER_SSH_PRIVATE_KEY }} + E2E_SSH_PUBLIC_KEY: ${{ secrets.E2E_DVP_BASE_CLUSTER_SSH_PUBLIC_KEY }} + E2E_CLUSTER_KUBECONFIG: ${{ secrets.E2E_DVP_BASE_CLUSTER_KUBECONFIG }} + + - name: Run E2E tests (build_dev-compatible flow) + if: inputs.pipeline_mode != 'noop' + id: run_tests + working-directory: ${{ inputs.module_path }} + env: + GOMODCACHE: ${{ runner.temp }}/e2e-gomodcache + GOCACHE: ${{ runner.temp }}/e2e-gocache + GOPROXY: ${{ secrets.GOPROXY }} + TEST_CLUSTER_CREATE_MODE: ${{ secrets.E2E_TEST_CLUSTER_CREATE_MODE || inputs.cluster_provider }} + TEST_CLUSTER_NAMESPACE: e2e-${{ inputs.module_slug }}-pr${{ inputs.pr_number }}-${{ github.run_id }} + TEST_CLUSTER_STORAGE_CLASS: ${{ secrets.E2E_TEST_CLUSTER_STORAGE_CLASS }} + TEST_CLUSTER_CLEANUP: ${{ secrets.E2E_TEST_CLUSTER_CLEANUP || 'false' }} + DKP_LICENSE_KEY: ${{ secrets.E2E_DECKHOUSE_LICENSE }} + REGISTRY_DOCKER_CFG: ${{ secrets.E2E_REGISTRY_DOCKER_CFG }} + SSH_HOST: ${{ secrets.E2E_SSH_HOST }} + SSH_USER: ${{ secrets.E2E_SSH_USER }} + SSH_JUMP_HOST: ${{ secrets.E2E_SSH_JUMP_HOST }} + SSH_JUMP_USER: ${{ secrets.E2E_SSH_JUMP_USER }} + SSH_VM_USER: ${{ secrets.SSH_VM_USER }} + E2E_TUNNEL_SSH_JUMP_HOST: ${{ secrets.E2E_SSH_JUMP_HOST }} + E2E_TUNNEL_SSH_JUMP_USER: ${{ secrets.E2E_SSH_JUMP_USER }} + LOG_LEVEL: ${{ vars.E2E_LOG_LEVEL || 'info' }} + run: | + set -euo pipefail + mkdir -p "${GOMODCACHE}" "${GOCACHE}" + if [ "${{ inputs.skip_storage_e2e_replace }}" != "true" ]; then + go mod edit -replace=github.com/deckhouse/storage-e2e=${{ github.workspace }}/storage-e2e + fi + + E2E_SSH_TUNNEL_PID="" + e2e_stop_ssh_tunnel() { + if [ -n "${E2E_SSH_TUNNEL_PID:-}" ]; then + kill "${E2E_SSH_TUNNEL_PID}" 2>/dev/null || true + wait "${E2E_SSH_TUNNEL_PID}" 2>/dev/null || true + fi + } + trap e2e_stop_ssh_tunnel EXIT + + E2E_KC_FILE="${E2E_KUBECONFIG_PATH:-${KUBE_CONFIG_PATH:-}}" + E2E_NEED_TUNNEL=false + E2E_LOCAL_API_PORT="" + SERVER_LINE="" + if [ -n "${E2E_KC_FILE}" ] && [ -f "${E2E_KC_FILE}" ]; then + SERVER_LINE="$(grep -F 'server:' "${E2E_KC_FILE}" | grep -E '127\.0\.0\.1|localhost|\[::1\]' | head -1 || true)" + if [ -n "${SERVER_LINE}" ]; then + E2E_NEED_TUNNEL=true + E2E_LOCAL_API_PORT="$(printf '%s' "${SERVER_LINE}" | sed -n 's/.*127\.0\.0\.1:\([0-9][0-9]*\).*/\1/p')" + if [ -z "${E2E_LOCAL_API_PORT}" ]; then + E2E_LOCAL_API_PORT="$(printf '%s' "${SERVER_LINE}" | sed -n 's/.*localhost:\([0-9][0-9]*\).*/\1/p')" + fi + if [ -z "${E2E_LOCAL_API_PORT}" ]; then + E2E_LOCAL_API_PORT="$(printf '%s' "${SERVER_LINE}" | sed -n 's/.*\[::1\]:\([0-9][0-9]*\).*/\1/p')" + fi + [ -z "${E2E_LOCAL_API_PORT}" ] && E2E_LOCAL_API_PORT=6445 + fi + fi + + if [ "${E2E_NEED_TUNNEL}" = true ] && [ -n "${SSH_HOST:-}" ] && [ -n "${SSH_USER:-}" ] && [ -f "${E2E_SSH_KEY_PATH:-}" ]; then + SSH_OPTS="-N -o StrictHostKeyChecking=no -o ConnectTimeout=15 -o ExitOnForwardFailure=yes -o ServerAliveInterval=30 -o ServerAliveCountMax=3" + TUNNEL_JUMP_USER="${E2E_TUNNEL_SSH_JUMP_USER:-$SSH_USER}" + if [ -n "${E2E_TUNNEL_SSH_JUMP_HOST:-}" ]; then + ssh ${SSH_OPTS} -i "${E2E_SSH_KEY_PATH}" \ + -J "${TUNNEL_JUMP_USER}@${E2E_TUNNEL_SSH_JUMP_HOST}" \ + -L "${E2E_LOCAL_API_PORT}:127.0.0.1:6445" "${SSH_USER}@${SSH_HOST}" & + else + ssh ${SSH_OPTS} -i "${E2E_SSH_KEY_PATH}" \ + -L "${E2E_LOCAL_API_PORT}:127.0.0.1:6445" "${SSH_USER}@${SSH_HOST}" & + fi + E2E_SSH_TUNNEL_PID=$! + sleep 3 + elif [ "${E2E_NEED_TUNNEL}" = true ]; then + echo "::error::Kubeconfig uses loopback API but SSH tunnel params are missing" + exit 1 + fi + + go mod download + go mod tidy + + if [ -z "${E2E_GINKGO_LABEL_FILTER:-}" ]; then + export E2E_GINKGO_LABEL_FILTER="!stress-test" + fi + + if [ -n "${GOMODCACHE:-}" ] && [ -d "${GOMODCACHE}/github.com/deckhouse" ]; then + shopt -s nullglob + for d in "${GOMODCACHE}"/github.com/deckhouse/storage-e2e@*; do + if [ -d "${d}" ]; then + chmod -R u+w "${d}" || true + mkdir -p "${d}/temp/cluster" || true + chmod -R u+w "${d}/temp" 2>/dev/null || true + fi + done + shopt -u nullglob + fi + + RAW_E2E_TEST_TIMEOUT="${E2E_TEST_TIMEOUT:-3h30m}" + if [[ "${RAW_E2E_TEST_TIMEOUT}" =~ ^([0-9]+h)?([0-9]+m)?([0-9]+s)?$ ]]; then + H="${BASH_REMATCH[1]%h}" + M="${BASH_REMATCH[2]%m}" + S="${BASH_REMATCH[3]%s}" + H="${H:-0}" + M="${M:-0}" + S="${S:-0}" + TOTAL_SECONDS=$((10#${H} * 3600 + 10#${M} * 60 + 10#${S})) + if [ "${TOTAL_SECONDS}" -lt 5400 ]; then + export E2E_TEST_TIMEOUT="90m" + else + export E2E_TEST_TIMEOUT="${RAW_E2E_TEST_TIMEOUT}" + fi + else + export E2E_TEST_TIMEOUT="90m" + fi + E2E_GO_TEST_TIMEOUT="3h30m" + echo "Ginkgo label filter: ${E2E_GINKGO_LABEL_FILTER}" + echo "go test -timeout: ${E2E_GO_TEST_TIMEOUT} (Ginkgo suite: ${E2E_TEST_TIMEOUT})" + go test -v -count=1 -timeout "${E2E_GO_TEST_TIMEOUT}" "${{ inputs.test_package }}" -run '^${{ inputs.test_suite }}$' \ + -ginkgo.label-filter="${E2E_GINKGO_LABEL_FILTER}" 2>&1 | tee "${GITHUB_WORKSPACE}/e2e-test-output.log" + TEST_EXIT_CODE=${PIPESTATUS[0]} + echo "${TEST_EXIT_CODE}" > "${GITHUB_WORKSPACE}/e2e-test-exit-code.txt" + exit "${TEST_EXIT_CODE}" + + - name: Upload E2E logs + uses: actions/upload-artifact@v4 + if: always() && inputs.pipeline_mode != 'noop' + with: + name: e2e-test-logs-${{ inputs.module_slug }}-${{ github.run_id }} + path: ${{ github.workspace }}/e2e-test-output.log + retention-days: 7 + if-no-files-found: ignore + + - name: Cleanup test resources + if: always() && inputs.pipeline_mode != 'noop' + env: + SSH_HOST: ${{ secrets.E2E_SSH_HOST }} + SSH_USER: ${{ secrets.E2E_SSH_USER }} + SSH_JUMP_HOST: ${{ secrets.E2E_SSH_JUMP_HOST }} + SSH_JUMP_USER: ${{ secrets.E2E_SSH_JUMP_USER }} + run: | + SSH_TUNNEL_PID="" + if [ -z "${E2E_KUBECONFIG_PATH:-}" ] || [ ! -f "${E2E_KUBECONFIG_PATH}" ]; then + echo "No kubeconfig, skip cleanup" + else + if [ -n "${SSH_HOST}" ] && [ -n "${SSH_USER}" ] && [ -f "${E2E_SSH_KEY_PATH}" ]; then + if [ -n "${SSH_JUMP_HOST}" ]; then + JUMP_USER="${SSH_JUMP_USER:-$SSH_USER}" + ssh -N -o StrictHostKeyChecking=no -o ConnectTimeout=15 -i "${E2E_SSH_KEY_PATH}" -J "${JUMP_USER}@${SSH_JUMP_HOST}" -L 6445:127.0.0.1:6445 "${SSH_USER}@${SSH_HOST}" & + else + ssh -N -o StrictHostKeyChecking=no -o ConnectTimeout=15 -i "${E2E_SSH_KEY_PATH}" -L 6445:127.0.0.1:6445 "${SSH_USER}@${SSH_HOST}" & + fi + SSH_TUNNEL_PID=$! + sleep 2 + fi + export KUBECONFIG="${E2E_KUBECONFIG_PATH}" + kubectl delete namespace "${TEST_NAMESPACE}" --ignore-not-found=true --timeout=5m 2>/dev/null || true + kubectl delete vd -n "${TEST_NAMESPACE}" --all --ignore-not-found=true --timeout=2m 2>/dev/null || true + kubectl delete vm -n "${TEST_NAMESPACE}" --all --ignore-not-found=true --timeout=2m 2>/dev/null || true + fi + if [ -n "${SSH_TUNNEL_PID}" ]; then + kill "${SSH_TUNNEL_PID}" 2>/dev/null || true + fi + + - name: Cleanup temp credentials + if: always() && inputs.pipeline_mode != 'noop' + run: rm -f "${E2E_SSH_KEY_PATH}" "${E2E_SSH_PUB_PATH}" "${E2E_KUBECONFIG_PATH}" 2>/dev/null || true + + teardown-cluster: + if: inputs.pipeline_mode == 'teardown-only' + name: teardown-cluster + runs-on: ${{ fromJSON(inputs.runner_labels) }} + steps: + - name: teardown-cluster (mocked no-op) + run: | + echo "teardown-cluster is mocked in this pipeline revision" + echo "Pipeline mode: ${{ inputs.pipeline_mode }}" + echo "PR: ${{ inputs.pr_number }}" diff --git a/.github/workflows/e2e-self-test.yml b/.github/workflows/e2e-self-test.yml new file mode 100644 index 0000000..6840a0a --- /dev/null +++ b/.github/workflows/e2e-self-test.yml @@ -0,0 +1,33 @@ +name: E2E self-test + +# Smoke-tests the reusable E2E workflow (noop mode — no cluster, no tests). +# Runs on PRs that touch CI workflow files or go.mod/go.sum. + +on: + pull_request: + paths: + - '.github/workflows/e2e-reusable.yml' + - '.github/scripts/**' + - 'go.mod' + - 'go.sum' + workflow_dispatch: + +concurrency: + group: e2e-self-test-${{ github.ref }} + cancel-in-progress: true + +jobs: + e2e: + uses: ./.github/workflows/e2e-reusable.yml + with: + pipeline_mode: noop + pr_number: ${{ github.event.pull_request.number || '0' }} + module_slug: storage-e2e + module_path: . + cluster_provider: alwaysCreateNew + cluster_config: e2e/tests/cluster_config.yaml + test_package: ./tests/test-template/ + test_suite: TestTemplate + skip_storage_e2e_replace: true + runner_labels: '["self-hosted","regular"]' + secrets: inherit diff --git a/README.md b/README.md index 74ee795..c6b23ff 100644 --- a/README.md +++ b/README.md @@ -4,6 +4,10 @@ End-to-end tests for Deckhouse storage components. +## CI (reusable workflow) + +See [docs/CI.md](docs/CI.md) for the reusable E2E pipeline (`create-cluster` → `run-tests` → `teardown-cluster`) and how to call it from a module repo. + ## Quick Start 1. Create test with script: `cd tests && ./create-test.sh ` diff --git a/docs/CI.md b/docs/CI.md new file mode 100644 index 0000000..2951c0a --- /dev/null +++ b/docs/CI.md @@ -0,0 +1,69 @@ +# Reusable CI pipeline (storage-e2e) + +Three-job workflow with **mocked** `create-cluster` / `teardown-cluster` and a full `run-tests` that mirrors the `build_dev` smoke flow. + +## Jobs + +| Job | Condition | What it does | +|-----|-----------|--------------| +| `create-cluster` | `pipeline_mode == 'create-and-test'` | No-op placeholder (mocked) | +| `run-tests` | after `create-cluster` succeeds | Sets up SSH tunnel + kubeconfig, runs `go test` directly in the module repo | +| `teardown-cluster` | `pipeline_mode == 'teardown-only'` | No-op placeholder (mocked) | + +Cluster lifecycle is handled inside the module's test suite via `pkg/cluster.CreateOrConnectToTestCluster`. + +## PR-scoped namespace + +`TEST_CLUSTER_NAMESPACE = e2e--pr-` — unique per run, set both in `env:` and forwarded to `go test`. + +## How to call from a module repo + +```yaml +jobs: + e2e: + uses: deckhouse/storage-e2e/.github/workflows/e2e-reusable.yml@ + secrets: inherit + with: + pipeline_mode: create-and-test # or teardown-only + pr_number: "123" + module_slug: sds-node-configurator + module_path: e2e + cluster_provider: alwaysCreateNew + cluster_config: e2e/tests/cluster_config.yml + test_package: ./tests/ + label_filter: "" # empty → !stress-test (all non-stress specs) + test_timeout: 90m + storage_e2e_ref: main # storage-e2e branch/tag to checkout and replace +``` + +## Required secrets (inherited) + +| Secret | Required | Purpose | +|--------|----------|---------| +| `E2E_SSH_PRIVATE_KEY` | Yes | SSH key for master node | +| `E2E_SSH_HOST` | Yes | Master node IP/hostname | +| `E2E_SSH_USER` | Yes | SSH user on master | +| `SSH_VM_USER` | No | User for VM nodes (default: `cloud`) | +| `E2E_SSH_JUMP_HOST` | No | Jump/bastion host for `10.10.10.x` networks | +| `E2E_SSH_JUMP_USER` | No | SSH user on jump host | +| `E2E_CLUSTER_KUBECONFIG` | No | Base64 kubeconfig for the virtualization cluster | +| `E2E_TEST_CLUSTER_STORAGE_CLASS` | No | StorageClass for VirtualDisks | +| `E2E_TEST_CLUSTER_CREATE_MODE` | No | Overrides `cluster_provider` input | +| `E2E_DECKHOUSE_LICENSE` | No | DKP license key | +| `E2E_REGISTRY_DOCKER_CFG` | No | Registry auth | +| `GOPROXY` | No | Go module proxy | + +## Label filter + +- `inputs.label_filter` → `E2E_GINKGO_LABEL_FILTER` +- If empty at runtime → auto-set to `!stress-test` (all specs except stress) +- Minimum suite timeout enforced: 90m + +## run-tests flow + +1. Checkout module repo + `storage-e2e` at `inputs.storage_e2e_ref` +2. `go mod edit -replace github.com/deckhouse/storage-e2e=./storage-e2e` to use local ref +3. Open SSH tunnel to master (ProxyJump via `E2E_SSH_JUMP_HOST` when set) +4. `go test -v -timeout 3h30m ./tests/ -run '^TestSdsNodeConfigurator$' -ginkgo.label-filter=...` +5. Upload `e2e-test-output.log` as artifact +6. Cleanup: delete test namespace + VMs (SSH tunnel, `if: always()`) diff --git a/docs/WORKLOG.md b/docs/WORKLOG.md index c73b561..c019ab6 100644 --- a/docs/WORKLOG.md +++ b/docs/WORKLOG.md @@ -140,6 +140,16 @@ All notable changes to this repository are documented here. New entries are appe seeding the built-in DVP provider, `Get` for registered/unregistered modes, `Register` add + replace semantics, `DefaultRegistry` contents, and a race-detector concurrency test for `Register`/`Get` +## 2026-06-22 + +- **Add** `.github/workflows/e2e-reusable.yml`: reusable three-job E2E pipeline (`create-cluster` mocked, `run-tests` mirrors `build_dev` flow, `teardown-cluster` mocked); SSH tunnel, `go mod replace`, Ginkgo label filter, 90m minimum suite timeout. +- **Add** `.github/scripts/e2e-prepare-env.sh`, `.github/scripts/e2e-prepare-workspace.sh`: helper scripts for secrets materialisation and self-hosted runner workspace cleanup. +- **Add** `docs/CI.md`: documents the reusable workflow design, inputs, secrets, and run-tests flow. +- **Update** `README.md`: add CI section linking to `docs/CI.md`. +- **Update** `.github/workflows/e2e-reusable.yml`: add `noop` pipeline_mode (all jobs echo mocked, no real steps run); add `test_suite` input (default `TestSdsNodeConfigurator`) to decouple hardcoded suite name from workflow. +- **Add** `.github/workflows/e2e-self-test.yml`: self-test caller that triggers the reusable workflow in `noop` mode on PRs touching CI files. +- **Update** `.github/workflows/e2e-reusable.yml`: add `skip_storage_e2e_replace` boolean input; gate `checkout storage-e2e`, `go mod edit -replace`, and `setup-go` (with dual-path cache) on this flag so storage-e2e can call the workflow without circular self-reference. +- **Update** `.github/workflows/e2e-self-test.yml`: set `skip_storage_e2e_replace: true`, `test_package: ./tests/test-template/`, `test_suite: TestTemplate`. --- ## 2026-06-23 diff --git a/e2e/tests/cluster_config.yaml b/e2e/tests/cluster_config.yaml new file mode 100644 index 0000000..b3d9633 --- /dev/null +++ b/e2e/tests/cluster_config.yaml @@ -0,0 +1,73 @@ +# Test nested cluster configuration +clusterDefinition: + masters: # Master nodes configuration + - hostname: "master-1" + hostType: "vm" + osType: "Ubuntu 22.04 6.2.0-39-generic" # See internal/config/images.go + cpu: 4 + coreFraction: 20 + ram: 8 + diskSize: 60 + workers: # Worker nodes configuration // TODO implement logic allowing to deploy different number of workes and masters with the same config. + - hostname: "worker-1" + hostType: "vm" + osType: "RedOS 8.0 6.6.26-1.red80.x86_64" # See internal/config/images.go + cpu: 2 + coreFraction: 20 + ram: 8 + diskSize: 20 + # - hostname: "worker-2" + # hostType: "vm" + # osType: "RedOS 7.3.6 5.15.78-2.el7.3.x86_64" # See internal/config/images.go + # cpu: 2 + # coreFraction: 50 + # ram: 2 + # diskSize: 50 + # - hostname: "worker-3" + # hostType: "vm" + # osType: "AltLinux Server 10.4" # See internal/config/images.go + # cpu: 2 + # coreFraction: 50 + # ram: 2 + # diskSize: 50 + # - hostname: "worker-4" + # hostType: "vm" + # osType: "AltLinux Server 11" # See internal/config/images.go + # cpu: 2 + # coreFraction: 20 + # ram: 8 + # diskSize: 20 + - hostname: "worker-5" + hostType: "vm" + osType: "Ubuntu 24.04 6.8.0-53-generic" # See internal/config/images.go + cpu: 2 + coreFraction: 20 + ram: 8 + diskSize: 20 + # DKP parameters + dkpParameters: + kubernetesVersion: "Automatic" + podSubnetCIDR: "10.112.0.0/16" + serviceSubnetCIDR: "10.225.0.0/16" + clusterDomain: "cluster.local" + registryRepo: "dev-registry.deckhouse.io/sys/deckhouse-oss" + devBranch: "main" + # Module configuration + modules: + - name: "snapshot-controller" # TODO add MPO + version: 1 + enabled: true + modulePullOverride: "main" # imageTag for ModulePullOverride. Main is default value, used if not specified. Created always when rehistryRepo starts with dev- + dependencies: [] + - name: "sds-local-volume" + version: 1 + enabled: true + dependencies: + - "snapshot-controller" + - "sds-node-configurator" + - name: "sds-node-configurator" + version: 1 + enabled: true + settings: + enableThinProvisioning: true + dependencies: []