diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 1d17740911..06b48d0513 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -290,8 +290,8 @@ jobs: uses: actions/cache@v4 with: path: | - ./dev/.gocache - ./dev/.gomodcache + ./dev/build/.gocache + ./dev/build/.gomodcache key: dryrun-tests-go-cache-${{ hashFiles('**/go.sum') }} restore-keys: | dryrun-tests-go-cache- diff --git a/.github/workflows/v3-e2e.yaml b/.github/workflows/v3-e2e.yaml new file mode 100644 index 0000000000..6a7fb0dfac --- /dev/null +++ b/.github/workflows/v3-e2e.yaml @@ -0,0 +1,226 @@ +name: V3 E2E + +on: + pull_request: + paths: &paths + - .github/workflows/v3-e2e.yaml + - api/** + - cmd/** + - dagger/** + - e2e/kots-release-install-v3/** + - e2e/licenses/ci-v3.yaml + - kinds/** + - local-artifact-mirror/** + - operator/** + - pkg/** + - pkg-new/** + - scripts/** + - utils/** + - web/** + - common.mk + - dagger.json + - go.mod + - go.sum + - Makefile + - versions.mk + + push: + branches: + - main + paths: *paths + +concurrency: + group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} + cancel-in-progress: true + +permissions: + contents: write + +jobs: + output-vars: + name: Output variables + runs-on: ubuntu-latest + outputs: + git_version: ${{ steps.output_vars.outputs.git_version }} + k0s_minor_version: ${{ steps.output_vars.outputs.k0s_minor_version }} + ec_version: ${{ steps.output_vars.outputs.ec_version }} + app_version: ${{ steps.output_vars.outputs.app_version }} + steps: + - name: Checkout + uses: actions/checkout@v6 + with: + fetch-depth: 0 # necessary for getting the last tag + + - name: Get git sha + id: git_sha + uses: ./.github/actions/git-sha + + - name: Output variables + id: output_vars + run: | + GIT_VERSION=$(git describe --tags --abbrev=4 --match='[0-9]*.[0-9]*.[0-9]*') + echo "GIT_VERSION=\"$GIT_VERSION\"" + echo "git_version=$GIT_VERSION" >> $GITHUB_OUTPUT + + K0S_MINOR_VERSION=$(make print-K0S_MINOR_VERSION) + echo "K0S_MINOR_VERSION=\"$K0S_MINOR_VERSION\"" + echo "k0s_minor_version=$K0S_MINOR_VERSION" >> $GITHUB_OUTPUT + + EC_VERSION="$(./scripts/print-ec-version.sh "$GIT_VERSION" "$K0S_MINOR_VERSION")-v3" + echo "EC_VERSION=\"$EC_VERSION\"" + echo "ec_version=$EC_VERSION" >> $GITHUB_OUTPUT + + APP_VERSION="appver-dev-v3-${{ steps.git_sha.outputs.git_sha }}" + echo "APP_VERSION=\"$APP_VERSION\"" + echo "app_version=$APP_VERSION" >> $GITHUB_OUTPUT + + build-release: + name: Build release + needs: + - output-vars + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Cache embedded bins + uses: actions/cache@v4 + with: + path: | + output/bins + key: bins-cache-${{ hashFiles('Makefile', 'versions.mk') }} + restore-keys: | + bins-cache- + + - name: Setup go + uses: actions/setup-go@v6 + with: + go-version-file: go.mod + cache-dependency-path: "**/*.sum" + + - name: Setup node + uses: actions/setup-node@v6 + with: + node-version-file: ./web/.nvmrc + + - name: Setup oras + uses: oras-project/setup-oras@v1 + + - name: Setup crane + uses: imjasonh/setup-crane@v0.4 + + - name: Setup dagger + run: | + curl -fsSL https://dl.dagger.io/dagger/install.sh | sh + sudo mv ./bin/dagger /usr/local/bin/dagger + + - name: Setup replicated cli + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + gh release download --repo replicatedhq/replicated --pattern '*linux_amd64.tar.gz' --output replicated.tar.gz + tar xf replicated.tar.gz replicated && rm replicated.tar.gz + mv replicated /usr/local/bin/replicated + + - name: Free up runner disk space # this is much faster than .github/actions/free-disk-space + run: | + df -h + sudo rm -rf \ + /usr/share/swift \ + /usr/share/dotnet \ + /usr/lib/jvm \ + /usr/local/share/boost \ + /usr/local/lib/heroku \ + /usr/local/julia* \ + /usr/local/.ghcup \ + /usr/local/share/powershell \ + /usr/local/bin/aliyun \ + /usr/local/bin/azcopy \ + /usr/local/bin/bicep \ + /usr/local/bin/cpack \ + /usr/local/bin/hub \ + /usr/local/bin/minikube \ + /usr/local/bin/packer \ + /usr/local/bin/pulumi* \ + /usr/local/bin/sam \ + /usr/local/bin/stack \ + /usr/local/bin/terraform \ + /usr/local/bin/oc + df -h + + - name: Build release + env: + K0S_MINOR_VERSION: "${{ needs.output-vars.outputs.k0s_minor_version }}" + VERSION: "${{ needs.output-vars.outputs.git_version }}" + EC_VERSION: ${{ needs.output-vars.outputs.ec_version }} + APP_VERSION: ${{ needs.output-vars.outputs.app_version }} + APP_CHANNEL: CI-V3 + APP_CHANNEL_ID: 36LoGcOOLvEPFXQXCUsFub28Abi + APP_CHANNEL_SLUG: ci-v3 + AWS_ACCESS_KEY_ID: ${{ secrets.STAGING_EMBEDDED_CLUSTER_UPLOAD_IAM_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.STAGING_EMBEDDED_CLUSTER_UPLOAD_IAM_SECRET }} + AWS_REGION: "us-east-1" + S3_BUCKET: tf-staging-embedded-cluster-bin + REPLICATED_API_TOKEN: ${{ secrets.STAGING_REPLICATED_API_TOKEN }} + USE_CHAINGUARD: "1" + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + run: | + make e2e-v3-initial-release + + e2e-headless-online: + name: E2E headless online + needs: + - output-vars + - build-release + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v6 + + - name: Setup dagger + run: | + curl -fsSL https://dl.dagger.io/dagger/install.sh | sh + sudo mv ./bin/dagger /usr/local/bin/dagger + + - name: Run test + id: test + env: + APP_VERSION: ${{ needs.output-vars.outputs.app_version }} + KUBE_VERSION: "1.${{ needs.output-vars.outputs.k0s_minor_version }}" + CMX_REPLICATED_API_TOKEN: ${{ secrets.CMX_REPLICATED_API_TOKEN }} + CMX_SSH_PRIVATE_KEY: ${{ secrets.CMX_SSH_PRIVATE_KEY }} + run: | + # Run test and export results directory + dagger call e-2-e-run-headless \ + --scenario=online \ + --app-version=$APP_VERSION \ + --kube-version=$KUBE_VERSION \ + --license-file ./e2e/licenses/ci-v3.yaml \ + --cmx-token=env://CMX_REPLICATED_API_TOKEN \ + --ssh-key=env://CMX_SSH_PRIVATE_KEY \ + export --path=./e2e-results + + # Read result.json to extract test result + if [ -f ./e2e-results/result.json ]; then + success=$(jq -r '.Success' ./e2e-results/result.json) + echo "Test success: $success" + + # Exit with test result + if [ "$success" = "true" ]; then + exit 0 + else + error=$(jq -r '.Error' ./e2e-results/result.json) + echo "::error::E2E test failed: $error" + exit 1 + fi + else + echo "::error::result.json not found in e2e-results directory" + exit 1 + fi + + - name: Upload test results + if: ${{ !cancelled() && hashFiles('./e2e-results/**') != '' }} + uses: actions/upload-artifact@v4 + with: + name: e2e-results-headless-online + path: ./e2e-results/ diff --git a/Makefile b/Makefile index 256f033ffa..93584b4364 100644 --- a/Makefile +++ b/Makefile @@ -132,10 +132,8 @@ cmd/installer/goods/bins/local-artifact-mirror: output/bins/fio-%: mkdir -p output/bins - docker build -t fio --build-arg FIO_VERSION=$(call split-hyphen,$*,1) --build-arg PLATFORM=$(OS)/$(call split-hyphen,$*,2) fio - docker rm -f fio && docker run --name fio fio - docker cp fio:/output/fio $@ - docker rm -f fio + dagger call build-fio --version=$(call split-hyphen,$*,1) --arch=$(call split-hyphen,$*,2) export --path=$@ + chmod +x $@ touch $@ .PHONY: cmd/installer/goods/bins/fio @@ -184,22 +182,21 @@ output/bin/embedded-cluster-release-builder: CGO_ENABLED=0 go build -o output/bin/embedded-cluster-release-builder e2e/embedded-cluster-release-builder/main.go .PHONY: e2e-v3-initial-release -e2e-v3-initial-release: export ARCH = amd64 +e2e-v3-initial-release: export ARCH ?= amd64 e2e-v3-initial-release: export UPLOAD_BINARIES = 1 e2e-v3-initial-release: export ENABLE_V3 = 1 e2e-v3-initial-release: initial-release .PHONY: initial-release -initial-release: export EC_VERSION = $(VERSION)-$(CURRENT_USER) -initial-release: export APP_VERSION = appver-dev-$(call random-string) +initial-release: export EC_VERSION ?= $(VERSION)-$(CURRENT_USER) +initial-release: export APP_VERSION ?= appver-dev-$(call random-string) initial-release: export RELEASE_YAML_DIR = $(if $(filter 1,$(ENABLE_V3)),e2e/kots-release-install-v3,e2e/kots-release-install) -initial-release: export V2_ENABLED = 0 initial-release: check-env-EC_VERSION check-env-APP_VERSION UPLOAD_BINARIES=$(if $(UPLOAD_BINARIES),$(UPLOAD_BINARIES),0) \ ./scripts/build-and-release.sh .PHONY: rebuild-release -rebuild-release: export EC_VERSION = $(VERSION)-$(CURRENT_USER) +rebuild-release: export EC_VERSION ?= $(VERSION)-$(CURRENT_USER) rebuild-release: export RELEASE_YAML_DIR = $(if $(filter 1,$(ENABLE_V3)),e2e/kots-release-install-v3,e2e/kots-release-install) rebuild-release: check-env-EC_VERSION check-env-APP_VERSION UPLOAD_BINARIES=$(if $(UPLOAD_BINARIES),$(UPLOAD_BINARIES),0) \ @@ -208,10 +205,9 @@ rebuild-release: check-env-EC_VERSION check-env-APP_VERSION .PHONY: upgrade-release upgrade-release: RANDOM_STRING = $(call random-string) -upgrade-release: export EC_VERSION = $(VERSION)-$(CURRENT_USER)-upgrade-$(RANDOM_STRING) -upgrade-release: export APP_VERSION = appver-dev-$(call random-string)-upgrade-$(RANDOM_STRING) +upgrade-release: export EC_VERSION ?= $(VERSION)-$(CURRENT_USER)-upgrade-$(RANDOM_STRING) +upgrade-release: export APP_VERSION ?= appver-dev-$(call random-string)-upgrade-$(RANDOM_STRING) upgrade-release: export RELEASE_YAML_DIR = $(if $(filter 1,$(ENABLE_V3)),e2e/kots-release-upgrade-v3,e2e/kots-release-upgrade) -upgrade-release: export V2_ENABLED = 0 upgrade-release: check-env-EC_VERSION check-env-APP_VERSION UPLOAD_BINARIES=$(if $(UPLOAD_BINARIES),$(UPLOAD_BINARIES),1) \ ./scripts/build-and-release.sh diff --git a/README.md b/README.md index cfb8ede70d..d879540e33 100644 --- a/README.md +++ b/README.md @@ -461,12 +461,17 @@ The V3 installer includes a Dagger-based E2E test framework that provides portab **Quick Start:** ```bash -# Provision a test VM +make e2e-v3-initial-release + dagger call with-one-password --service-account=env:OP_SERVICE_ACCOUNT_TOKEN \ - test-provision-vm string + e-2-e-run-headless \ + --scenario=online \ + --app-version= \ + --kube-version=1.33 \ + --license-file=./local-dev/license.yaml ``` -**Documentation:** See [dagger/e2e/README.md](dagger/e2e/README.md) for comprehensive E2E testing guide, including: +**Documentation:** See [dagger/README.md](dagger/README.md) for comprehensive E2E testing guide, including: - Setup and prerequisites - Available test scenarios - Troubleshooting diff --git a/common.mk b/common.mk index 485aeb7403..e671e9f112 100644 --- a/common.mk +++ b/common.mk @@ -1,4 +1,5 @@ SHELL := /bin/bash +MAKEFLAGS += --no-print-directory ARCH ?= $(shell go env GOARCH) CURRENT_USER := $(if $(GITHUB_USER),$(GITHUB_USER),$(shell id -u -n)) diff --git a/dagger.json b/dagger.json index f8022d9509..d8ed87f59a 100644 --- a/dagger.json +++ b/dagger.json @@ -1,13 +1,15 @@ { "name": "embedded-cluster", - "engineVersion": "v0.19.6", + "engineVersion": "v0.19.7", "sdk": { "source": "go" }, "include": [ "!build", "!cmd/installer/goods/bins/**", + "cmd/installer/goods/bins/.placeholder", "!cmd/installer/goods/internal/bins/**", + "cmd/installer/goods/internal/bins/.placeholder", "!dagger/embedded-cluster", "!dev/build", "!local-artifact-mirror/bin", @@ -39,8 +41,8 @@ }, { "name": "replicated", - "source": "github.com/replicatedhq/daggerverse/replicated@bf96fd742cf08239442ac155de7143bb7ef8f7fd", - "pin": "bf96fd742cf08239442ac155de7143bb7ef8f7fd" + "source": "github.com/replicatedhq/daggerverse/replicated@41123ddba40a9594307f24cb69ef6fbafda38e44", + "pin": "41123ddba40a9594307f24cb69ef6fbafda38e44" } ], "source": "dagger", diff --git a/dagger/README.md b/dagger/README.md index 61697449f3..d97d601a3c 100644 --- a/dagger/README.md +++ b/dagger/README.md @@ -80,7 +80,7 @@ The V3 E2E test framework provides: 2. **1Password Service Account Token** - The E2E framework uses 1Password for secret management. You need a service account token with access to the secrets. + The E2E framework uses 1Password for secret management. You need a service account token with access to the Vault. ```bash export OP_SERVICE_ACCOUNT_TOKEN="your-token-here" @@ -93,54 +93,113 @@ The V3 E2E test framework provides: ##### Required Secrets -The following secrets must be available in 1Password in the **"Developer Automation"** vault under the **"EC CI"** item: +The following secrets are available in 1Password in the **"Developer Automation"** vault under the **"EC Dev"** item: | Secret Field Name | Purpose | |-------------------|---------| | `CMX_REPLICATED_API_TOKEN` | CMX API access for VM provisioning | | `CMX_SSH_PRIVATE_KEY` | SSH private key for accessing provisioned VMs | +| `ARTIFACT_UPLOAD_AWS_ACCESS_KEY_ID` | AWS S3 access key for artifact uploads | +| `ARTIFACT_UPLOAD_AWS_SECRET_ACCESS_KEY` | AWS S3 secret key for artifact uploads | +| `STAGING_REPLICATED_API_TOKEN` | Replicated API access for creating releases | **Note:** The vault name defaults to "Developer Automation" and can be overridden via `--vault-name` parameter in the `with-one-password` call. -Additional secrets (for future PRs): -- `STAGING_REPLICATED_API_TOKEN` - Replicated API access for creating releases -- `STAGING_EMBEDDED_CLUSTER_UPLOAD_IAM_KEY_ID` - AWS S3 access key for artifact uploads -- `STAGING_EMBEDDED_CLUSTER_UPLOAD_IAM_SECRET` - AWS S3 secret key -- `GITHUB_TOKEN` - GitHub API access - #### Quick Start -##### 1. Provision a Test VM +##### Building a Release + +Before running E2E tests, you must build and release a version of embedded-cluster that can be tested against: + +```bash +make e2e-v3-initial-release +``` + +This command: +- Builds the embedded-cluster binaries +- Uploads artifacts to S3 +- Creates a release in the Replicated staging environment +- Generates an app version that can be used in E2E tests (e.g., `appver-dev-xpXCTO`) + +The app version returned by this command should be used as the `--app-version` parameter in your E2E test commands. + +##### Running a Complete E2E Test + +Run a complete headless installation test (provisions VM, installs, validates, and cleans up): + +**Online Installation:** + +```bash +dagger call with-one-password --service-account=env:OP_SERVICE_ACCOUNT_TOKEN \ + e-2-e-run-headless \ + --scenario=online \ + --app-version=appver-dev-xpXCTO \ + --kube-version=1.33 \ + --license-file=./local-dev/ethan-dev-license.yaml +``` -Create a fresh CMX VM for testing: +**Airgap Installation:** ```bash dagger call with-one-password --service-account=env:OP_SERVICE_ACCOUNT_TOKEN \ - test-provision-vm --name="my-test-vm" string + e-2-e-run-headless \ + --scenario=airgap \ + --app-version=appver-dev-xpXCTO \ + --kube-version=1.33 \ + --license-file=./local-dev/ethan-dev-license.yaml ``` This will: -- Create a Ubuntu 22.04 VM with default settings -- Wait for SSH to become available -- Discover the private IP address -- Return VM details (ID, name, network ID, IP address) +- Provision a fresh Ubuntu 22.04 VM (8GB RAM, 4 CPUs) +- For airgap: apply network policy to block internet access +- Perform a headless CLI installation +- Validate the installation +- Clean up the VM automatically +- Return comprehensive test results + +##### Running Test Steps Individually -##### 2. Run Commands on VM +For debugging or development, you can run each test step separately: -Execute commands on the provisioned VM: +**1. Provision a Test VM:** ```bash dagger call with-one-password --service-account=env:OP_SERVICE_ACCOUNT_TOKEN \ - test-provision-vm run-command --command="ls,-la,/tmp" + provision-cmx-vm --name="my-test-vm" string ``` -##### 3. Cleanup +This returns the VM ID which you'll use in subsequent steps. + +**2. Install Embedded Cluster:** + +```bash +dagger call with-one-password --service-account=env:OP_SERVICE_ACCOUNT_TOKEN \ + with-cmx-vm --vm-id=YOUR_VM_ID \ + install-headless \ + --scenario=online \ + --app-version=appver-dev-xpXCTO \ + --license-file=./local-dev/ethan-dev-license.yaml \ + --config-values-file=./assets/config-values.yaml +``` -Remove the VM when done: +**3. Validate Installation:** ```bash dagger call with-one-password --service-account=env:OP_SERVICE_ACCOUNT_TOKEN \ - test-provision-vm cleanup + with-cmx-vm --vm-id=YOUR_VM_ID \ + validate \ + --scenario=online \ + --expected-kube-version=1.33 \ + --expected-app-version=appver-dev-xpXCTO \ + string +``` + +**4. Cleanup (when done):** + +```bash +dagger call with-one-password --service-account=env:OP_SERVICE_ACCOUNT_TOKEN \ + with-cmx-vm --vm-id=YOUR_VM_ID \ + cleanup ``` #### Available Commands @@ -151,7 +210,7 @@ Initializes the E2E test module. 1Password integration is configured via `with-o ```bash dagger call with-one-password --service-account=env:OP_SERVICE_ACCOUNT_TOKEN \ - test-provision-vm \ + provision-cmx-vm \ --name="test-vm" \ --distribution="ubuntu" \ --version="22.04" \ @@ -176,7 +235,7 @@ Execute commands on a provisioned VM: ```bash dagger call with-one-password --service-account=env:OP_SERVICE_ACCOUNT_TOKEN \ - test-provision-vm run-command --command="ls,-la,/tmp" + provision-cmx-vm run-command --command="ls,-la,/tmp" ``` Commands are automatically executed with `sudo` and `PATH=$PATH:/usr/local/bin`. @@ -187,7 +246,7 @@ Expose a port on the VM and get a public hostname: ```bash dagger call with-one-password --service-account=env:OP_SERVICE_ACCOUNT_TOKEN \ - test-provision-vm expose-port --port=30000 --protocol="https" + provision-cmx-vm expose-port --port=30000 --protocol="https" ``` **Parameters:** @@ -200,18 +259,106 @@ Remove a provisioned VM: ```bash dagger call with-one-password --service-account=env:OP_SERVICE_ACCOUNT_TOKEN \ - test-provision-vm cleanup + provision-cmx-vm cleanup +``` + +##### Install Headless + +Perform a headless (CLI) installation of embedded-cluster: + +```bash +dagger call with-one-password --service-account=env:OP_SERVICE_ACCOUNT_TOKEN \ + provision-cmx-vm install-headless \ + --scenario=online \ + --app-version=v1.0.0 \ + --license="..." \ + --license-id="..." +``` + +**Parameters:** +- `scenario`: Installation scenario ("online" or "airgap") +- `app-version`: App version to install +- `license`: License content +- `license-id`: License ID for downloading +- `config-file`: Optional config file content + +##### Validate Installation + +Validate an embedded-cluster installation after it completes: + +```bash +dagger call with-one-password --service-account=env:OP_SERVICE_ACCOUNT_TOKEN \ + provision-cmx-vm validate \ + --expected-k8s-version=1.31 \ + --expected-app-version=v1.0.0 \ + --airgap=false ``` -#### Test Scenarios (PR 1 Foundation Only) +**Parameters:** +- `expected-k8s-version`: Expected Kubernetes version (e.g., "1.31") +- `expected-app-version`: Expected app version (e.g., "v1.0.0") +- `airgap`: Whether to use airgap validation mode (default: false) + +**Validation Checks:** + +The validation performs comprehensive checks including: + +1. **Kubernetes Cluster Health** + - Verifies all nodes are running the expected k8s version + - Checks kubelet version matches expected version on all nodes + - Validates node readiness status + +2. **Installation CRD Status** + - Verifies Installation resource exists and is in "Installed" state + - Confirms embedded-cluster operator successfully completed installation + +3. **Application Deployment** + - Waits for application's nginx pods to be Running + - Verifies correct app version is deployed + - Confirms no upgrade artifacts present + +4. **Admin Console Components** + - Confirms kotsadm pods are healthy + - Confirms kotsadm API is healthy (kubectl kots get apps works) + - Validates admin console branding configmap has DR label + +5. **Data Directory Configuration** + - Validates K0s data directory is configured correctly + - Validates OpenEBS data directory is configured correctly + - Validates Velero pod volume path is configured correctly + - Verifies all components use expected base directory + +6. **Pod and Job Health** + - All non-Job pods are in Running/Completed/Succeeded state + - All Running pods have ready containers + - All Jobs have completed successfully + +**Return Value:** + +Returns a `ValidationResult` containing: +- `Success`: Overall validation status (bool) +- `ClusterHealth`: Cluster health check result +- `InstallationCRD`: Installation CRD check result +- `AppDeployment`: App deployment check result +- `AdminConsole`: Admin console check result +- `DataDirectories`: Data directories check result +- `PodsAndJobs`: Pod and job health check result + +Each check result includes: +- `Passed`: Whether the check passed (bool) +- `ErrorMessage`: Error message if failed (string) +- `Details`: Additional context or details (string) + +#### Test Scenarios -This is **PR 1: Foundation and Secret Management**. The following scenarios will be implemented in future PRs: +This framework supports the following E2E test scenarios: -##### Future: PR 3 - Headless Installation Tests +##### Headless Installation Tests (PR 3) - Online installation (headless CLI) - Airgap installation (headless CLI) +- Comprehensive validation after installation -##### Future: PR 5 - Browser-Based Installation Tests +##### Future: Browser-Based Installation Tests (PR 5) - Online installation (Playwright UI) - Airgap installation (Playwright UI) @@ -227,7 +374,7 @@ To develop E2E tests locally: 2. Test VM provisioning: ```bash dagger call with-one-password --service-account=env:OP_SERVICE_ACCOUNT_TOKEN \ - test-provision-vm --name="dev-test" + provision-cmx-vm --name="dev-test" ``` 3. Make changes to `dagger/e2e.go` diff --git a/dagger/assets/config-values.yaml b/dagger/assets/config-values.yaml new file mode 100644 index 0000000000..19a90e4536 --- /dev/null +++ b/dagger/assets/config-values.yaml @@ -0,0 +1,23 @@ +apiVersion: kots.io/v1beta1 +kind: ConfigValues +spec: + values: + text_required: + value: "text required values" + text_required_with_regex: + value: "ethan@replicated.com" + password_required: + value: "password required value" + textarea_required: + value: "textarea required value" + checkbox_required: + value: "1" + dropdown_required: + value: "option1" + radio_required: + value: "option1" + file_required: + value: "ZmlsZSByZXF1aXJlZCB2YWx1ZQo=" + filename: "file_required.txt" + hidden_required: + value: "hidden required value" diff --git a/dagger/chainguard.go b/dagger/chainguard.go index 07dada83ca..d9d34af609 100644 --- a/dagger/chainguard.go +++ b/dagger/chainguard.go @@ -20,7 +20,7 @@ func (m *chainguard) melangeBuildGo( keygen := m.melangeKeygen(imageTag) // Create cache volumes for improved build performance - goModCache := dag.CacheVolume("ec-melange-gomodcache") + goCache := dag.CacheVolume("ec-melange-gocache") apkCache := dag.CacheVolume("ec-melange-apkcache") c := dag.Container(). @@ -30,7 +30,7 @@ func (m *chainguard) melangeBuildGo( WithFile("/workspace/melange.rsa", keygen.File("/workspace/melange.rsa")). WithEnvVariable("MELANGE_CACHE_DIR", "/cache/melange"). WithEnvVariable("MELANGE_APK_CACHE_DIR", "/cache/apk"). - WithMountedCache("/cache/melange", goModCache, dagger.ContainerWithMountedCacheOpts{ + WithMountedCache("/cache/melange", goCache, dagger.ContainerWithMountedCacheOpts{ Sharing: dagger.CacheSharingModeShared, }). WithMountedCache("/cache/apk", apkCache, dagger.ContainerWithMountedCacheOpts{ diff --git a/dagger/cmx.go b/dagger/cmx.go index 0c13616477..7afe5a5229 100644 --- a/dagger/cmx.go +++ b/dagger/cmx.go @@ -3,14 +3,212 @@ package main import ( "context" "fmt" + "strconv" "strings" "time" "dagger/embedded-cluster/internal/dagger" ) -// CMXInstance wraps the CMX VM instance. -type CMXInstance struct { +const ( + SSHUser = "ec-e2e-test" + DataDir = "/var/lib/embedded-cluster-smoke-test-staging-app" + AppNamespace = "embedded-cluster-smoke-test-staging-app" +) + +// Provisions a new CMX VM for E2E testing. +// +// This creates a fresh VM with the specified configuration and waits for it to be ready. +// The VM is automatically configured with SSH access and networking. +// +// Example: +// +// dagger call with-one-password --service-account=env:OP_SERVICE_ACCOUNT_TOKEN \ +// provision-cmx-vm --name="my-test-vm" +func (m *EmbeddedCluster) ProvisionCmxVm( + ctx context.Context, + // Name for the VM + // +default="ec-e2e-test" + name string, + // OS distribution + // +default="ubuntu" + distribution string, + // Distribution version + // +default="22.04" + version string, + // Instance type + // +default="r1.medium" + instanceType string, + // Disk size in GB + // +default=50 + diskSize int, + // How long to wait for VM to be ready + // +default="10m" + wait string, + // TTL for the VM + // +default="2h" + ttl string, + // CMX API token + // +optional + cmxToken *dagger.Secret, + // SSH key + // +optional + sshKey *dagger.Secret, +) (*CmxInstance, error) { + // Get CMX API token and SSH key from 1Password if not provided + cmxToken = m.mustResolveSecret(cmxToken, "CMX_REPLICATED_API_TOKEN") + sshKey = m.mustResolveSecret(sshKey, "CMX_SSH_PRIVATE_KEY") + + // Create VM using Replicated Dagger module + vms, err := dag. + Replicated(cmxToken). + VMCreate( + ctx, + dagger.ReplicatedVMCreateOpts{ + Name: name, + Wait: wait, + TTL: ttl, + Distribution: distribution, + Version: version, + Count: 1, + Disk: diskSize, + InstanceType: instanceType, + }, + ) + if err != nil { + return nil, fmt.Errorf("create vm: %w", err) + } + + // Get the first VM + if len(vms) == 0 { + return nil, fmt.Errorf("no VMs created") + } + vm := vms[0] + + return m.cmxVmToCmxInstance(ctx, &vm, cmxToken, sshKey) +} + +// WithCmxVm connects to an existing CMX VM by ID. +// +// This queries the CMX API to get the VM details and creates a CmxInstance. +// Unlike ProvisionCmxVm, this does not create a new VM - it connects to one that already exists. +// +// Example: +// +// dagger call with-one-password --service-account=env:OP_SERVICE_ACCOUNT_TOKEN \ +// with-cmx-vm --vm-id="abc123" +func (m *EmbeddedCluster) WithCmxVm( + ctx context.Context, + // VM ID + vmId string, + // SSH user + // +default="ec-e2e-test" + sshUser string, + // CMX API token + // +optional + cmxToken *dagger.Secret, + // SSH key + // +optional + sshKey *dagger.Secret, +) (*CmxInstance, error) { + // Get CMX API token and SSH key from 1Password if not provided + cmxToken = m.mustResolveSecret(cmxToken, "CMX_REPLICATED_API_TOKEN") + sshKey = m.mustResolveSecret(sshKey, "CMX_SSH_PRIVATE_KEY") + + // List all VMs and find the one with matching ID + vms, err := dag. + Replicated(cmxToken). + VMList(ctx) + if err != nil { + return nil, fmt.Errorf("list vms: %w", err) + } + + // Find VM with matching ID + var vm *dagger.ReplicatedVM + for _, v := range vms { + id, err := v.ItemID(ctx) + if err != nil { + continue + } + if string(id) == vmId { + vm = &v + break + } + } + + if vm == nil { + return nil, fmt.Errorf("vm with id %s not found", vmId) + } + + return m.cmxVmToCmxInstance(ctx, vm, cmxToken, sshKey) +} + +func (m *EmbeddedCluster) cmxVmToCmxInstance(ctx context.Context, vm *dagger.ReplicatedVM, cmxToken *dagger.Secret, sshKey *dagger.Secret) (*CmxInstance, error) { + // Get VM details + vmID, err := vm.ItemID(ctx) + if err != nil { + return nil, fmt.Errorf("get vm id: %w", err) + } + + vmName, err := vm.Name(ctx) + if err != nil { + return nil, fmt.Errorf("get vm name: %w", err) + } + + networkID, err := vm.NetworkID(ctx) + if err != nil { + return nil, fmt.Errorf("get network id: %w", err) + } + + sshEndpoint, err := vm.DirectSshendpoint(ctx) + if err != nil { + return nil, fmt.Errorf("get ssh endpoint: %w", err) + } + + directSSHPort, err := vm.DirectSshport(ctx) + if err != nil { + return nil, fmt.Errorf("get direct ssh port: %w", err) + } + + instance := &CmxInstance{ + VmID: string(vmID), + Name: vmName, + NetworkID: networkID, + SSHEndpoint: sshEndpoint, + SSHPort: directSSHPort, + SSHUser: SSHUser, + SSHKey: sshKey, + CMXToken: cmxToken, + } + + // Wait for SSH to be available + if err := instance.waitForSSH(ctx); err != nil { + return nil, fmt.Errorf("wait for ssh: %w", err) + } + + // Discover private IP + privateIP, err := instance.discoverPrivateIP(ctx) + if err != nil { + return nil, fmt.Errorf("discover private ip: %w", err) + } + instance.PrivateIP = privateIP + + // Log SSH access instructions + fmt.Printf("\n=== SSH Access Instructions ===\n") + fmt.Printf("VM ID: %s\n", instance.VmID) + fmt.Printf("VM Name: %s\n", instance.Name) + fmt.Printf("\n1. Get the SSH private key:\n") + fmt.Printf(" dagger call with-one-password --service-account=env:OP_SERVICE_ACCOUNT_TOKEN one-password find-secret --field CMX_SSH_PRIVATE_KEY plaintext > /tmp/cmx-ssh-key\n") + fmt.Printf(" chmod 600 /tmp/cmx-ssh-key\n") + fmt.Printf("\n2. SSH into the server:\n") + fmt.Printf(" ssh -i /tmp/cmx-ssh-key -p %d %s@%s\n", instance.SSHPort, instance.SSHUser, instance.SSHEndpoint) + fmt.Printf("===============================\n\n") + + return instance, nil +} + +// CmxInstance wraps the CMX VM instance. +type CmxInstance struct { // VM ID VmID string // VM name @@ -32,13 +230,13 @@ type CMXInstance struct { } // String returns a string representation of the CMX instance. -func (i *CMXInstance) String() string { - return fmt.Sprintf("CMXInstance{VmID: %s, SSHEndpoint: %s, SSHPort: %d}", i.VmID, i.SSHEndpoint, i.SSHPort) +func (i *CmxInstance) String() string { + return fmt.Sprintf("CmxInstance{VmID: %s, SSHEndpoint: %s, SSHPort: %d}", i.VmID, i.SSHEndpoint, i.SSHPort) } // sshClient returns a container with openssh-client installed and the SSH key configured. // The key is mounted at /root/.ssh/id_rsa with proper permissions and formatting. -func (i *CMXInstance) sshClient() *dagger.Container { +func (i *CmxInstance) sshClient() *dagger.Container { return dag.Container(). From("ubuntu:24.04"). WithEnvVariable("DEBIAN_FRONTEND", "noninteractive"). @@ -52,7 +250,7 @@ func (i *CMXInstance) sshClient() *dagger.Container { } // waitForSSH waits for SSH to become available on the VM. -func (i *CMXInstance) waitForSSH(ctx context.Context) error { +func (i *CmxInstance) waitForSSH(ctx context.Context) error { timeout := time.After(5 * time.Minute) ticker := time.NewTicker(5 * time.Second) defer ticker.Stop() @@ -62,7 +260,7 @@ func (i *CMXInstance) waitForSSH(ctx context.Context) error { case <-timeout: return fmt.Errorf("timed out waiting for ssh") case <-ticker.C: - _, err := i.RunCommand(ctx, []string{"uptime"}) + _, err := i.Command(`uptime`).Stdout(ctx) if err == nil { return nil } @@ -72,8 +270,8 @@ func (i *CMXInstance) waitForSSH(ctx context.Context) error { } // discoverPrivateIP discovers the private IP address of the VM. -func (i *CMXInstance) discoverPrivateIP(ctx context.Context) (string, error) { - stdout, err := i.RunCommand(ctx, []string{"hostname", "-I"}) +func (i *CmxInstance) discoverPrivateIP(ctx context.Context) (string, error) { + stdout, err := i.Command(`hostname -I`).Stdout(ctx) if err != nil { return "", fmt.Errorf("run hostname command: %w", err) } @@ -88,64 +286,124 @@ func (i *CMXInstance) discoverPrivateIP(ctx context.Context) (string, error) { return "", fmt.Errorf("no private ip found starting with 10") } -// RunCommand runs a command on the CMX VM. +// Command returns a dagger container that runs a command on the CMX VM. // // Commands are executed with sudo and the PATH is set to include /usr/local/bin. -// Arguments are properly shell-escaped to handle spaces and special characters. +// The returned container can be further customized before calling .Stdout() or other methods. // // Example: // // dagger call with-one-password --service-account=env:OP_SERVICE_ACCOUNT_TOKEN \ -// test-provision-vm run-command --command="ls,-la,/tmp" -func (i *CMXInstance) RunCommand( - ctx context.Context, - // Command to run (as array of strings) - command []string, -) (string, error) { - if len(command) == 0 { - return "", fmt.Errorf("command cannot be empty") - } +// with-cmx-vm --vm-id 8a2a66ef \ +// command --command="ls -la /tmp" stdout +func (i *CmxInstance) Command( + // Command to run + command string, +) *dagger.Container { + return i.CommandWithEnv(command, nil) +} + +// CommandWithEnv runs a command with custom environment variables set on the remote system. +// +// Environment variables are passed as KEY=value pairs and will be available to the command. +// +// Example: +// +// dagger call with-one-password --service-account=env:OP_SERVICE_ACCOUNT_TOKEN \ +// with-cmx-vm --vm-id 8a2a66ef \ +// command-with-env --command="kubectl get pods" --env="KUBECONFIG=/path/to/kubeconfig" stdout +func (i *CmxInstance) CommandWithEnv( + // Command to run + command string, + // Environment variables (e.g., "KUBECONFIG=/path/to/config") + // +optional + env []string, +) *dagger.Container { + env = append([]string{ + // Use \$PATH to escape the variable so it survives bash -c and expands on the remote side + fmt.Sprintf("PATH=$PATH:%s/bin", DataDir), + fmt.Sprintf("KUBECONFIG=%s", fmt.Sprintf("%s/k0s/pki/admin.conf", DataDir)), + }, env...) - // Build the full command with sudo prefix - fullCmd := append([]string{"sudo", "PATH=$PATH:/usr/local/bin"}, command...) + // Build environment variable string + envVars := strings.Join(env, " ") - // Shell-escape each argument to handle spaces and special characters - escapedArgs := make([]string, len(fullCmd)) - for i, arg := range fullCmd { - escapedArgs[i] = shellEscape(arg) - } - cmdStr := strings.Join(escapedArgs, " ") + // Escape single quotes in the command + command = strings.ReplaceAll(command, `'`, `'"'"'`) - stdout, err := i.sshClient(). - WithExec([]string{ - "ssh", - "-i", "/root/.ssh/id_rsa", - "-o", "StrictHostKeyChecking=no", - "-o", "BatchMode=yes", - "-p", fmt.Sprintf("%d", i.SSHPort), - fmt.Sprintf("%s@%s", i.SSHUser, i.SSHEndpoint), - cmdStr, - }). - Stdout(ctx) + // Build the full remote command + // Use 'env' to set environment variables for sudo to avoid secure_path issues + remoteCmd := fmt.Sprintf(`sudo -E env %s bash -c '%s'`, envVars, command) - if err != nil { - return "", fmt.Errorf("run command failed: %w", err) + // Build SSH command + sshCmd := []string{ + "ssh", + "-i", "/root/.ssh/id_rsa", + "-o", "StrictHostKeyChecking=no", + "-o", "BatchMode=yes", + "-o", "ServerAliveInterval=60", + "-o", "ServerAliveCountMax=10", + "-p", fmt.Sprintf("%d", i.SSHPort), + fmt.Sprintf("%s@%s", i.SSHUser, i.SSHEndpoint), + remoteCmd, } - return stdout, nil + // Return container with SSH exec + // We use double quotes around the remote command so variables can expand + return i.sshClient(). + WithEnvVariable("CACHE_BUSTER", time.Now().String()). + WithExec(sshCmd) } -// shellEscape escapes a string for safe use in a shell command. -// It wraps the string in single quotes and escapes any single quotes within. -func shellEscape(s string) string { - // If the string doesn't contain any special characters, return as-is - if !strings.ContainsAny(s, " \t\n'\"\\$`!*?[](){};<>|&~") { - return s +// UploadFile uploads file content to a path on the VM using SCP. +// +// This uses SCP to transfer the file to /tmp first (avoiding permission issues), +// then moves it to the final destination using sudo. +// +// Example: +// +// dagger call with-one-password --service-account=env:OP_SERVICE_ACCOUNT_TOKEN \ +// with-cmx-vm --vm-id 8a2a66ef \ +// upload-file --path=/tmp/myfile.txt --file=/path/to/file.txt +func (i *CmxInstance) UploadFile( + ctx context.Context, + // Destination path on the VM + path string, + // File to upload + file *dagger.File, +) error { + // Create a temporary file in the container with the content + tempPath := "/tmp/upload-file" + tmpDest := fmt.Sprintf("/tmp/upload-%d", time.Now().UnixNano()) + + container := i.sshClient(). + WithFile(tempPath, file). + WithEnvVariable("CACHE_BUSTER", time.Now().String()) + + // Use SCP to upload the file to /tmp on the VM (user has write access) + scpCmd := []string{ + "scp", + "-i", "/root/.ssh/id_rsa", + "-o", "StrictHostKeyChecking=no", + "-o", "BatchMode=yes", + "-o", "ServerAliveInterval=60", + "-o", "ServerAliveCountMax=10", + "-P", fmt.Sprintf("%d", i.SSHPort), + tempPath, + fmt.Sprintf("%s@%s:%s", i.SSHUser, i.SSHEndpoint, tmpDest), + } + + if _, err := container.WithExec(scpCmd).Stdout(ctx); err != nil { + return fmt.Errorf("scp upload to %s: %w", tmpDest, err) } - // Use single quotes and escape any single quotes in the string - // by replacing ' with '\'' - return "'" + strings.ReplaceAll(s, "'", "'\\''") + "'" + // Move the file to the final destination using sudo + moveCmd := fmt.Sprintf("mv %s %s", tmpDest, path) + if _, err := i.Command(moveCmd).Stdout(ctx); err != nil { + return fmt.Errorf("move file to %s: %w", path, err) + } + + return nil } // ExposePort exposes a port on the VM and returns the public hostname. @@ -153,8 +411,9 @@ func shellEscape(s string) string { // Example: // // dagger call with-one-password --service-account=env:OP_SERVICE_ACCOUNT_TOKEN \ -// e2e-init e2e-provision-vm expose-port --port=30000 --protocol="https" -func (i *CMXInstance) ExposePort( +// with-cmx-vm --vm-id 8a2a66ef \ +// expose-port --port=30000 --protocol="https" +func (i *CmxInstance) ExposePort( ctx context.Context, // Port to expose port int, @@ -187,8 +446,8 @@ func (i *CMXInstance) ExposePort( // Example: // // dagger call with-one-password --service-account=env:OP_SERVICE_ACCOUNT_TOKEN \ -// e2e-init e2e-provision-vm cleanup -func (i *CMXInstance) Cleanup(ctx context.Context) (string, error) { +// with-cmx-vm --vm-id 8a2a66ef cleanup +func (i *CmxInstance) Cleanup(ctx context.Context) (string, error) { result, err := dag. Replicated(i.CMXToken). VMRemove(ctx, dagger.ReplicatedVMRemoveOpts{ @@ -201,3 +460,404 @@ func (i *CMXInstance) Cleanup(ctx context.Context) (string, error) { return result, nil } + +// ApplyAirgapNetworkPolicy applies a network policy to block internet access for airgap testing. +// +// This calls the Replicated module's NetworkUpdatePolicy to set the VM's network to airgap mode, +// which prevents the VM from accessing external networks during airgap installation tests. +func (i *CmxInstance) ApplyAirgapNetworkPolicy(ctx context.Context) error { + _, err := dag. + Replicated(i.CMXToken). + NetworkUpdatePolicy(ctx, "airgap", dagger.ReplicatedNetworkUpdatePolicyOpts{ + NetworkID: i.NetworkID, + }) + + if err != nil { + return fmt.Errorf("update network policy to airgap: %w", err) + } + + return nil +} + +// InstallKotsCli installs the kubectl-kots CLI if not already present. +// This is needed for validation commands that use kubectl kots. +func (i *CmxInstance) InstallKotsCli(ctx context.Context) error { + // Check if kubectl-kots is already installed + _, err := i.Command("command -v kubectl-kots").Stdout(ctx) + if err == nil { + // Already installed + return nil + } + + // Install curl if needed + _, err = i.Command("command -v curl").Stdout(ctx) + if err != nil { + installCurlCmd := "apt-get update && apt-get install -y curl" + if _, err := i.Command(installCurlCmd).Stdout(ctx); err != nil { + return fmt.Errorf("install curl: %w", err) + } + } + + // Get AdminConsole version from embedded-cluster + versionOutput, err := i.Command("embedded-cluster-smoke-test-staging-app version").Stdout(ctx) + if err != nil { + return fmt.Errorf("get embedded-cluster version: %w", err) + } + + // Parse version from output like "AdminConsole: v1.117.2-ec.2" + // We want to extract "1.117.2" (without the 'v' prefix and '-ec.2' suffix) + getVersionCmd := `embedded-cluster-smoke-test-staging-app version | grep AdminConsole | awk '{print substr($4,2)}' | cut -d'-' -f1` + kotsVersion, err := i.Command(getVersionCmd).Stdout(ctx) + if err != nil { + return fmt.Errorf("parse kots version: %w", err) + } + + kotsVersion = strings.TrimSpace(kotsVersion) + if kotsVersion == "" { + return fmt.Errorf("could not determine kots version from: %s", versionOutput) + } + + // Download and install kots CLI + installKotsCmd := fmt.Sprintf(`curl --retry 5 -fL -o /tmp/kotsinstall.sh "https://kots.io/install/%s" && chmod +x /tmp/kotsinstall.sh && /tmp/kotsinstall.sh`, kotsVersion) + if _, err := i.Command(installKotsCmd).Stdout(ctx); err != nil { + return fmt.Errorf("install kots cli: %w", err) + } + + return nil +} + +// PrepareRelease downloads embedded-cluster release from replicated.app +// and prepares it for installation. This matches how customers get the binary. +// +// The method downloads the release tarball, extracts it, and places the binary and +// license file in the expected locations for installation. +func (i *CmxInstance) PrepareRelease( + ctx context.Context, + // Installation scenario (online, airgap) + scenario string, + // App version to download + appVersion string, + // License file + licenseFile *dagger.File, +) error { + // Get license content as plain text for passing to install command + _, licenseID, channelID, err := parseLicense(ctx, licenseFile) + if err != nil { + return fmt.Errorf("parse license: %w", err) + } + + // Download embedded-cluster release from replicated.app + releaseURL := fmt.Sprintf("https://ec-e2e-replicated-app.testcluster.net/embedded/embedded-cluster-smoke-test-staging-app/%s/%s", channelID, appVersion) + + if scenario == "airgap" { + releaseURL = fmt.Sprintf("%s?airgap=true", releaseURL) + } + + // For airgap, retry up to 20 times with 1 minute sleep between attempts + // The API returns 400 when the bundle is still being built + if scenario == "airgap" { + if err := i.downloadAirgapBundleWithRetry(ctx, releaseURL, licenseID); err != nil { + return fmt.Errorf("download airgap bundle: %w", err) + } + } else { + downloadCmd := fmt.Sprintf(`curl --retry 5 --retry-all-errors -fL -o /tmp/ec-release.tgz "%s" -H "Authorization: %s"`, releaseURL, licenseID) + if _, err := i.Command(downloadCmd).Stdout(ctx); err != nil { + return fmt.Errorf("download release: %w", err) + } + } + + // Extract release tarball + if _, err := i.Command(`tar xzf /tmp/ec-release.tgz -C /tmp`).Stdout(ctx); err != nil { + return fmt.Errorf("extract release: %w", err) + } + + // Create assets directory + if _, err := i.Command(`mkdir -p /assets`).Stdout(ctx); err != nil { + return fmt.Errorf("create assets directory: %w", err) + } + + // Move binary to /usr/local/bin + moveBinaryCmd := `mv /tmp/embedded-cluster-smoke-test-staging-app /usr/local/bin/embedded-cluster-smoke-test-staging-app` + if _, err := i.Command(moveBinaryCmd).Stdout(ctx); err != nil { + return fmt.Errorf("move binary: %w", err) + } + + // Move license to /assets + moveLicenseCmd := `mv /tmp/license.yaml /assets/license.yaml` + if _, err := i.Command(moveLicenseCmd).Stdout(ctx); err != nil { + return fmt.Errorf("move license: %w", err) + } + + if scenario == "airgap" { + // Move airgap bundle to /assets + moveAirgapBundleCmd := `mv /tmp/embedded-cluster-smoke-test-staging-app.airgap /assets/embedded-cluster-smoke-test-staging-app.airgap` + if _, err := i.Command(moveAirgapBundleCmd).Stdout(ctx); err != nil { + return fmt.Errorf("move airgap bundle: %w", err) + } + } + + // Install kots CLI if not already installed + if err := i.InstallKotsCli(ctx); err != nil { + return fmt.Errorf("install kots cli: %w", err) + } + + return nil +} + +// downloadAirgapBundleWithRetry downloads an airgap bundle with retry logic. +// +// The airgap bundle API may return 400 errors when the bundle is still being built. +// This method retries up to 20 times with a 1 minute sleep between attempts, +// and verifies the downloaded file is at least 1GB to ensure it's complete. +func (i *CmxInstance) downloadAirgapBundleWithRetry(ctx context.Context, url string, licenseID string) error { + for attempt := 1; attempt <= 20; attempt++ { + fmt.Printf("Attempting to download airgap bundle (attempt %d/20)...\n", attempt) + + // Download with curl -f which will fail on HTTP 4xx/5xx errors + downloadCmd := fmt.Sprintf(`curl -fL -o /tmp/ec-release.tgz "%s" -H "Authorization: %s"`, url, licenseID) + _, err := i.Command(downloadCmd).Stdout(ctx) + + if err != nil { + fmt.Printf("Download attempt %d failed: %v\n", attempt, err) + if attempt < 20 { + fmt.Printf("Waiting 1 minute before retry...\n") + time.Sleep(1 * time.Minute) + continue + } + return fmt.Errorf("failed after 20 attempts: %w", err) + } + + // Check file size - airgap bundles should be at least 1GB + sizeCmd := `du -b /tmp/ec-release.tgz | awk '{print $1}'` + sizeStr, err := i.Command(sizeCmd).Stdout(ctx) + if err != nil { + fmt.Printf("Failed to check file size: %v\n", err) + if attempt < 20 { + fmt.Printf("Waiting 1 minute before retry...\n") + time.Sleep(1 * time.Minute) + continue + } + return fmt.Errorf("failed to check file size after 20 attempts: %w", err) + } + + sizeStr = strings.TrimSpace(sizeStr) + sizeBytes, err := strconv.ParseInt(sizeStr, 10, 64) + if err != nil { + return fmt.Errorf("failed to parse file size %q: %w", sizeStr, err) + } + + minSize := int64(1024 * 1024 * 1024) // 1GB + if sizeBytes < minSize { + fmt.Printf("Downloaded file is only %d bytes (%.2f GB), expected at least 1GB. Retrying...\n", + sizeBytes, float64(sizeBytes)/(1024*1024*1024)) + // Remove the incomplete download + if _, err := i.Command("rm -f /tmp/ec-release.tgz").Stdout(ctx); err != nil { + fmt.Printf("Warning: failed to remove incomplete download: %v\n", err) + } + if attempt < 20 { + fmt.Printf("Waiting 1 minute before retry...\n") + time.Sleep(1 * time.Minute) + continue + } + return fmt.Errorf("downloaded file too small after 20 attempts: %d bytes", sizeBytes) + } + + fmt.Printf("Successfully downloaded airgap bundle (%.2f GB) on attempt %d\n", + float64(sizeBytes)/(1024*1024*1024), attempt) + return nil + } + + return fmt.Errorf("failed to download airgap bundle after 20 attempts") +} + +// InstallHeadless performs a headless (CLI) installation without Playwright. +// +// This method downloads the release, optionally uploads a config file, builds the +// installation command with appropriate flags, and runs the installation with a +// 30-minute timeout. It supports both online and airgap scenarios. +// +// Example: +// +// dagger call with-one-password --service-account=env:OP_SERVICE_ACCOUNT_TOKEN \ +// with-cmx-vm --vm-id 8a2a66ef \ +// install-headless --scenario online \ +// --app-version=appver-dev-xpXCTO \ +// --license-file ./local-dev/ethan-dev-3-license.yaml \ +// --config-values-file ./assets/config-values.yaml +func (i *CmxInstance) InstallHeadless( + ctx context.Context, + // Installation scenario (online, airgap) + scenario string, + // App version to install + appVersion string, + // License file + licenseFile *dagger.File, + // Config values file + configValuesFile *dagger.File, +) (*InstallResult, error) { + // Upload config file if provided + if err := i.UploadFile(ctx, "/assets/config-values.yaml", configValuesFile); err != nil { + return nil, fmt.Errorf("upload config file: %w", err) + } + + // Build install command + installCmd := `ENABLE_V3=1 /usr/local/bin/embedded-cluster-smoke-test-staging-app install ` + + `--license /assets/license.yaml ` + + `--target linux ` + + `--headless ` + + `--config-values /assets/config-values.yaml ` + + `--admin-console-password password ` + + `--yes` + + // Add airgap bundle for airgap scenario + if scenario == "airgap" { + installCmd = fmt.Sprintf(`%s --airgap-bundle /assets/embedded-cluster-smoke-test-staging-app.airgap`, installCmd) + } + + // Run installation command with timeout + // Note: We use a simple approach here - start the command and wait for it to complete + // The command itself may take up to 30 minutes + ctx, cancel := context.WithTimeout(ctx, 30*time.Minute) + defer cancel() + + stdout, err := i.Command(installCmd).Stdout(ctx) + if err != nil { + return &InstallResult{ + Success: false, + InstallationLog: stdout, + }, fmt.Errorf("installation failed: %w", err) + } + + // Installation succeeded + return &InstallResult{ + Success: true, + KubeconfigPath: fmt.Sprintf("%s/k0s/pki/admin.conf", DataDir), + InstallationLog: stdout, + }, nil +} + +// DownloadFile downloads a file from the VM to the Dagger container. +// +// This method uses SCP to copy a file from the remote VM to the local container, +// then returns it as a Dagger File that can be exported or used by other functions. +func (i *CmxInstance) DownloadFile( + ctx context.Context, + // Source path on the VM + remotePath string, +) (*dagger.File, error) { + // Local path in container where we'll download the file + localPath := fmt.Sprintf("/tmp/download-%d", time.Now().UnixNano()) + + container := i.sshClient(). + WithEnvVariable("CACHE_BUSTER", time.Now().String()) + + // Use SCP to download the file from the VM + scpCmd := []string{ + "scp", + "-i", "/root/.ssh/id_rsa", + "-o", "StrictHostKeyChecking=no", + "-o", "BatchMode=yes", + "-o", "ServerAliveInterval=60", + "-o", "ServerAliveCountMax=10", + "-P", fmt.Sprintf("%d", i.SSHPort), + fmt.Sprintf("%s@%s:%s", i.SSHUser, i.SSHEndpoint, remotePath), + localPath, + } + + container, err := container.WithExec(scpCmd).Sync(ctx) + if err != nil { + return nil, fmt.Errorf("scp download from %s: %w", remotePath, err) + } + + // Return the downloaded file + return container.File(localPath), nil +} + +// CollectClusterSupportBundle collects a cluster support bundle from the VM. +// +// This method runs kubectl support-bundle to collect diagnostic information from the cluster. +// It tries two approaches: +// 1. First attempts to collect using --load-cluster-specs (automatic spec discovery) +// 2. If that fails, tries with the explicit cluster support bundle spec at /automation/troubleshoot/cluster-support-bundle.yaml +// +// The collected support bundle is downloaded from the VM and returned as a Dagger File +// that can be exported as an artifact. +// +// Example: +// +// dagger call with-one-password --service-account=env:OP_SERVICE_ACCOUNT_TOKEN \ +// with-cmx-vm --vm-id=8a2a66ef \ +// collect-cluster-support-bundle \ +// export --path=./support-bundle.tar.gz +func (i *CmxInstance) CollectClusterSupportBundle(ctx context.Context) (*dagger.File, error) { + bundlePath := "/tmp/cluster-support-bundle.tar.gz" + + // Try collecting support bundle with --load-cluster-specs first + cmd1 := fmt.Sprintf("kubectl support-bundle --output %s --interactive=false --load-cluster-specs", bundlePath) + _, err := i.Command(cmd1).Stdout(ctx) + + // If first attempt failed, try with explicit spec path + if err != nil { + fmt.Printf("First support bundle attempt failed, trying with explicit spec: %v\n", err) + cmd2 := fmt.Sprintf("kubectl support-bundle --output %s --interactive=false --load-cluster-specs /automation/troubleshoot/cluster-support-bundle.yaml", bundlePath) + stdout, err := i.Command(cmd2).Stdout(ctx) + if err != nil { + return nil, fmt.Errorf("failed to collect cluster support bundle (both attempts): %w\nOutput: %s", err, stdout) + } + } + + fmt.Printf("Support bundle collected successfully at %s\n", bundlePath) + + // Download the support bundle from the VM + file, err := i.DownloadFile(ctx, bundlePath) + if err != nil { + return nil, fmt.Errorf("download support bundle: %w", err) + } + + return file, nil +} + +// CollectHostSupportBundle collects a host support bundle from the VM. +// +// This method collects diagnostic information about the host system. +// It tries two approaches: +// 1. First attempts to collect using kubectl-support_bundle with the host spec +// 2. If that fails, falls back to collecting installer logs from /var/log/embedded-cluster +// +// The collected support bundle is downloaded from the VM and returned as a Dagger File +// that can be exported as an artifact. +// +// Example: +// +// dagger call with-one-password --service-account=env:OP_SERVICE_ACCOUNT_TOKEN \ +// with-cmx-vm --vm-id=8a2a66ef \ +// collect-host-support-bundle \ +// export --path=./host-support-bundle.tar.gz +func (i *CmxInstance) CollectHostSupportBundle(ctx context.Context) (*dagger.File, error) { + bundlePath := "/tmp/host-support-bundle.tar.gz" + + // Try collecting host support bundle with kubectl-support_bundle first + cmd1 := fmt.Sprintf("%s/bin/kubectl-support_bundle --output %s --interactive=false %s/support/host-support-bundle.yaml", DataDir, bundlePath, DataDir) + _, err := i.Command(cmd1).Stdout(ctx) + + // If first attempt failed, try collecting installer logs as fallback + if err != nil { + fmt.Printf("Host support bundle attempt failed, trying to collect installer logs: %v\n", err) + cmd2 := fmt.Sprintf("tar -czf %s -C / var/log/embedded-cluster 2>/dev/null || tar -czf %s --files-from=/dev/null", bundlePath, bundlePath) + stdout, err := i.Command(cmd2).Stdout(ctx) + if err != nil { + return nil, fmt.Errorf("failed to collect host support bundle and installer logs (both attempts): %w\nOutput: %s", err, stdout) + } + fmt.Printf("Installer logs collected as fallback\n") + } else { + fmt.Printf("Host support bundle collected successfully at %s\n", bundlePath) + } + + // Download the support bundle from the VM + file, err := i.DownloadFile(ctx, bundlePath) + if err != nil { + return nil, fmt.Errorf("download host support bundle: %w", err) + } + + return file, nil +} diff --git a/dagger/deps.go b/dagger/deps.go new file mode 100644 index 0000000000..07a9732864 --- /dev/null +++ b/dagger/deps.go @@ -0,0 +1,61 @@ +package main + +import ( + "dagger/embedded-cluster/internal/dagger" + "fmt" +) + +// BuildFio builds the fio binary +// +// This can be called standalone or used internally by the build process. +// +// Example: +// +// dagger call build-fio --version=3.41 --arch=amd64 export --path=./fio +func (m *EmbeddedCluster) BuildFio( + // FIO version to build + version string, + // Architecture to build for (amd64 or arm64) + // +default="amd64" + arch string, +) *dagger.File { + // Map arch to Dagger platform + var platform dagger.Platform + switch arch { + case "amd64": + platform = "linux/amd64" + case "arm64": + platform = "linux/arm64" + default: + platform = "linux/amd64" + } + + // Build stage - compile fio from source + buildContainer := ubuntuUtilsContainer(dagger.ContainerOpts{Platform: platform}). + WithExec([]string{"mkdir", "-p", "/fio"}). + WithWorkdir("/fio"). + WithExec([]string{"curl", "-fsSL", "-o", "fio.tar.gz", fmt.Sprintf("https://api.github.com/repos/axboe/fio/tarball/fio-%s", version)}). + WithExec([]string{"tar", "-xzf", "fio.tar.gz", "--strip-components=1"}). + WithExec([]string{"./configure", "--build-static", "--disable-native"}). + WithExec([]string{"sh", "-c", "make -j$(nproc)"}) + + // Extract the binary + return buildContainer.File("/fio/fio") +} + +// ubuntuUtilsContainer returns a container with the necessary tools for building. +func ubuntuUtilsContainer(opts ...dagger.ContainerOpts) *dagger.Container { + return dag.Container(opts...). + From("ubuntu:24.04"). + WithEnvVariable("TZ", "Etc/UTC"). + WithEnvVariable("DEBIAN_FRONTEND", "noninteractive"). + WithExec([]string{"apt-get", "update"}). + WithExec([]string{"apt-get", "install", "-y", + "build-essential", "cmake", "curl", "gettext", "git", "gzip", "jq", "libstdc++6", "make", "pkg-config", "tar", "unzip", + }). + // Set the working directory to /workspace + WithWorkdir("/workspace"). + // Configure Git to allow unsafe directories + WithExec([]string{"git", "config", "--global", "--add", "safe.directory", "/workspace"}) + +} diff --git a/dagger/e2e.go b/dagger/e2e.go index 3b3bee6be5..204faf5ad1 100644 --- a/dagger/e2e.go +++ b/dagger/e2e.go @@ -2,125 +2,278 @@ package main import ( "context" + _ "embed" + "encoding/json" "fmt" "dagger/embedded-cluster/internal/dagger" + + "go.yaml.in/yaml/v3" ) -// Provisions a new CMX VM for E2E testing. +//go:embed assets/config-values.yaml +var configValuesFileContent string + +// E2eRunHeadless runs a headless installation E2E test. +// +// This method provisions a fresh CMX VM, performs a headless installation via CLI, +// validates the installation, and cleans up the VM afterward. It supports both +// online and airgap installation scenarios. // -// This creates a fresh VM with the specified configuration and waits for it to be ready. -// The VM is automatically configured with SSH access and networking. +// The test: +// - Provisions an Ubuntu 22.04 VM with r1.large instance type (8GB RAM, 4 CPUs) +// - For airgap: applies network policy to block internet access +// - Downloads and installs embedded-cluster via CLI with license +// - Validates installation success using Kubernetes client +// - Returns comprehensive test results including validation details // -// Example: +// Example (online): // // dagger call with-one-password --service-account=env:OP_SERVICE_ACCOUNT_TOKEN \ -// test-provision-vm --name="my-test-vm" -func (m *EmbeddedCluster) TestProvisionVM( +// e-2-e-run-headless --scenario=online --app-version=appver-dev-xpXCTO --kube-version=1.33 --license-file=./local-dev/ethan-dev-license.yaml \ +// export --path=./e2e-results-online +// +// Example (airgap): +// +// dagger call with-one-password --service-account=env:OP_SERVICE_ACCOUNT_TOKEN \ +// e-2-e-run-headless --scenario=airgap --app-version=appver-dev-xpXCTO --kube-version=1.33 --license-file=./local-dev/ethan-dev-license.yaml \ +// export --path=./e2e-results-airgap +func (m *EmbeddedCluster) E2eRunHeadless( ctx context.Context, - // Name for the VM - // +default="ec-e2e-test" - name string, - // OS distribution - // +default="ubuntu" - distribution string, - // Distribution version - // +default="22.04" - version string, - // Instance type - // +default="r1.medium" - instanceType string, - // Disk size in GB - // +default=50 - diskSize int, - // How long to wait for VM to be ready - // +default="10m" - wait string, - // TTL for the VM - // +default="2h" - ttl string, - // SSH user - // +default="ec-e2e-test" - sshUser string, -) (*CMXInstance, error) { - // Get CMX API token and SSH key from 1Password - cmxToken := m.mustResolveSecret(nil, "CMX_REPLICATED_API_TOKEN") - sshKey := m.mustResolveSecret(nil, "CMX_SSH_PRIVATE_KEY") - - // Create VM using Replicated Dagger module - vms, err := dag. - Replicated(cmxToken). - VMCreate( - ctx, - dagger.ReplicatedVMCreateOpts{ - Name: name, - Wait: wait, - TTL: ttl, - Distribution: distribution, - Version: version, - Count: 1, - Disk: diskSize, - InstanceType: instanceType, - }, - ) - if err != nil { - return nil, fmt.Errorf("create vm: %w", err) + // Scenario (online or airgap) + scenario string, + // App version to install + appVersion string, + // Expected Kubernetes version (e.g., "1.31") + kubeVersion string, + // License file + licenseFile *dagger.File, + // CMX API token + // +optional + cmxToken *dagger.Secret, + // SSH key + // +optional + sshKey *dagger.Secret, + // Skip cleanup + // +default=false + skipCleanup bool, + // Skip support bundle collection + // +default=false + skipSupportBundleCollection bool, +) (resultsDir *dagger.Directory) { + mode := "headless" + + // Initialize test result that will be built up throughout the function + testResult := &TestResult{ + Scenario: scenario, + Mode: mode, + Success: false, } - // Get the first VM - if len(vms) == 0 { - return nil, fmt.Errorf("no VMs created") + // Initialize results directory + resultsDir = dag.Directory() + + // Log test start + fmt.Printf("Starting E2E test: scenario=%s mode=%s app-version=%s kube-version=%s\n", + scenario, mode, appVersion, kubeVersion) + + // Provision a fresh CMX VM for testing + fmt.Printf("Provisioning CMX VM for %s %s test...\n", scenario, mode) + vm, provisionErr := m.ProvisionCmxVm( + ctx, + fmt.Sprintf("ec-e2e-%s-%s", scenario, mode), + "ubuntu", + "22.04", + "r1.large", // 8GB RAM, 4 CPUs - enough for single-node cluster + 50, // 50GB disk + "10m", // 10 minute wait for VM to be ready + "2h", // 2 hour TTL + cmxToken, + sshKey, + ) + if provisionErr != nil { + testResult.Error = fmt.Sprintf("failed to provision VM: %v", provisionErr) + resultJSON, _ := json.MarshalIndent(testResult, "", " ") + resultsDir = resultsDir.WithNewFile("result.json", string(resultJSON)) + return } - vm := vms[0] - // Get VM details - vmID, err := vm.ItemID(ctx) - if err != nil { - return nil, fmt.Errorf("get vm id: %w", err) + fmt.Printf("Provisioned VM: %s\n", vm.VmID) + testResult.VMID = vm.VmID + + // Defer function to collect support bundle and cleanup VM + defer func() { + // Collect support bundle before cleanup + if vm != nil && !skipSupportBundleCollection { + fmt.Printf("Collecting support bundle from VM %s...\n", vm.VmID) + resultsDir = collectSupportBundles(ctx, vm, resultsDir) + } + + // Marshal final test result to JSON + resultJSON, marshalErr := json.MarshalIndent(testResult, "", " ") + if marshalErr != nil { + fmt.Printf("Warning: failed to marshal test result: %v\n", marshalErr) + return + } + resultsDir = resultsDir.WithNewFile("result.json", string(resultJSON)) + + // Cleanup VM + if skipCleanup { + return + } + fmt.Printf("Cleaning up CMX VM %s...\n", vm.VmID) + if _, cleanupErr := vm.Cleanup(ctx); cleanupErr != nil { + fmt.Printf("Warning: failed to cleanup VM %s: %v\n", vm.VmID, cleanupErr) + } + }() + + // Download and prepare embedded-cluster release + if prepareErr := vm.PrepareRelease(ctx, scenario, appVersion, licenseFile); prepareErr != nil { + testResult.Error = fmt.Sprintf("failed to prepare release: %v", prepareErr) + return } - vmName, err := vm.Name(ctx) - if err != nil { - return nil, fmt.Errorf("get vm name: %w", err) + // For airgap scenarios, apply network policy to block internet access + if scenario == "airgap" { + fmt.Printf("Applying airgap network policy on VM %s...\n", vm.VmID) + if airgapErr := vm.ApplyAirgapNetworkPolicy(ctx); airgapErr != nil { + testResult.Error = fmt.Sprintf("failed to apply airgap network policy: %v", airgapErr) + return + } } - networkID, err := vm.NetworkID(ctx) - if err != nil { - return nil, fmt.Errorf("get network id: %w", err) + // Run headless installation + fmt.Printf("Running headless installation on VM %s...\n", vm.VmID) + installResult, installErr := vm.InstallHeadless( + ctx, + scenario, + appVersion, + licenseFile, + dag.File("config-values.yaml", configValuesFileContent), + ) + if installErr != nil { + testResult.Error = fmt.Sprintf("installation failed: %v", installErr) + return } - sshEndpoint, err := vm.DirectSshendpoint(ctx) - if err != nil { - return nil, fmt.Errorf("get ssh endpoint: %w", err) + if !installResult.Success { + testResult.Error = "installation reported failure" + return } - directSSHPort, err := vm.DirectSshport(ctx) - if err != nil { - return nil, fmt.Errorf("get direct ssh port: %w", err) + // Validate installation + fmt.Printf("Validating installation on VM %s...\n", vm.VmID) + validationResult := vm.Validate( + ctx, + scenario, + kubeVersion, + appVersion, + ) + + // Update final test result + testResult.Success = validationResult.Success + testResult.ValidationResults = validationResult + if !validationResult.Success { + testResult.Error = "validation checks failed" } - instance := &CMXInstance{ - VmID: string(vmID), - Name: vmName, - NetworkID: networkID, - SSHEndpoint: sshEndpoint, - SSHPort: directSSHPort, - SSHUser: sshUser, - SSHKey: sshKey, - CMXToken: cmxToken, + // Print formatted test results + printResults(validationResult, scenario, mode, appVersion, kubeVersion, vm) + + return +} + +func collectSupportBundles(ctx context.Context, vm *CmxInstance, resultsDir *dagger.Directory) *dagger.Directory { + fmt.Printf("Collecting support bundle from VM %s...\n", vm.VmID) + supportBundle, bundleErr := vm.CollectClusterSupportBundle(ctx) + if bundleErr != nil { + fmt.Printf("Warning: failed to collect support bundle: %v\n", bundleErr) + resultsDir = resultsDir.WithNewFile("support-bundle-error.txt", fmt.Sprintf("Failed to collect support bundle: %v", bundleErr)) + } else { + resultsDir = resultsDir.WithFile("support-bundle.tar.gz", supportBundle) } - // Wait for SSH to be available - if err := instance.waitForSSH(ctx); err != nil { - return nil, fmt.Errorf("wait for ssh: %w", err) + hostSupportBundle, hostBundleErr := vm.CollectHostSupportBundle(ctx) + if hostBundleErr != nil { + fmt.Printf("Warning: failed to collect host support bundle: %v\n", hostBundleErr) + resultsDir = resultsDir.WithNewFile("host-support-bundle-error.txt", fmt.Sprintf("Failed to collect host support bundle: %v", hostBundleErr)) + } else { + resultsDir = resultsDir.WithFile("host-support-bundle.tar.gz", hostSupportBundle) } - // Discover private IP - privateIP, err := instance.discoverPrivateIP(ctx) - if err != nil { - return nil, fmt.Errorf("discover private ip: %w", err) + return resultsDir +} + +func printResults(validationResult *ValidationResult, scenario string, mode string, appVersion string, kubeVersion string, vm *CmxInstance) { + fmt.Printf("\n") + fmt.Printf("================================\n") + if validationResult.Success { + fmt.Printf("✓ E2E TEST PASSED\n") + } else { + fmt.Printf("✗ E2E TEST FAILED\n") + } + fmt.Printf("================================\n") + fmt.Printf("Scenario: %s\n", scenario) + fmt.Printf("Mode: %s\n", mode) + fmt.Printf("App Version: %s\n", appVersion) + fmt.Printf("Kube Version: %s\n", kubeVersion) + fmt.Printf("VM ID: %s\n", vm.VmID) + fmt.Printf("\n") + fmt.Printf("Validation Results:\n") + fmt.Printf(" Cluster Health: %s\n", formatCheckResult(validationResult.ClusterHealth)) + fmt.Printf(" Installation CRD: %s\n", formatCheckResult(validationResult.InstallationCRD)) + fmt.Printf(" App Deployment: %s\n", formatCheckResult(validationResult.AppDeployment)) + fmt.Printf(" Admin Console: %s\n", formatCheckResult(validationResult.AdminConsole)) + fmt.Printf(" Data Directories: %s\n", formatCheckResult(validationResult.DataDirectories)) + fmt.Printf(" Pods and Jobs: %s\n", formatCheckResult(validationResult.PodsAndJobs)) + + if !validationResult.Success { + fmt.Printf("\n") + fmt.Printf("Failed Checks:\n") + if !validationResult.ClusterHealth.Passed { + fmt.Printf(" • Cluster Health: %s\n", validationResult.ClusterHealth.ErrorMessage) + } + if !validationResult.InstallationCRD.Passed { + fmt.Printf(" • Installation CRD: %s\n", validationResult.InstallationCRD.ErrorMessage) + } + if !validationResult.AppDeployment.Passed { + fmt.Printf(" • App Deployment: %s\n", validationResult.AppDeployment.ErrorMessage) + } + if !validationResult.AdminConsole.Passed { + fmt.Printf(" • Admin Console: %s\n", validationResult.AdminConsole.ErrorMessage) + } + if !validationResult.DataDirectories.Passed { + fmt.Printf(" • Data Directories: %s\n", validationResult.DataDirectories.ErrorMessage) + } + if !validationResult.PodsAndJobs.Passed { + fmt.Printf(" • Pods and Jobs: %s\n", validationResult.PodsAndJobs.ErrorMessage) + } } - instance.PrivateIP = privateIP + fmt.Printf("================================\n\n") +} + +func formatCheckResult(check *CheckResult) string { + if check.Passed { + return "✓ PASSED" + } + return "✗ FAILED" +} - return instance, nil +func parseLicense(ctx context.Context, licenseFile *dagger.File) (contents string, licenseID string, channelID string, err error) { + contents, err = licenseFile.Contents(ctx) + if err != nil { + return + } + var license struct { + Spec struct { + LicenseID string `yaml:"licenseID"` + ChannelID string `yaml:"channelID"` + } `yaml:"spec"` + } + if err = yaml.Unmarshal([]byte(contents), &license); err != nil { + return + } + licenseID = license.Spec.LicenseID + channelID = license.Spec.ChannelID + return } diff --git a/dagger/e2etypes.go b/dagger/e2etypes.go new file mode 100644 index 0000000000..d27a2a00f3 --- /dev/null +++ b/dagger/e2etypes.go @@ -0,0 +1,99 @@ +package main + +import "fmt" + +// HeadlessConfig holds configuration for headless (CLI) installations. +type HeadlessConfig struct { + // Installation scenario (online, airgap) + Scenario string + // App version to install + AppVersion string + // License content + License string + // License ID for downloading + LicenseID string + // Path to config file (optional) + ConfigFile string +} + +// InstallResult contains information about a completed installation. +type InstallResult struct { + // Whether installation succeeded + Success bool + // Path to kubeconfig file + KubeconfigPath string + // Admin console URL (for browser-based installs) + AdminConsoleURL string + // UI port (for browser-based installs) + UIPort int + // Installation log output + InstallationLog string +} + +// ValidationResult contains the results of all validation checks. +type ValidationResult struct { + // Whether all validation checks passed + Success bool + // Kubernetes cluster health check result + ClusterHealth *CheckResult + // Installation CRD status check result + InstallationCRD *CheckResult + // Application deployment check result + AppDeployment *CheckResult + // Admin console components check result + AdminConsole *CheckResult + // Data directory configuration check result + DataDirectories *CheckResult + // Pod and job health check result + PodsAndJobs *CheckResult +} + +func (v *ValidationResult) String() string { + return fmt.Sprintf( + "ValidationResult{\n Success: %t\n ClusterHealth: %s\n InstallationCRD: %s\n AppDeployment: %s\n AdminConsole: %s\n DataDirectories: %s\n PodsAndJobs: %s\n}", + v.Success, + v.ClusterHealth.String(), + v.InstallationCRD.String(), + v.AppDeployment.String(), + v.AdminConsole.String(), + v.DataDirectories.String(), + v.PodsAndJobs.String(), + ) +} + +// CheckResult contains the result of a single validation check. +type CheckResult struct { + // Whether the check passed + Passed bool + // Error message if the check failed (empty if passed) + ErrorMessage string + // Additional context or details about the check + Details string +} + +func (c *CheckResult) String() string { + return fmt.Sprintf("CheckResult{Passed: %t, ErrorMessage: %q, Details: %q}", c.Passed, c.ErrorMessage, c.Details) +} + +// TestResult contains the result of an E2E test execution. +type TestResult struct { + // Test scenario (online, airgap) + Scenario string + // Installation mode (headless, browser-based) + Mode string + // Whether the test succeeded + Success bool + // Error message if test failed + Error string + // Test execution duration + Duration string + // VM ID used for the test (for cleanup or support bundle collection) + VMID string + // Validation results from the test + ValidationResults *ValidationResult +} + +func (t *TestResult) String() string { + return fmt.Sprintf("TestResult{Scenario: %s, Mode: %s, Success: %t, Error: %q, Duration: %s, VMID: %s, ValidationResults: %s}", + t.Scenario, t.Mode, t.Success, t.Error, t.Duration, t.VMID, t.ValidationResults) +} diff --git a/dagger/go.mod b/dagger/go.mod index ecc3a558bc..80d09dc628 100644 --- a/dagger/go.mod +++ b/dagger/go.mod @@ -16,6 +16,7 @@ require ( go.opentelemetry.io/otel/sdk/log v0.14.0 go.opentelemetry.io/otel/trace v1.38.0 go.opentelemetry.io/proto/otlp v1.8.0 + go.yaml.in/yaml/v3 v3.0.4 golang.org/x/sync v0.17.0 google.golang.org/grpc v1.76.0 ) diff --git a/dagger/go.sum b/dagger/go.sum index 063b231239..241eabbbcc 100644 --- a/dagger/go.sum +++ b/dagger/go.sum @@ -24,11 +24,17 @@ github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 h1:8Tjv8EJ+pM1xP8mK6egEbD1OgnVTyacbefKhmbLhIhU= github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2/go.mod h1:pkJQ2tZHJ0aFOVEEot6oZmaVEZcRme73eIFmhiVuRWs= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 h1:n661drycOFuPLCN3Uc8sB6B/s6Z4t2xvBgU1htSHuq8= github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3/go.mod h1:A0bzQcvG0E7Rwjx0REVgAGH58e96+X0MeOfepqsbeW4= github.com/sosodev/duration v1.3.1 h1:qtHBDMQ6lvMQsL15g4aopM4HEfOaYuhWBw3NPTtlqq4= @@ -75,6 +81,8 @@ go.opentelemetry.io/proto/otlp v1.8.0 h1:fRAZQDcAFHySxpJ1TwlA1cJ4tvcrw7nXl9xWWC8 go.opentelemetry.io/proto/otlp v1.8.0/go.mod h1:tIeYOeNBU4cvmPqpaji1P+KbB4Oloai8wN4rWzRrFF0= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.yaml.in/yaml/v3 v3.0.4 h1:tfq32ie2Jv2UxXFdLJdh3jXuOzWiL1fo0bu/FbuKpbc= +go.yaml.in/yaml/v3 v3.0.4/go.mod h1:DhzuOOF2ATzADvBadXxruRBLzYTpT36CKvDb3+aBEFg= golang.org/x/net v0.44.0 h1:evd8IRDyfNBMBTTY5XRF1vaZlD+EmWx6x8PkhR04H/I= golang.org/x/net v0.44.0/go.mod h1:ECOoLqd5U3Lhyeyo/QDCEVQ4sNgYsqvCZ722XogGieY= golang.org/x/sync v0.17.0 h1:l60nONMj9l5drqw6jlhIELNv9I0A4OFgRsG9k2oT9Ug= @@ -95,6 +103,8 @@ google.golang.org/protobuf v1.36.9 h1:w2gp2mA27hUeUzj9Ex9FBjsBm40zfaDtEWow293U7I google.golang.org/protobuf v1.36.9/go.mod h1:fuxRtAxBytpl4zzqUh6/eyUujkJdNiuEkXntxiD/uRU= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/dagger/internal/e2e/types.go b/dagger/internal/e2e/types.go deleted file mode 100644 index dc07699305..0000000000 --- a/dagger/internal/e2e/types.go +++ /dev/null @@ -1,133 +0,0 @@ -package e2e - -import ( - "time" -) - -// CMXConfig holds configuration for provisioning a CMX VM. -type CMXConfig struct { - // Name of the VM (defaults to "ec-test-suite") - Name string - // Distribution to use (e.g., "ubuntu") - Distribution string - // Version of the distribution (e.g., "22.04") - Version string - // Instance type (e.g., "t3.medium") - InstanceType string - // Disk size in GB - DiskSize int - // Number of VMs to create - Count int - // How long to wait for VM to be ready - Wait time.Duration - // TTL for the VM - TTL time.Duration - // Network ID (optional, creates new if empty) - NetworkID string -} - -// DefaultCMXConfig returns a CMXConfig with sensible defaults for E2E tests. -func DefaultCMXConfig() *CMXConfig { - return &CMXConfig{ - Name: "ec-e2e-test", - Distribution: "ubuntu", - Version: "22.04", - InstanceType: "t3.medium", - DiskSize: 50, - Count: 1, - Wait: 15 * time.Minute, - TTL: 2 * time.Hour, - } -} - -// CMXInstance represents a provisioned CMX VM instance. -type CMXInstance struct { - // VM ID from CMX - ID string - // VM name - Name string - // Network ID - NetworkID string - // Private IP address - PrivateIP string - // SSH endpoint for connecting to the VM - SSHEndpoint string - // Exposed ports (port -> hostname) - ExposedPorts map[string]string -} - -// TestResult represents the result of a test execution. -type TestResult struct { - // Name of the test scenario - Scenario string - // Installation mode (browser-based or headless) - Mode string - // Whether the test passed - Success bool - // Error message if test failed - Error string - // Duration of the test - Duration time.Duration - // Validation results - ValidationResults *ValidationResult -} - -// ValidationResult contains results from installation validation. -type ValidationResult struct { - // Overall success status - Success bool - // Individual check results - Checks map[string]CheckResult -} - -// CheckResult represents the result of a single validation check. -type CheckResult struct { - // Whether the check passed - Passed bool - // Error if check failed - Error error - // Additional details - Details string -} - -// InstallResult contains information about a completed installation. -type InstallResult struct { - // Whether installation succeeded - Success bool - // Path to kubeconfig file - KubeconfigPath string - // Admin console URL (for browser-based installs) - AdminConsoleURL string - // UI port (for browser-based installs) - UIPort int - // Installation log output - InstallationLog string -} - -// PlaywrightConfig holds configuration for Playwright-based UI tests. -type PlaywrightConfig struct { - // Installation scenario (online, airgap) - Scenario string - // App version to install - AppVersion string - // License content - License string - // License ID for downloading - LicenseID string - // Base URL for admin console - BaseURL string -} - -// HeadlessConfig holds configuration for headless (CLI) installations. -type HeadlessConfig struct { - // Installation scenario (online, airgap) - Scenario string - // App version to install - AppVersion string - // License content - License string - // License ID for downloading - LicenseID string - // Path to config file (optional) - ConfigFile string -} diff --git a/dagger/localartifactmirror.go b/dagger/localartifactmirror.go index 666e201a94..1ee3479f75 100644 --- a/dagger/localartifactmirror.go +++ b/dagger/localartifactmirror.go @@ -26,7 +26,7 @@ func (m *EmbeddedCluster) BuildLocalArtifactMirrorImage( arch string, ) *dagger.File { - tag := strings.Replace(ecVersion, "+", "-", -1) + tag := strings.ReplaceAll(ecVersion, "+", "-") image := fmt.Sprintf("%s:%s", repo, tag) apkoFile := m.apkoTemplateLocalArtifactMirror(src, ecVersion, kzerosMinorVersion) @@ -67,7 +67,7 @@ func (m *EmbeddedCluster) PublishLocalArtifactMirrorImage( arch string, ) (string, error) { - tag := strings.Replace(ecVersion, "+", "-", -1) + tag := strings.ReplaceAll(ecVersion, "+", "-") image := fmt.Sprintf("%s:%s", repo, tag) apkoFile := m.apkoTemplateLocalArtifactMirror(src, ecVersion, kzerosMinorVersion) @@ -110,8 +110,19 @@ func (m *EmbeddedCluster) BuildLocalArtifactMirrorPackage( melangeFile := m.melangeTemplateLocalArtifactMirror(src, ecVersion, kzerosMinorVersion) dir := dag.Directory(). - WithDirectory("local-artifact-mirror", src.Directory("local-artifact-mirror")). - WithDirectory("cmd", src.Directory("cmd")) + WithDirectory("local-artifact-mirror", + src.Directory("local-artifact-mirror"). + WithoutDirectory("bin"). + WithoutDirectory("build"). + WithoutDirectory("cache"), + ). + WithDirectory("cmd", + src.Directory("cmd"). + WithoutDirectory("installer/goods/bins"). + WithNewFile("installer/goods/bins/.placeholder", ".placeholder"). + WithoutDirectory("installer/goods/internal/bins"). + WithNewFile("installer/goods/internal/bins/.placeholder", ".placeholder"), + ) build := m.chainguard.melangeBuildGo( directoryWithCommonGoFiles(dir, src), diff --git a/dagger/main.go b/dagger/main.go index 9c8fab559a..86eba9e675 100644 --- a/dagger/main.go +++ b/dagger/main.go @@ -9,6 +9,8 @@ import ( const ( APKOImageVersion = "latest" MelangeImageVersion = "latest" + + NodeVersion = "22" ) type EmbeddedCluster struct { @@ -48,7 +50,13 @@ func directoryWithCommonGoFiles(dir *dagger.Directory, src *dagger.Directory) *d WithFile("go.sum", src.File("go.sum")). WithDirectory("pkg", src.Directory("pkg")). WithDirectory("pkg-new", src.Directory("pkg-new")). - WithDirectory("cmd/installer/goods", src.Directory("cmd/installer/goods")). + WithDirectory("cmd/installer/goods", + src.Directory("cmd/installer/goods"). + WithoutDirectory("bins"). + WithNewFile("bins/.placeholder", ".placeholder"). + WithoutDirectory("internal/bins"). + WithNewFile("internal/bins/.placeholder", ".placeholder"), + ). WithDirectory("api", src.Directory("api")). WithDirectory("kinds", src.Directory("kinds")). WithDirectory("utils", src.Directory("utils")) diff --git a/dagger/onepassword.go b/dagger/onepassword.go index 3a13ef2a56..96e219cf51 100644 --- a/dagger/onepassword.go +++ b/dagger/onepassword.go @@ -41,7 +41,7 @@ func (m *OnePassword) FindSecret( // Example: // // dagger call with-one-password --service-account=env:OP_SERVICE_ACCOUNT_TOKEN \ -// test-provision-vm +// provision-cmx-vm func (m *EmbeddedCluster) WithOnePassword( serviceAccount *dagger.Secret, // +default="Developer Automation" diff --git a/dagger/operator.go b/dagger/operator.go index 896a8acbd8..2114d14e2e 100644 --- a/dagger/operator.go +++ b/dagger/operator.go @@ -26,7 +26,7 @@ func (m *EmbeddedCluster) BuildOperatorImage( arch string, ) *dagger.File { - tag := strings.Replace(ecVersion, "+", "-", -1) + tag := strings.ReplaceAll(ecVersion, "+", "-") image := fmt.Sprintf("%s:%s", repo, tag) apkoFile := m.apkoTemplateOprator(src, ecVersion, kzerosMinorVersion) @@ -67,7 +67,7 @@ func (m *EmbeddedCluster) PublishOperatorImage( arch string, ) (string, error) { - tag := strings.Replace(ecVersion, "+", "-", -1) + tag := strings.ReplaceAll(ecVersion, "+", "-") image := fmt.Sprintf("%s:%s", repo, tag) apkoFile := m.apkoTemplateOprator(src, ecVersion, kzerosMinorVersion) @@ -111,7 +111,12 @@ func (m *EmbeddedCluster) BuildOperatorPackage( melangeFile := m.melangeTemplateOperator(src, ecVersion, kzerosMinorVersion) dir := dag.Directory(). - WithDirectory("operator", src.Directory("operator")) + WithDirectory("operator", + src.Directory("operator"). + WithoutDirectory("bin"). + WithoutDirectory("build"). + WithoutDirectory("cache"), + ) build := m.chainguard.melangeBuildGo( directoryWithCommonGoFiles(dir, src), diff --git a/dagger/validation.go b/dagger/validation.go new file mode 100644 index 0000000000..c85f4a271b --- /dev/null +++ b/dagger/validation.go @@ -0,0 +1,631 @@ +package main + +import ( + "context" + "fmt" + "strings" + "time" +) + +// Validate performs comprehensive installation validation using Kubernetes client. +// +// This method orchestrates all validation checks and returns a ValidationResult +// containing the results of each check. All checks are run regardless of individual +// failures to provide a complete picture of the installation state. +// +// Example: +// +// dagger call with-one-password --service-account=env:OP_SERVICE_ACCOUNT_TOKEN \ +// with-cmx-vm --vm-id 8a2a66ef \ +// validate --scenario=online --expected-kube-version=1.33 --expected-app-version=appver-dev-xpXCTO string +func (i *CmxInstance) Validate( + ctx context.Context, + // Scenario (online, airgap) + scenario string, + // Expected Kubernetes version (e.g., "1.31") + expectedKubeVersion string, + // Expected app version (e.g., "v1.0.0") + expectedAppVersion string, +) *ValidationResult { + validationResult := &ValidationResult{ + Success: true, + } + + airgap := scenario == "airgap" + + // Run validation checks in order and populate fields + validationResult.ClusterHealth = i.validateClusterHealth(ctx, expectedKubeVersion) + validationResult.InstallationCRD = i.validateInstallationCRD(ctx) + validationResult.AppDeployment = i.validateAppDeployment(ctx, expectedAppVersion, airgap) + validationResult.AdminConsole = i.validateAdminConsole(ctx) + validationResult.DataDirectories = i.validateDataDirectories(ctx) + validationResult.PodsAndJobs = i.validatePodsAndJobs(ctx) + + // Determine overall success + allChecks := []*CheckResult{ + validationResult.ClusterHealth, + validationResult.InstallationCRD, + validationResult.AppDeployment, + validationResult.AdminConsole, + validationResult.DataDirectories, + validationResult.PodsAndJobs, + } + + for _, check := range allChecks { + fmt.Println(check.String()) + + if check != nil && !check.Passed { + validationResult.Success = false + } + } + + return validationResult +} + +// validateClusterHealth validates Kubernetes cluster health. +// +// This check verifies: +// - All nodes are running the expected Kubernetes version +// - Kubelet version matches expected version on all nodes +// - All nodes are in Ready state (none in NotReady) +// +// Based on: e2e/scripts/common.sh::ensure_nodes_match_kube_version +func (i *CmxInstance) validateClusterHealth(ctx context.Context, expectedK8sVersion string) *CheckResult { + result := &CheckResult{Passed: true} + + // Check node versions match expected k8s version + stdout, err := i.Command(`kubectl get nodes -o jsonpath={.items[*].status.nodeInfo.kubeletVersion}`).Stdout(ctx) + if err != nil { + result.Passed = false + result.ErrorMessage = fmt.Sprintf("failed to get node versions: %v", err) + return result + } + + // Verify all nodes have the expected version + if !strings.Contains(stdout, expectedK8sVersion) { + result.Passed = false + result.ErrorMessage = fmt.Sprintf("node version mismatch: got %s, want %s", stdout, expectedK8sVersion) + result.Details = "Not all nodes are running the expected Kubernetes version" + return result + } + + // Check node readiness - works for both single-node and multi-node + stdout, err = i.Command(`kubectl get nodes --no-headers`).Stdout(ctx) + if err != nil { + result.Passed = false + result.ErrorMessage = fmt.Sprintf("failed to get nodes: %v", err) + return result + } + + // Check that no nodes are NotReady + if strings.Contains(stdout, "NotReady") { + result.Passed = false + result.ErrorMessage = fmt.Sprintf("one or more nodes are not ready: %s", stdout) + result.Details = "Found nodes in NotReady state" + return result + } + + // Verify all nodes contain "Ready" status + lines := strings.Split(strings.TrimSpace(stdout), "\n") + for _, line := range lines { + if line == "" { + continue + } + if !strings.Contains(line, "Ready") { + result.Passed = false + result.ErrorMessage = fmt.Sprintf("node not ready: %s", line) + result.Details = "Node does not have Ready status" + return result + } + } + + result.Details = fmt.Sprintf("All nodes running k8s %s and in Ready state", expectedK8sVersion) + return result +} + +// validateInstallationCRD validates the Installation CRD status. +// +// This check verifies: +// - Installation resource exists +// - Installation is in "Installed" state +// - Embedded-cluster operator successfully completed installation +// +// Based on: e2e/scripts/common.sh::ensure_installation_is_installed +func (i *CmxInstance) validateInstallationCRD(ctx context.Context) *CheckResult { + result := &CheckResult{Passed: true} + + // Check if Installation resource exists and is in Installed state + stdout, err := i.Command(`kubectl get installations --no-headers`).Stdout(ctx) + if err != nil { + result.Passed = false + result.ErrorMessage = fmt.Sprintf("failed to get installations: %v", err) + + // Get more details for debugging + details, _ := i.Command(`kubectl get installations`).Stdout(ctx) + describeOut, _ := i.Command(`kubectl describe installations`).Stdout(ctx) + result.Details = fmt.Sprintf("installations output:\n%s\n\ndescribe:\n%s", details, describeOut) + return result + } + + if !strings.Contains(stdout, "Installed") { + result.Passed = false + result.ErrorMessage = "installation is not in Installed state" + + // Gather debugging information + installations, _ := i.Command(`kubectl get installations`).Stdout(ctx) + describe, _ := i.Command(`kubectl describe installations`).Stdout(ctx) + charts, _ := i.Command(`kubectl get charts -A`).Stdout(ctx) + pods, _ := i.Command(`kubectl get pods -A`).Stdout(ctx) + + result.Details = fmt.Sprintf("installations:\n%s\n\ndescribe:\n%s\n\ncharts:\n%s\n\npods:\n%s", + installations, describe, charts, pods) + return result + } + + result.Details = "Installation resource exists and is in Installed state" + return result +} + +// validateAppDeployment validates the application deployment status. +// +// This check verifies: +// - Application's nginx pods are Running +// - Correct app version is deployed +// - No upgrade artifacts present (kube-state-metrics namespace, "second" app pods) +// +// Based on: e2e/scripts/common.sh::wait_for_nginx_pods, ensure_app_deployed, ensure_app_not_upgraded +func (i *CmxInstance) validateAppDeployment(ctx context.Context, expectedAppVersion string, airgap bool) *CheckResult { + result := &CheckResult{Passed: true} + + // Wait for nginx pods to be Running (with timeout) + nginxReady := false + timeout := time.After(1 * time.Minute) + ticker := time.NewTicker(5 * time.Second) + defer ticker.Stop() + +waitLoop: + for { + select { + case <-timeout: + result.Passed = false + result.ErrorMessage = "nginx pods did not appear within timeout" + + // Get debugging info + pods, _ := i.Command(fmt.Sprintf(`kubectl get pods -n %s`, AppNamespace)).Stdout(ctx) + kotsadmPods, _ := i.Command(fmt.Sprintf(`kubectl get pods -n %s`, AppNamespace)).Stdout(ctx) + logs, _ := i.Command(fmt.Sprintf(`kubectl logs -n %s -l app=kotsadm --tail=50`, AppNamespace)).Stdout(ctx) + + result.Details = fmt.Sprintf("app pods:\n%s\n\nkotsadm pods:\n%s\n\nkotsadm logs:\n%s", + pods, kotsadmPods, logs) + return result + + case <-ticker.C: + stdout, err := i.Command(fmt.Sprintf(`kubectl get pods -n %s --no-headers`, AppNamespace)).Stdout(ctx) + if err == nil && strings.Contains(stdout, "nginx") && strings.Contains(stdout, "Running") { + nginxReady = true + break waitLoop + } + } + } + + if !nginxReady { + result.Passed = false + result.ErrorMessage = "nginx pods not in Running state" + return result + } + + // Verify app version is deployed + if airgap { + // For airgap, use kotsadm API to check version + if err := i.ensureAppDeployedAirgap(ctx, expectedAppVersion); err != nil { + result.Passed = false + result.ErrorMessage = fmt.Sprintf("app version validation failed (airgap): %v", err) + return result + } + } else { + // For online, use kubectl kots + versions, err := i.Command(fmt.Sprintf(`kubectl kots get versions -n %s embedded-cluster-smoke-test-staging-app`, AppNamespace)).Stdout(ctx) + if err != nil { + result.Passed = false + result.ErrorMessage = fmt.Sprintf("failed to get app versions: %v", err) + result.Details = versions + return result + } + + // Check for expected version with "deployed" status + // Format: version number deployed + if !strings.Contains(versions, expectedAppVersion) || !strings.Contains(versions, "deployed") { + result.Passed = false + result.ErrorMessage = fmt.Sprintf("app version %s not deployed", expectedAppVersion) + result.Details = fmt.Sprintf("versions output:\n%s", versions) + return result + } + } + + // Ensure no upgrade artifacts present + // Check for kube-state-metrics namespace (should not exist for fresh install) + nsOutput, _ := i.Command(`kubectl get ns`).Stdout(ctx) + if strings.Contains(nsOutput, "kube-state-metrics") { + result.Passed = false + result.ErrorMessage = "found kube-state-metrics namespace (upgrade artifact)" + result.Details = fmt.Sprintf("namespaces:\n%s", nsOutput) + return result + } + + // Check for "second" app pods (should not exist for fresh install) + secondPods, _ := i.Command(fmt.Sprintf(`kubectl get pods -n %s -l app=second`, AppNamespace)).Stdout(ctx) + if strings.Contains(secondPods, "second") { + result.Passed = false + result.ErrorMessage = "found pods from app update (upgrade artifact)" + result.Details = fmt.Sprintf("second pods:\n%s", secondPods) + return result + } + + result.Details = fmt.Sprintf("App version %s deployed successfully, nginx pods running, no upgrade artifacts", expectedAppVersion) + return result +} + +// ensureAppDeployedAirgap checks app deployment status using kotsadm API (for airgap scenarios). +// +// Based on: e2e/scripts/common.sh::ensure_app_deployed_airgap +func (i *CmxInstance) ensureAppDeployedAirgap(ctx context.Context, expectedVersion string) error { + // Get kotsadm authstring + authStringCmd := fmt.Sprintf(`kubectl get secret -n %s kotsadm-authstring -o jsonpath={.data.kotsadm-authstring}`, AppNamespace) + authString64, err := i.Command(authStringCmd).Stdout(ctx) + if err != nil { + return fmt.Errorf("get authstring: %w", err) + } + + // Decode authstring (base64) + decodeCmd := fmt.Sprintf(`sh -c "echo '%s' | base64 -d"`, authString64) + authString, err := i.Command(decodeCmd).Stdout(ctx) + if err != nil { + return fmt.Errorf("decode authstring: %w", err) + } + + // Get kotsadm service IP + kotsadmIPCmd := fmt.Sprintf(`kubectl get svc -n %s kotsadm -o jsonpath={.spec.clusterIP}`, AppNamespace) + kotsadmIP, err := i.Command(kotsadmIPCmd).Stdout(ctx) + if err != nil { + return fmt.Errorf("get kotsadm IP: %w", err) + } + + // Get kotsadm service port + kotsadmPortCmd := fmt.Sprintf(`kubectl get svc -n %s kotsadm -o jsonpath={.spec.ports[?(@.name=="http")].port}`, AppNamespace) + kotsadmPort, err := i.Command(kotsadmPortCmd).Stdout(ctx) + if err != nil { + return fmt.Errorf("get kotsadm port: %w", err) + } + + // Query kotsadm API for versions + apiURL := fmt.Sprintf("http://%s:%s/api/v1/app/embedded-cluster-smoke-test-staging-app/versions?currentPage=0&pageSize=1", + strings.TrimSpace(kotsadmIP), strings.TrimSpace(kotsadmPort)) + + curlCmd := fmt.Sprintf(`curl -k -X GET %s -H "Authorization: %s"`, apiURL, strings.TrimSpace(authString)) + versions, err := i.Command(curlCmd).Stdout(ctx) + if err != nil { + return fmt.Errorf("query kotsadm API: %w", err) + } + + // Search for the version and that it is deployed + // Format: "versionLabel":"v1.0.0"..."status":"deployed" + // There should not be a '}' between the version and the status + versionPattern := fmt.Sprintf(`"versionLabel":"%s"`, expectedVersion) + if !strings.Contains(versions, versionPattern) { + return fmt.Errorf("version %s not found in API response: %s", expectedVersion, versions) + } + + // Check that the version is deployed + if !strings.Contains(versions, `"status":"deployed"`) { + return fmt.Errorf("version %s not deployed: %s", expectedVersion, versions) + } + + return nil +} + +// validateAdminConsole validates the admin console components. +// +// This check verifies: +// - kotsadm pods are healthy +// - kotsadm API is healthy (kubectl kots get apps works) +// - Admin console branding configmap exists with DR label +// +// Based on: e2e/scripts/check-installation-state.sh +func (i *CmxInstance) validateAdminConsole(ctx context.Context) *CheckResult { + result := &CheckResult{Passed: true} + + // Check kotsadm pods are running + kotsadmPods, err := i.Command(fmt.Sprintf(`kubectl get pods -n %s -l app=kotsadm --no-headers`, AppNamespace)).Stdout(ctx) + if err != nil { + result.Passed = false + result.ErrorMessage = fmt.Sprintf("failed to get kotsadm pods: %v", err) + return result + } + + if !strings.Contains(kotsadmPods, "Running") { + result.Passed = false + result.ErrorMessage = "kotsadm pods are not running" + result.Details = fmt.Sprintf("kotsadm pods:\n%s", kotsadmPods) + return result + } + + // Check kubectl kots command works + _, err = i.Command(fmt.Sprintf(`kubectl kots get apps -n %s`, AppNamespace)).Stdout(ctx) + if err != nil { + result.Passed = false + result.ErrorMessage = fmt.Sprintf("kubectl kots get apps failed: %v", err) + result.Details = "kotsadm API is not healthy" + return result + } + + // Check admin console branding configmap has DR label + cmCheck, err := i.Command(fmt.Sprintf(`kubectl get cm -n %s kotsadm-application-metadata --show-labels`, AppNamespace)).Stdout(ctx) + if err != nil { + result.Passed = false + result.ErrorMessage = fmt.Sprintf("failed to get kotsadm-application-metadata configmap: %v", err) + return result + } + + if !strings.Contains(cmCheck, "replicated.com/disaster-recovery=infra") { + result.Passed = false + result.ErrorMessage = "kotsadm-application-metadata configmap missing DR label" + + // Get full configmap details + cmDetails, _ := i.Command(fmt.Sprintf(`kubectl get cm -n %s kotsadm-application-metadata -o yaml`, AppNamespace)).Stdout(ctx) + result.Details = fmt.Sprintf("configmap:\n%s", cmDetails) + return result + } + + result.Details = "Admin console components healthy, kotsadm API responding, branding configmap has DR label" + return result +} + +// validateDataDirectories validates data directory configuration. +// +// This check verifies: +// - K0s data directory is configured correctly +// - OpenEBS data directory is configured correctly +// - Velero pod volume path is configured correctly +// - All components use expected base directory +// +// Based on: e2e/scripts/common.sh::validate_data_dirs +func (i *CmxInstance) validateDataDirectories(ctx context.Context) *CheckResult { + result := &CheckResult{Passed: true} + expectedBaseDir := DataDir + expectedK0sDataDir := fmt.Sprintf("%s/k0s", DataDir) + expectedOpenEBSDataDir := fmt.Sprintf("%s/openebs-local", DataDir) + + var errors []string + + // Validate OpenEBS data directory + openebsChart, err := i.Command(`kubectl get charts -n kube-system k0s-addon-chart-openebs -o yaml`).Stdout(ctx) + if err == nil { + // Chart exists, validate basePath + if !strings.Contains(openebsChart, fmt.Sprintf("basePath: %s", expectedOpenEBSDataDir)) { + errors = append(errors, fmt.Sprintf("OpenEBS basePath not set to %s", expectedOpenEBSDataDir)) + + // Get specific section for details + grepCmd := `sh -c "kubectl get charts -n kube-system k0s-addon-chart-openebs -o yaml | grep -v apiVersion | grep 'basePath:' -A5 -B5"` + section, _ := i.Command(grepCmd).Stdout(ctx) + result.Details += fmt.Sprintf("\nOpenEBS chart basePath section:\n%s", section) + } + } + + // Validate SeaweedFS data directory (if present) + seaweedfsChart, err := i.Command(`kubectl get charts -n kube-system k0s-addon-chart-seaweedfs -o yaml`).Stdout(ctx) + if err == nil { + // Chart exists, validate hostPathPrefix + expectedPattern := fmt.Sprintf("%s/seaweedfs/", expectedBaseDir) + if !strings.Contains(seaweedfsChart, expectedPattern) { + errors = append(errors, fmt.Sprintf("SeaweedFS hostPathPrefix not under %s", expectedPattern)) + + // Get specific section for details + grepCmd := `sh -c "kubectl get charts -n kube-system k0s-addon-chart-seaweedfs -o yaml | grep -v apiVersion | grep -m 1 'hostPathPrefix:' -A5 -B5"` + section, _ := i.Command(grepCmd).Stdout(ctx) + result.Details += fmt.Sprintf("\nSeaweedFS chart hostPathPrefix section:\n%s", section) + } + } + + // Validate Velero pod volume path + veleroChart, err := i.Command(`kubectl get charts -n kube-system k0s-addon-chart-velero -o yaml`).Stdout(ctx) + if err == nil { + // Chart exists, validate podVolumePath + expectedVeleroPath := fmt.Sprintf("%s/kubelet/pods", expectedK0sDataDir) + if !strings.Contains(veleroChart, fmt.Sprintf("podVolumePath: %s", expectedVeleroPath)) { + errors = append(errors, fmt.Sprintf("Velero podVolumePath not set to %s", expectedVeleroPath)) + + // Get specific section for details + grepCmd := `sh -c "kubectl get charts -n kube-system k0s-addon-chart-velero -o yaml | grep -v apiVersion | grep 'podVolumePath:' -A5 -B5"` + section, _ := i.Command(grepCmd).Stdout(ctx) + result.Details += fmt.Sprintf("\nVelero chart podVolumePath section:\n%s", section) + } + } + + if len(errors) > 0 { + result.Passed = false + result.ErrorMessage = strings.Join(errors, "; ") + return result + } + + result.Details = fmt.Sprintf("Data directories configured correctly (base: %s, k0s: %s, openebs: %s)", + expectedBaseDir, expectedK0sDataDir, expectedOpenEBSDataDir) + return result +} + +// validatePodsAndJobs validates pod and job health. +// +// This check verifies: +// - All non-Job pods are in Running/Completed/Succeeded state +// - All Running pods have ready containers +// - All Jobs have completed successfully +// +// Based on: e2e/scripts/common.sh::validate_all_pods_healthy +func (i *CmxInstance) validatePodsAndJobs(ctx context.Context) *CheckResult { + result := &CheckResult{Passed: true} + timeout := 5 * time.Minute + startTime := time.Now() + + for { + elapsed := time.Since(startTime) + if elapsed >= timeout { + result.Passed = false + result.ErrorMessage = "timed out waiting for pods and jobs to be healthy after 5 minutes" + + // Gather failure details + nonJobCheck := i.validateNonJobPodsHealthy(ctx) + jobCheck := i.validateJobsCompleted(ctx) + + result.Details = fmt.Sprintf("Non-Job pods: %s\nJobs: %s", + nonJobCheck.ErrorMessage, jobCheck.ErrorMessage) + return result + } + + // Check if both validations pass + podsHealthy := i.validateNonJobPodsHealthy(ctx) + jobsHealthy := i.validateJobsCompleted(ctx) + + if podsHealthy.Passed && jobsHealthy.Passed { + result.Details = "All pods and jobs are healthy" + return result + } + + // Wait before retrying + time.Sleep(10 * time.Second) + } +} + +// validateNonJobPodsHealthy checks that all non-Job pods are healthy. +// +// Based on: e2e/scripts/common.sh::validate_non_job_pods_healthy +func (i *CmxInstance) validateNonJobPodsHealthy(ctx context.Context) *CheckResult { + result := &CheckResult{Passed: true} + + // Get all pods with custom columns + podsCommand := `kubectl get pods -A --no-headers -o custom-columns=NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase,OWNER:.metadata.ownerReferences[0].kind` + podsOutput, err := i.Command(podsCommand).Stdout(ctx) + if err != nil { + result.Passed = false + result.ErrorMessage = fmt.Sprintf("failed to get pods: %v", err) + return result + } + + // Check for unhealthy non-Job pods + var unhealthyPods []string + lines := strings.Split(strings.TrimSpace(podsOutput), "\n") + for _, line := range lines { + if line == "" { + continue + } + fields := strings.Fields(line) + if len(fields) < 4 { + continue + } + + namespace, name, status, owner := fields[0], fields[1], fields[2], fields[3] + if owner == "Job" { + continue // Skip Job pods + } + + // Check if pod is in acceptable state + if status != "Running" && status != "Completed" && status != "Succeeded" { + unhealthyPods = append(unhealthyPods, fmt.Sprintf("%s/%s (%s)", namespace, name, status)) + } + } + + if len(unhealthyPods) > 0 { + result.Passed = false + result.ErrorMessage = fmt.Sprintf("found non-Job pods in unhealthy state: %s", strings.Join(unhealthyPods, ", ")) + return result + } + + // Check container readiness for Running pods + readyCommand := `kubectl get pods -A --no-headers -o custom-columns=NAMESPACE:.metadata.namespace,NAME:.metadata.name,STATUS:.status.phase,READY:.status.containerStatuses[*].ready,OWNER:.metadata.ownerReferences[0].kind` + readyOutput, err := i.Command(readyCommand).Stdout(ctx) + if err != nil { + result.Passed = false + result.ErrorMessage = fmt.Sprintf("failed to check pod readiness: %v", err) + return result + } + + var unreadyPods []string + lines = strings.Split(strings.TrimSpace(readyOutput), "\n") + for _, line := range lines { + if line == "" { + continue + } + fields := strings.Fields(line) + if len(fields) < 5 { + continue + } + + namespace, name, status, ready, owner := fields[0], fields[1], fields[2], fields[3], fields[4] + if owner == "Job" || status != "Running" { + continue // Skip Job pods and non-Running pods + } + + // Check if all containers are ready (should be "true" for all) + if ready == "" || !strings.Contains(ready, "true") { + unreadyPods = append(unreadyPods, fmt.Sprintf("%s/%s (not ready)", namespace, name)) + } + } + + if len(unreadyPods) > 0 { + result.Passed = false + result.ErrorMessage = fmt.Sprintf("found Running pods that are not ready: %s", strings.Join(unreadyPods, ", ")) + return result + } + + result.Details = "All non-Job pods are healthy" + return result +} + +// validateJobsCompleted checks that all Jobs have completed successfully. +// +// Based on: e2e/scripts/common.sh::validate_jobs_completed +func (i *CmxInstance) validateJobsCompleted(ctx context.Context) *CheckResult { + result := &CheckResult{Passed: true} + + // Get all Jobs with completions status + jobsCommand := `kubectl get jobs -A --no-headers -o custom-columns=NAMESPACE:.metadata.namespace,NAME:.metadata.name,COMPLETIONS:.spec.completions,SUCCESSFUL:.status.succeeded` + jobsOutput, err := i.Command(jobsCommand).Stdout(ctx) + if err != nil { + result.Passed = false + result.ErrorMessage = fmt.Sprintf("failed to get jobs: %v", err) + return result + } + + // Check that all Jobs have succeeded + var incompleteJobs []string + lines := strings.Split(strings.TrimSpace(jobsOutput), "\n") + for _, line := range lines { + if line == "" { + continue + } + fields := strings.Fields(line) + if len(fields) < 4 { + continue + } + + namespace, name, completions, succeeded := fields[0], fields[1], fields[2], fields[3] + + // Check if succeeded count matches completions count + if succeeded != completions { + incompleteJobs = append(incompleteJobs, + fmt.Sprintf("%s/%s (succeeded: %s/%s)", namespace, name, succeeded, completions)) + } + } + + if len(incompleteJobs) > 0 { + result.Passed = false + result.ErrorMessage = fmt.Sprintf("found Jobs that have not completed successfully: %s", + strings.Join(incompleteJobs, ", ")) + + // Get job details for debugging + allJobs, _ := i.Command(`kubectl get jobs -A`).Stdout(ctx) + result.Details = fmt.Sprintf("Job details:\n%s", allJobs) + return result + } + + result.Details = "All Jobs have completed successfully" + return result +} diff --git a/dev/patches/operator-up.yaml b/dev/patches/operator-up.yaml index 621fca49c8..c638d11bf0 100644 --- a/dev/patches/operator-up.yaml +++ b/dev/patches/operator-up.yaml @@ -16,9 +16,9 @@ spec: mountPath: /replicatedhq/embedded-cluster # parent of workdir in the container env: - name: GOCACHE - value: /replicatedhq/embedded-cluster/dev/.gocache # from dev volume mount + value: /replicatedhq/embedded-cluster/dev/build/.gocache # from dev volume mount - name: GOMODCACHE - value: /replicatedhq/embedded-cluster/dev/.gomodcache # from dev volume mount + value: /replicatedhq/embedded-cluster/dev/build/.gomodcache # from dev volume mount livenessProbe: ~ readinessProbe: ~ resources: ~ diff --git a/dev/scripts/up.sh b/dev/scripts/up.sh index 54fe07759b..9c7e693a33 100755 --- a/dev/scripts/up.sh +++ b/dev/scripts/up.sh @@ -13,7 +13,7 @@ if [ -z "$component" ]; then fi # Ensure dev go cache / go mod cache directories exists -mkdir -p dev/.gocache dev/.gomodcache +mkdir -p dev/build/.gocache dev/build/.gomodcache # Build and load the image into the embedded cluster ec_build_and_load "$component" diff --git a/e2e/kots-release-upgrade-v3/cluster-config.yaml b/e2e/kots-release-upgrade-v3/cluster-config.yaml index c0976d6d48..11f47bb10c 100644 --- a/e2e/kots-release-upgrade-v3/cluster-config.yaml +++ b/e2e/kots-release-upgrade-v3/cluster-config.yaml @@ -4,7 +4,6 @@ metadata: name: "testconfig" spec: version: "__version_string__" - v2Enabled: __v2_enabled__ binaryOverrideUrl: "__release_url__" metadataOverrideUrl: "__metadata_url__" domains: diff --git a/e2e/kots-release-upgrade/cluster-config.yaml b/e2e/kots-release-upgrade/cluster-config.yaml index c0976d6d48..11f47bb10c 100644 --- a/e2e/kots-release-upgrade/cluster-config.yaml +++ b/e2e/kots-release-upgrade/cluster-config.yaml @@ -4,7 +4,6 @@ metadata: name: "testconfig" spec: version: "__version_string__" - v2Enabled: __v2_enabled__ binaryOverrideUrl: "__release_url__" metadataOverrideUrl: "__metadata_url__" domains: diff --git a/e2e/licenses/ci-v3.yaml b/e2e/licenses/ci-v3.yaml new file mode 100644 index 0000000000..db55f3c91d --- /dev/null +++ b/e2e/licenses/ci-v3.yaml @@ -0,0 +1,38 @@ +apiVersion: kots.io/v1beta1 +kind: License +metadata: + name: civ3 +spec: + appSlug: embedded-cluster-smoke-test-staging-app + channelID: 36LoGcOOLvEPFXQXCUsFub28Abi + channelName: CI-V3 + channels: + - channelID: 36LoGcOOLvEPFXQXCUsFub28Abi + channelName: CI-V3 + channelSlug: ci-v3 + endpoint: https://ec-e2e-replicated-app.testcluster.net + isDefault: true + replicatedProxyDomain: ec-e2e-proxy.testcluster.net + customerEmail: ci-v3@replicated.com + customerName: CI V3 + endpoint: https://ec-e2e-replicated-app.testcluster.net + entitlements: + expires_at: + description: License Expiration + signature: + v1: IEnM6uBp3L/yTlvOLV/lO3Wf98hk0Fb3fSoOOSVSfIERH2OWuvEub6IEkdGd3MVTv2gRZ15UoK9ZKH4KzJ9XRbtXtVlvcI/bQtxw1PUS7cr462ntVcbEyOKXG4nRFYlsahhzESRcDqyfR3+Cjw4I62a+TZrpPz5RFOzQ6Hqf9kBjd6Tuk7bjkJ/p+D2QsOMgH5zFSd4p878xRzE1QztrpfwFUSeBy4l/QSh6IFwGUnTqMgQUze/8rNU/i05h76ZOU9PuClK685wdGr0oqZMpvmoxiUR68dSM5a+x8cMd75GlWUBbTcnjusoLy3EMM/W7tdHD9X75741g6FGTgGdp/g== + title: Expiration + value: "" + valueType: String + isAirgapSupported: true + isDisasterRecoverySupported: true + isEmbeddedClusterDownloadEnabled: true + isEmbeddedClusterMultiNodeEnabled: true + isKotsInstallEnabled: true + isNewKotsUiEnabled: true + isSupportBundleUploadSupported: true + licenseID: 36LpK1WzrkAu2IRVGYwPyJ5eJNI + licenseSequence: 1 + licenseType: dev + replicatedProxyDomain: ec-e2e-proxy.testcluster.net + signature: eyJsaWNlbnNlRGF0YSI6ImV5SmhjR2xXWlhKemFXOXVJam9pYTI5MGN5NXBieTkyTVdKbGRHRXhJaXdpYTJsdVpDSTZJa3hwWTJWdWMyVWlMQ0p0WlhSaFpHRjBZU0k2ZXlKdVlXMWxJam9pWTJsMk15SjlMQ0p6Y0dWaklqcDdJbXhwWTJWdWMyVkpSQ0k2SWpNMlRIQkxNVmQ2Y210QmRUSkpVbFpIV1hkUWVVbzFaVXBPU1NJc0lteHBZMlZ1YzJWVWVYQmxJam9pWkdWMklpd2lZM1Z6ZEc5dFpYSk9ZVzFsSWpvaVEwa2dWak1pTENKaGNIQlRiSFZuSWpvaVpXMWlaV1JrWldRdFkyeDFjM1JsY2kxemJXOXJaUzEwWlhOMExYTjBZV2RwYm1jdFlYQndJaXdpWTJoaGJtNWxiRWxFSWpvaU16Wk1iMGRqVDA5TWRrVlFSbGhSV0VOVmMwWjFZakk0UVdKcElpd2lZMmhoYm01bGJFNWhiV1VpT2lKRFNTMVdNeUlzSW1OMWMzUnZiV1Z5UlcxaGFXd2lPaUpqYVMxMk0wQnlaWEJzYVdOaGRHVmtMbU52YlNJc0ltTm9ZVzV1Wld4eklqcGJleUpqYUdGdWJtVnNTVVFpT2lJek5reHZSMk5QVDB4MlJWQkdXRkZZUTFWelJuVmlNamhCWW1raUxDSmphR0Z1Ym1Wc1UyeDFaeUk2SW1OcExYWXpJaXdpWTJoaGJtNWxiRTVoYldVaU9pSkRTUzFXTXlJc0ltbHpSR1ZtWVhWc2RDSTZkSEoxWlN3aVpXNWtjRzlwYm5RaU9pSm9kSFJ3Y3pvdkwyVmpMV1V5WlMxeVpYQnNhV05oZEdWa0xXRndjQzUwWlhOMFkyeDFjM1JsY2k1dVpYUWlMQ0p5WlhCc2FXTmhkR1ZrVUhKdmVIbEViMjFoYVc0aU9pSmxZeTFsTW1VdGNISnZlSGt1ZEdWemRHTnNkWE4wWlhJdWJtVjBJbjFkTENKc2FXTmxibk5sVTJWeGRXVnVZMlVpT2pFc0ltVnVaSEJ2YVc1MElqb2lhSFIwY0hNNkx5OWxZeTFsTW1VdGNtVndiR2xqWVhSbFpDMWhjSEF1ZEdWemRHTnNkWE4wWlhJdWJtVjBJaXdpY21Wd2JHbGpZWFJsWkZCeWIzaDVSRzl0WVdsdUlqb2laV010WlRKbExYQnliM2g1TG5SbGMzUmpiSFZ6ZEdWeUxtNWxkQ0lzSW1WdWRHbDBiR1Z0Wlc1MGN5STZleUpsZUhCcGNtVnpYMkYwSWpwN0luUnBkR3hsSWpvaVJYaHdhWEpoZEdsdmJpSXNJbVJsYzJOeWFYQjBhVzl1SWpvaVRHbGpaVzV6WlNCRmVIQnBjbUYwYVc5dUlpd2lkbUZzZFdVaU9pSWlMQ0oyWVd4MVpWUjVjR1VpT2lKVGRISnBibWNpTENKemFXZHVZWFIxY21VaU9uc2lkakVpT2lKSlJXNU5OblZDY0ROTUwzbFViSFpQVEZZdmJFOHpWMlk1T0dock1FWmlNMlpUYjA5UFUxWlRaa2xGVWtneVQxZDFka1YxWWpaSlJXdGtSMlF6VFZaVWRqSm5VbG94TlZWdlN6bGFTMGcwUzNwS09WaFNZblJZZEZac2RtTkpMMkpSZEhoM01WQlZVemRqY2pRMk1tNTBWbU5pUlhsUFMxaEhORzVTUmxsc2MyRm9hSHBGVTFKalJIRjVabEl6SzBOcWR6UkpOakpoSzFSYWNuQlFlalZTUms5NlVUWkljV1k1YTBKcVpEWlVkV3MzWW1wclNpOXdLMFF5VVhOUFRXZElOWHBHVTJRMGNEZzNPSGhTZWtVeFVYcDBjbkJtZDBaVlUyVkNlVFJzTDFGVGFEWkpSbmRIVlc1VWNVMW5VVlY2WlM4NGNrNVZMMmt3TldnM05scFBWVGxRZFVOc1N6WTROWGRrUjNJd2IzRmFUWEIyYlc5NGFWVlNOamhrVTAwMVlTdDRPR05OWkRjMVIyeFhWVUppVkdOdWFuVnpiMHg1TTBWTlRTOVhOM1JrU0VRNVdEYzFOelF4WnpaR1IxUm5SMlJ3TDJjOVBTSjlmWDBzSW1selFXbHlaMkZ3VTNWd2NHOXlkR1ZrSWpwMGNuVmxMQ0pwYzBScGMyRnpkR1Z5VW1WamIzWmxjbmxUZFhCd2IzSjBaV1FpT25SeWRXVXNJbWx6VG1WM1MyOTBjMVZwUlc1aFlteGxaQ0k2ZEhKMVpTd2lhWE5UZFhCd2IzSjBRblZ1Wkd4bFZYQnNiMkZrVTNWd2NHOXlkR1ZrSWpwMGNuVmxMQ0pwYzBWdFltVmtaR1ZrUTJ4MWMzUmxja1J2ZDI1c2IyRmtSVzVoWW14bFpDSTZkSEoxWlN3aWFYTkZiV0psWkdSbFpFTnNkWE4wWlhKTmRXeDBhVTV2WkdWRmJtRmliR1ZrSWpwMGNuVmxMQ0pwYzB0dmRITkpibk4wWVd4c1JXNWhZbXhsWkNJNmRISjFaWDE5IiwiaW5uZXJTaWduYXR1cmUiOiJleUpzYVdObGJuTmxVMmxuYm1GMGRYSmxJam9pY0ZoUFRsSlZTMmxpYVVoWWNsSXZMelZZVmpnMGQwUkphelJwVm5OVFl6TkJSRm8wUjNoWVN6WmtaMFJ5Tmt0dVFTOHlWRUZsY0Vvd1ZGQkRaa3hYYm0weU56VXpiMnBzU1ZBNVpXbGxiMDl5VGtKdU16TkJNbG96UzA4NVpqSTVlSGRUVWxGT1dVRlBORE15WlZWdGJVaHhjQ3RrYW1wSGIxaDVSMGRWV1ZWUUwzbFZMM2xTUkV0SmRXUTVVMnBoTkRGcE0yTjZSR2hKYVhsaFlWbDNSVUZzUTFOWlN6Y3laVzVqU25SVlRuZGpLelpVVlVKaFdYTm1PRmxrZVZwVlFsTlhPVEZOVVhadVZVSjRjaTlTUkZSU1VqTnlOMWs0YkZOS2JHMDVaR2xqVFVaUU5qQTRlalpMV1V4blowNWlhVVpoYmtSRkszQkhVRkExWlhGTWRYVlBSMUkxVldSRll6TjFlRU5IYkVoMVJWcExVRlJ1UzI1cVF5OW1hMlZxV1ZwRWNsTk9URWhPU0ZSQmIwbGxNR1JPU1VWcVRsSmtOM3BDVG1Kek5rOXFWMDVyVjNOdllURm9Xa1ZWVVhGRVRYRk9iREozUFQwaUxDSndkV0pzYVdOTFpYa2lPaUl0TFMwdExVSkZSMGxPSUZCVlFreEpReUJMUlZrdExTMHRMVnh1VFVsSlFrbHFRVTVDWjJ0eGFHdHBSemwzTUVKQlVVVkdRVUZQUTBGUk9FRk5TVWxDUTJkTFEwRlJSVUUwVkhWRlUwWjFWV0pMV1U4eFdIRlVkVlUzVTF4dWJFMDFWV0V2ZFdGSE0zY3hiR0YyV1VveFZrOXBNMlpDWjFKU1JtRXdOMGsxVDBscWRrcFNTVkZVZDJRMVJ5OVZObTlyYWxKaFRWZ3hObVk0WXpoMGFGeHVkV1JSYVRFNGFrWTRUbVpsZFV4RWNHNWxOVkpoU2tOc05XOVphMDlEUVZobmRHSkpkVWh3UmtSNFMwYzNRVFZrTVhaV1JHMVFhbmxrWlVKelNIWjJRMXh1WlhoM2NVRktlRWRrTWpsTlNITkJZVU5XVFhwbFVuUldPVVZLZDBWdUwyUnRPVVZ6WTBoaVJ6ZzJZbHBqWVhsRVpqTm5UWEl4TWpjd09URlBUazlXUlZ4dU9WVkdaVVpXWVd4aE1uRmxOVEZpTURrM05rZGtZM1JzUlhKMmRtTlFUVnB3U21RcmVqSlRkbU54VTJSS2NFOVpRakV6YW1SV2FuTk5kbUZCU1c4eFQxeHVOMXBGVTBWcFVrRjZOVWhhY0hwMldXNHpjaTl4YTB4NlNHUnBUVU5PUnl0bWVtUmxkemRsWlVKdFkza3JhMUJWV0ZnelpHRnlNMkZhVnl0dFprTlNPVnh1YkZGSlJFRlJRVUpjYmkwdExTMHRSVTVFSUZCVlFreEpReUJMUlZrdExTMHRMVnh1SWl3aWEyVjVVMmxuYm1GMGRYSmxJam9pWlhsS2VtRlhaSFZaV0ZJeFkyMVZhVTlwU2xGWk1rbDRaRzA0TTJGclZYcFRSMDVhWld0YU5GWXdUa1pYV0VadVpXNUJNMVJ1YTNwT1ZHd3pXbGhTUjFsWFNrZFphelZLVFVoS00xTkZWbFpVUkd4dFkycG9UR1F4VWxkbFZ6bHhXVEprY2xkWE9YcE1NVXBJVFVaR1ZWRXdPSEprU0VVeFdURkdhV0p1WXpWU01FWlNVbFJLYlZOV1ZsRlRhbEpwV1d0c05rMVlTbXRQVlVwM1VrTTVOVmx0VW1GYVJUUjJZMjFHVVZFeWRFbE5SVFY1WWtSV1RWUXlkRE5WTW5Cc1dtMU5lV05ZV1ROV00wRXpXbmwwVjFwSFRYSlVXRkpUVXpOc1RsVlVSbTVTVjNoV1RXdGFhVnByWTNsVGVtUnBaRlpCZGxWc1duUlJNREJ5VVdwS1RWUXhZM2xOU0VwdVV6Qk9SMkl4UlRWT1YzUnBWMGhCTVZkc2NETldSVnBzVDFST1ZtTXdkRTFrVjFweFkwaENWMU13ZEVSalYwNDBWak53ZUdKRlZUVk5WRXBxV1ZSRk0yTXpWbXBXVmxZeVpERm9NMkpwZEc1a1ZFSlJVekJPV1ZONldtMVBSa1Z5VGpKck5WZHBkRlJUUjFaclRVUmFOVmxzY0dwWlZrSXhWbFZ2ZVdNeGJIbFVha0ozWkd4d1VsVldTbWhpVms1TFRWUnNWRTVFVlRCTGVYUjNVa2RLTWxScldtaGpla293WVcweFRHSnFXWGxMTUdNd1QxVmpkbUpGU2xaWldFWlFaRlpzU0U1dE9YcGFSbWQyWW0xa1FtUllZemxRVTBselNXMWtjMkl5U21oaVJYUnNaVlZzYTBscWIybGFSMVY1V1hwSk0wNVVXVEZPYlZGM1RrZEplRmx0U1hkYWFrVXhXVEpaTTAxSFdYZGFWMFY1V1ZSSmFXWlJQVDBpZlE9PSJ9 diff --git a/fio/Dockerfile b/fio/Dockerfile deleted file mode 100644 index 04db3d60ea..0000000000 --- a/fio/Dockerfile +++ /dev/null @@ -1,27 +0,0 @@ -ARG PLATFORM=linux/amd64 - -FROM --platform=$PLATFORM ubuntu:22.04 AS build - -ARG DEBIAN_FRONTEND=noninteractive -ARG TZ=Etc/UTC - -RUN apt-get update \ - && apt-get install -y \ - build-essential cmake libstdc++6 pkg-config unzip wget - -RUN mkdir -p /fio -WORKDIR /fio - -ARG FIO_VERSION -RUN [ -z "$FIO_VERSION" ] && echo "FIO_VERSION is required" && exit 1 || true -RUN wget -O fio.tar.gz -q https://api.github.com/repos/axboe/fio/tarball/fio-$FIO_VERSION -RUN tar -xzf fio.tar.gz --strip-components=1 - -RUN ./configure --build-static --disable-native -RUN make -j$(nproc) - -FROM ubuntu:22.04 -COPY --from=build /fio/fio /output/fio -# ensure that the binary is statically linked -RUN ldd /output/fio 2>&1 | grep 'not a dynamic executable' -CMD [ "echo", "Done" ] diff --git a/hack/dev-embed.go b/hack/dev-embed.go deleted file mode 100644 index ca7e09307d..0000000000 --- a/hack/dev-embed.go +++ /dev/null @@ -1,39 +0,0 @@ -package main - -import ( - "flag" - "fmt" - "log" - "os" - - "github.com/replicatedhq/embedded-cluster/utils/pkg/embed" -) - -func main() { - os.Exit(run()) -} - -func run() int { - binaryPath := flag.String("binary", "", "Path to the binary file") - releasePath := flag.String("release", "", "Path to the release tar.gz file") - outputPath := flag.String("output", "", "Path to the output file") - label := flag.String("label", "", "Release label") - sequence := flag.Int("sequence", 0, "Release sequence number") - channel := flag.String("channel", "", "Channel slug") - - flag.Parse() - - if *binaryPath == "" || *releasePath == "" || *outputPath == "" { - fmt.Printf("Usage: %s --binary --release --output [--label