Skip to content

Commit 0a7b7ff

Browse files
emosbaughemosbaugh
committed
f
1 parent 3d123bf commit 0a7b7ff

File tree

2 files changed

+139
-6
lines changed

2 files changed

+139
-6
lines changed

‎dagger/cmx.go‎

Lines changed: 123 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ package main
33
import (
44
"context"
55
"fmt"
6+
"strconv"
67
"strings"
78
"time"
89

@@ -552,9 +553,17 @@ func (i *CmxInstance) PrepareRelease(
552553
releaseURL = fmt.Sprintf("%s?airgap=true", releaseURL)
553554
}
554555

555-
downloadCmd := fmt.Sprintf(`curl --retry 5 --retry-all-errors -fL -o /tmp/ec-release.tgz "%s" -H "Authorization: %s"`, releaseURL, licenseID)
556-
if _, err := i.Command(downloadCmd).Stdout(ctx); err != nil {
557-
return fmt.Errorf("download release: %w", err)
556+
// For airgap, retry up to 20 times with 1 minute sleep between attempts
557+
// The API returns 400 when the bundle is still being built
558+
if scenario == "airgap" {
559+
if err := i.downloadAirgapBundleWithRetry(ctx, releaseURL, licenseID); err != nil {
560+
return fmt.Errorf("download airgap bundle: %w", err)
561+
}
562+
} else {
563+
downloadCmd := fmt.Sprintf(`curl --retry 5 --retry-all-errors -fL -o /tmp/ec-release.tgz "%s" -H "Authorization: %s"`, releaseURL, licenseID)
564+
if _, err := i.Command(downloadCmd).Stdout(ctx); err != nil {
565+
return fmt.Errorf("download release: %w", err)
566+
}
558567
}
559568

560569
// Extract release tarball
@@ -595,6 +604,72 @@ func (i *CmxInstance) PrepareRelease(
595604
return nil
596605
}
597606

607+
// downloadAirgapBundleWithRetry downloads an airgap bundle with retry logic.
608+
//
609+
// The airgap bundle API may return 400 errors when the bundle is still being built.
610+
// This method retries up to 20 times with a 1 minute sleep between attempts,
611+
// and verifies the downloaded file is at least 1GB to ensure it's complete.
612+
func (i *CmxInstance) downloadAirgapBundleWithRetry(ctx context.Context, url string, licenseID string) error {
613+
for attempt := 1; attempt <= 20; attempt++ {
614+
fmt.Printf("Attempting to download airgap bundle (attempt %d/20)...\n", attempt)
615+
616+
// Download with curl -f which will fail on HTTP 4xx/5xx errors
617+
downloadCmd := fmt.Sprintf(`curl -fL -o /tmp/ec-release.tgz "%s" -H "Authorization: %s"`, url, licenseID)
618+
_, err := i.Command(downloadCmd).Stdout(ctx)
619+
620+
if err != nil {
621+
fmt.Printf("Download attempt %d failed: %v\n", attempt, err)
622+
if attempt < 20 {
623+
fmt.Printf("Waiting 1 minute before retry...\n")
624+
time.Sleep(1 * time.Minute)
625+
continue
626+
}
627+
return fmt.Errorf("failed after 20 attempts: %w", err)
628+
}
629+
630+
// Check file size - airgap bundles should be at least 1GB
631+
sizeCmd := `du -b /tmp/ec-release.tgz | awk '{print $1}'`
632+
sizeStr, err := i.Command(sizeCmd).Stdout(ctx)
633+
if err != nil {
634+
fmt.Printf("Failed to check file size: %v\n", err)
635+
if attempt < 20 {
636+
fmt.Printf("Waiting 1 minute before retry...\n")
637+
time.Sleep(1 * time.Minute)
638+
continue
639+
}
640+
return fmt.Errorf("failed to check file size after 20 attempts: %w", err)
641+
}
642+
643+
sizeStr = strings.TrimSpace(sizeStr)
644+
sizeBytes, err := strconv.ParseInt(sizeStr, 10, 64)
645+
if err != nil {
646+
return fmt.Errorf("failed to parse file size %q: %w", sizeStr, err)
647+
}
648+
649+
minSize := int64(1024 * 1024 * 1024) // 1GB
650+
if sizeBytes < minSize {
651+
fmt.Printf("Downloaded file is only %d bytes (%.2f GB), expected at least 1GB. Retrying...\n",
652+
sizeBytes, float64(sizeBytes)/(1024*1024*1024))
653+
// Remove the incomplete download
654+
if _, err := i.Command("rm -f /tmp/ec-release.tgz").Stdout(ctx); err != nil {
655+
fmt.Printf("Warning: failed to remove incomplete download: %v\n", err)
656+
}
657+
if attempt < 20 {
658+
fmt.Printf("Waiting 1 minute before retry...\n")
659+
time.Sleep(1 * time.Minute)
660+
continue
661+
}
662+
return fmt.Errorf("downloaded file too small after 20 attempts: %d bytes", sizeBytes)
663+
}
664+
665+
fmt.Printf("Successfully downloaded airgap bundle (%.2f GB) on attempt %d\n",
666+
float64(sizeBytes)/(1024*1024*1024), attempt)
667+
return nil
668+
}
669+
670+
return fmt.Errorf("failed to download airgap bundle after 20 attempts")
671+
}
672+
598673
// InstallHeadless performs a headless (CLI) installation without Playwright.
599674
//
600675
// This method downloads the release, optionally uploads a config file, builds the
@@ -741,3 +816,48 @@ func (i *CmxInstance) CollectClusterSupportBundle(ctx context.Context) (*dagger.
741816

742817
return file, nil
743818
}
819+
820+
// CollectHostSupportBundle collects a host support bundle from the VM.
821+
//
822+
// This method collects diagnostic information about the host system.
823+
// It tries two approaches:
824+
// 1. First attempts to collect using kubectl-support_bundle with the host spec
825+
// 2. If that fails, falls back to collecting installer logs from /var/log/embedded-cluster
826+
//
827+
// The collected support bundle is downloaded from the VM and returned as a Dagger File
828+
// that can be exported as an artifact.
829+
//
830+
// Example:
831+
//
832+
// dagger call with-one-password --service-account=env:OP_SERVICE_ACCOUNT_TOKEN \
833+
// with-cmx-vm --vm-id=8a2a66ef \
834+
// collect-host-support-bundle \
835+
// export --path=./host-support-bundle.tar.gz
836+
func (i *CmxInstance) CollectHostSupportBundle(ctx context.Context) (*dagger.File, error) {
837+
bundlePath := "/tmp/host-support-bundle.tar.gz"
838+
839+
// Try collecting host support bundle with kubectl-support_bundle first
840+
cmd1 := fmt.Sprintf("%s/bin/kubectl-support_bundle --output %s --interactive=false %s/support/host-support-bundle.yaml", DataDir, bundlePath, DataDir)
841+
_, err := i.Command(cmd1).Stdout(ctx)
842+
843+
// If first attempt failed, try collecting installer logs as fallback
844+
if err != nil {
845+
fmt.Printf("Host support bundle attempt failed, trying to collect installer logs: %v\n", err)
846+
cmd2 := fmt.Sprintf("tar -czf %s -C / var/log/embedded-cluster 2>/dev/null || tar -czf %s --files-from=/dev/null", bundlePath, bundlePath)
847+
stdout, err := i.Command(cmd2).Stdout(ctx)
848+
if err != nil {
849+
return nil, fmt.Errorf("failed to collect host support bundle and installer logs (both attempts): %w\nOutput: %s", err, stdout)
850+
}
851+
fmt.Printf("Installer logs collected as fallback\n")
852+
} else {
853+
fmt.Printf("Host support bundle collected successfully at %s\n", bundlePath)
854+
}
855+
856+
// Download the support bundle from the VM
857+
file, err := i.DownloadFile(ctx, bundlePath)
858+
if err != nil {
859+
return nil, fmt.Errorf("download host support bundle: %w", err)
860+
}
861+
862+
return file, nil
863+
}

‎dagger/e2e.go‎

Lines changed: 16 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -30,12 +30,14 @@ var configValuesFileContent string
3030
// Example (online):
3131
//
3232
// dagger call with-one-password --service-account=env:OP_SERVICE_ACCOUNT_TOKEN \
33-
// e-2-e-run-headless --scenario=online --app-version=appver-dev-xpXCTO --kube-version=1.33 --license-file=./local-dev/ethan-dev-license.yaml
33+
// e-2-e-run-headless --scenario=online --app-version=appver-dev-xpXCTO --kube-version=1.33 --license-file=./local-dev/ethan-dev-license.yaml \
34+
// export --path=./e2e-results-online
3435
//
3536
// Example (airgap):
3637
//
3738
// dagger call with-one-password --service-account=env:OP_SERVICE_ACCOUNT_TOKEN \
38-
// e-2-e-run-headless --scenario=airgap --app-version=appver-dev-xpXCTO --kube-version=1.33 --license-file=./local-dev/ethan-dev-license.yaml
39+
// e-2-e-run-headless --scenario=airgap --app-version=appver-dev-xpXCTO --kube-version=1.33 --license-file=./local-dev/ethan-dev-license.yaml \
40+
// export --path=./e2e-results-airgap
3941
func (m *EmbeddedCluster) E2eRunHeadless(
4042
ctx context.Context,
4143
// Scenario (online or airgap)
@@ -55,6 +57,9 @@ func (m *EmbeddedCluster) E2eRunHeadless(
5557
// Skip cleanup
5658
// +default=false
5759
skipCleanup bool,
60+
// Skip support bundle collection
61+
// +default=false
62+
skipSupportBundleCollection bool,
5863
) (resultsDir *dagger.Directory) {
5964
mode := "headless"
6065

@@ -99,7 +104,7 @@ func (m *EmbeddedCluster) E2eRunHeadless(
99104
// Defer function to collect support bundle and cleanup VM
100105
defer func() {
101106
// Collect support bundle before cleanup
102-
if vm != nil {
107+
if vm != nil && !skipSupportBundleCollection {
103108
fmt.Printf("Collecting support bundle from VM %s...\n", vm.VmID)
104109
supportBundle, bundleErr := vm.CollectClusterSupportBundle(ctx)
105110
if bundleErr != nil {
@@ -108,6 +113,14 @@ func (m *EmbeddedCluster) E2eRunHeadless(
108113
} else {
109114
resultsDir = resultsDir.WithFile("support-bundle.tar.gz", supportBundle)
110115
}
116+
117+
hostSupportBundle, hostBundleErr := vm.CollectHostSupportBundle(ctx)
118+
if hostBundleErr != nil {
119+
fmt.Printf("Warning: failed to collect host support bundle: %v\n", hostBundleErr)
120+
resultsDir = resultsDir.WithNewFile("host-support-bundle-error.txt", fmt.Sprintf("Failed to collect host support bundle: %v", hostBundleErr))
121+
} else {
122+
resultsDir = resultsDir.WithFile("host-support-bundle.tar.gz", hostSupportBundle)
123+
}
111124
}
112125

113126
// Marshal final test result to JSON

0 commit comments

Comments
 (0)