Skip to content

Commit 962f590

Browse files
committed
feat: add VM-based E2E testing with Tart
Add comprehensive end-to-end test suite using disposable Tart macOS VMs. Tests cover the full user journey from bare macOS to configured dev environment, validating real system state rather than just command output. Infrastructure: - testutil/tartvm.go: VM lifecycle management (clone, boot, SSH, expect, destroy) - Three-tier Makefile targets: test-vm-quick (5min), test-vm-release (20min), test-vm-full (60min) - Cirrus CI config for cloud-based parallel VM testing Test coverage (48 test functions): - 9 user journey tests (first-time install, dry-run safety, snapshot accuracy, diff consistency, idempotency, packages-only, manual uninstall recovery, full setup, error messages) - 10 edge case tests (snapshot import/restore, clean removal, cask verification, npm packages, shell functionality, remote config, developer preset, post-install scripts, version upgrade compatibility, partial failure recovery) - 13 command tests (every command and flag) - 6 interactive tests (expect-based TUI simulation) - 4 install flow tests + 4 lifecycle tests + 1 infra test Discovered and fixed 3 bugs via these tests (see previous commit).
1 parent 4df612b commit 962f590

File tree

11 files changed

+2934
-1
lines changed

11 files changed

+2934
-1
lines changed

.cirrus.yml

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
# Cirrus CI — VM-based E2E tests using Tart (native macOS virtualization)
2+
#
3+
# Cirrus CI runs on Apple Silicon and supports Tart natively, enabling
4+
# parallel macOS VM tests that aren't possible on GitHub Actions.
5+
#
6+
# Trigger: manual dispatch or release tags.
7+
# For PR testing, use GitHub Actions (unit + integration tests).
8+
9+
env:
10+
HOMEBREW_NO_AUTO_UPDATE: 1
11+
HOMEBREW_NO_INSTALL_CLEANUP: 1
12+
13+
# =============================================================================
14+
# L2: Release validation — runs on every tag push
15+
# =============================================================================
16+
release_validation_task:
17+
name: "E2E Release Validation"
18+
only_if: $CIRRUS_TAG != '' || $CIRRUS_BRANCH == 'main'
19+
macos_instance:
20+
image: ghcr.io/cirruslabs/macos-sequoia-base:latest
21+
install_deps_script:
22+
- eval "$(/opt/homebrew/bin/brew shellenv)"
23+
- brew install go sshpass hudochenkov/sshpass/sshpass cirruslabs/cli/tart
24+
build_script:
25+
- eval "$(/opt/homebrew/bin/brew shellenv)"
26+
- make build
27+
test_script:
28+
- eval "$(/opt/homebrew/bin/brew shellenv)"
29+
- make test-vm-release
30+
timeout_in: 45m
31+
32+
# =============================================================================
33+
# L3: Full validation — manual trigger or release tags
34+
# =============================================================================
35+
# Split into parallel tasks to maximize throughput.
36+
# Each task runs a subset of tests in its own VM.
37+
38+
full_journey_task:
39+
name: "E2E Journey Tests"
40+
trigger_type: manual
41+
macos_instance:
42+
image: ghcr.io/cirruslabs/macos-sequoia-base:latest
43+
install_deps_script:
44+
- eval "$(/opt/homebrew/bin/brew shellenv)"
45+
- brew install go sshpass hudochenkov/sshpass/sshpass cirruslabs/cli/tart
46+
build_script:
47+
- eval "$(/opt/homebrew/bin/brew shellenv)"
48+
- make build
49+
test_script:
50+
- eval "$(/opt/homebrew/bin/brew shellenv)"
51+
- go test -v -timeout 60m -tags="e2e,vm" -run "TestVM_Journey" ./test/e2e/...
52+
timeout_in: 90m
53+
54+
full_edge_cases_task:
55+
name: "E2E Edge Case Tests"
56+
trigger_type: manual
57+
macos_instance:
58+
image: ghcr.io/cirruslabs/macos-sequoia-base:latest
59+
install_deps_script:
60+
- eval "$(/opt/homebrew/bin/brew shellenv)"
61+
- brew install go sshpass hudochenkov/sshpass/sshpass cirruslabs/cli/tart
62+
build_script:
63+
- eval "$(/opt/homebrew/bin/brew shellenv)"
64+
- make build
65+
test_script:
66+
- eval "$(/opt/homebrew/bin/brew shellenv)"
67+
- go test -v -timeout 60m -tags="e2e,vm" -run "TestVM_Edge" ./test/e2e/...
68+
timeout_in: 90m
69+
70+
full_commands_task:
71+
name: "E2E Command + Interactive Tests"
72+
trigger_type: manual
73+
macos_instance:
74+
image: ghcr.io/cirruslabs/macos-sequoia-base:latest
75+
install_deps_script:
76+
- eval "$(/opt/homebrew/bin/brew shellenv)"
77+
- brew install go sshpass hudochenkov/sshpass/sshpass cirruslabs/cli/tart
78+
build_script:
79+
- eval "$(/opt/homebrew/bin/brew shellenv)"
80+
- make build
81+
test_script:
82+
- eval "$(/opt/homebrew/bin/brew shellenv)"
83+
- go test -v -timeout 60m -tags="e2e,vm" -run "TestVM_Cmd|TestVM_Install|TestVM_Interactive|TestVM_Lifecycle" ./test/e2e/...
84+
timeout_in: 90m

Makefile

Lines changed: 32 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
1-
.PHONY: test-unit test-integration test-e2e test-destructive test-smoke test-smoke-prebuilt test-coverage test-all
1+
.PHONY: test-unit test-integration test-e2e test-destructive test-smoke test-smoke-prebuilt test-coverage test-all \
2+
test-vm test-vm-short test-vm-run test-vm-quick test-vm-release test-vm-full
23

34
BINARY_NAME=openboot
45
BINARY_PATH=./$(BINARY_NAME)
@@ -38,6 +39,36 @@ test-all:
3839
-$(MAKE) test-e2e
3940
$(MAKE) test-coverage
4041

42+
# =============================================================================
43+
# VM-based E2E tests (Tart VMs) — three levels
44+
# =============================================================================
45+
46+
# L1: Quick validation (~5min) — run after code changes
47+
# Uses -short flag: skips tests that install real packages
48+
test-vm-quick: build
49+
go test -v -timeout 10m -tags="e2e,vm" -short ./test/e2e/...
50+
51+
# L2: Release validation (~20min) — run before tagging a release
52+
# Core user journeys: dry-run safety, install + verify, diff/clean cycle,
53+
# manual uninstall recovery, full setup, error messages
54+
test-vm-release: build
55+
go test -v -timeout 30m -tags="e2e,vm" \
56+
-run "TestVM_Infra|TestVM_Journey_DryRun|TestVM_Journey_FirstTimeUser|TestVM_Journey_ManualUninstall|TestVM_Journey_DiffConsistency|TestVM_Journey_FullSetup|TestVM_Journey_ErrorMessages" \
57+
./test/e2e/...
58+
59+
# L3: Full validation (~60min) — run for major releases or CI
60+
# All 48 tests: journeys + edge cases + commands + interactive
61+
test-vm-full: build
62+
go test -v -timeout 90m -tags="e2e,vm" ./test/e2e/...
63+
64+
# Aliases
65+
test-vm: test-vm-release
66+
test-vm-short: test-vm-quick
67+
68+
# Single VM test by name (e.g. make test-vm-run TEST=TestVM_Journey_DryRun)
69+
test-vm-run: build
70+
go test -v -timeout 45m -tags="e2e,vm" -run $(TEST) ./test/e2e/...
71+
4172
build:
4273
go build -ldflags="$(LDFLAGS)" -o $(BINARY_PATH) ./cmd/openboot
4374

0 commit comments

Comments
 (0)