Skip to content

Fix unstable integration tests. #1060

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jglogan merged 1 commit into from
Jan 16, 2026
+36 −5
Merged

Fix unstable integration tests. #1060

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 2 additions & 1 deletion Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -184,7 +184,8 @@ integration: init-block
$(SWIFT) test -c $(BUILD_CONFIGURATION) $(SWIFT_CONFIGURATION) --filter TestCLIRunLifecycle || exit_code=1 ; \
$(SWIFT) test -c $(BUILD_CONFIGURATION) $(SWIFT_CONFIGURATION) --filter TestCLIExecCommand || exit_code=1 ; \
$(SWIFT) test -c $(BUILD_CONFIGURATION) $(SWIFT_CONFIGURATION) --filter TestCLICreateCommand || exit_code=1 ; \
$(SWIFT) test -c $(BUILD_CONFIGURATION) $(SWIFT_CONFIGURATION) --filter TestCLIRunCommand || exit_code=1 ; \
$(SWIFT) test -c $(BUILD_CONFIGURATION) $(SWIFT_CONFIGURATION) --filter TestCLIRunCommand1 || exit_code=1 ; \
$(SWIFT) test -c $(BUILD_CONFIGURATION) $(SWIFT_CONFIGURATION) --filter TestCLIRunCommand2 || exit_code=1 ; \
$(SWIFT) test -c $(BUILD_CONFIGURATION) $(SWIFT_CONFIGURATION) --filter TestCLIStatsCommand || exit_code=1 ; \
$(SWIFT) test -c $(BUILD_CONFIGURATION) $(SWIFT_CONFIGURATION) --filter TestCLIImagesCommand || exit_code=1 ; \
$(SWIFT) test -c $(BUILD_CONFIGURATION) $(SWIFT_CONFIGURATION) --filter TestCLIRunBase || exit_code=1 ; \
Expand Down
23 changes: 20 additions & 3 deletions Tests/CLITests/Subcommands/Run/TestCLIRunCommand.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,12 +21,19 @@ import ContainerizationOS
import Foundation
import Testing

class TestCLIRunCommand: CLITest {
private func getTestName() -> String {
// FIXME: We've split the tests into two suites to prevent swamping
// the API server with so many run commands that all wind up pulling
// images.
//
// When https://github.com/swiftlang/swift-testing/pull/1390 lands
// and is available on the CI runners, we can try setting the
// environment variable to limit concurrency and rejoin these suites.
class TestCLIRunCommand1: CLITest {
func getTestName() -> String {
Test.current!.name.trimmingCharacters(in: ["(", ")"]).lowercased()
}

private func getLowercasedTestName() -> String {
func getLowercasedTestName() -> String {
getTestName().lowercased()
}

Expand Down Expand Up @@ -194,6 +201,16 @@ class TestCLIRunCommand: CLITest {
return
}
}
}

class TestCLIRunCommand2: CLITest {
func getTestName() -> String {
Test.current!.name.trimmingCharacters(in: ["(", ")"]).lowercased()
}

func getLowercasedTestName() -> String {
getTestName().lowercased()
}

@Test func testRunCommandMount() throws {
do {
Expand Down
15 changes: 14 additions & 1 deletion Tests/CLITests/TestCLINoParallelCases.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,13 +22,18 @@ import Testing
/// Tests that need total control over environment to avoid conflicts.
@Suite(.serialized)
class TestCLINoParallelCases: CLITest {
private func getTestName() -> String {
func getTestName() -> String {
Test.current!.name.trimmingCharacters(in: ["(", ")"]).lowercased()
}

func getLowercasedTestName() -> String {
getTestName().lowercased()
}

@Test func testImageSingleConcurrentDownload() throws {
// removing this image during parallel tests breaks stuff!
_ = try? run(arguments: ["image", "rm", alpine])
defer { _ = try? run(arguments: ["image", "rm", "--all"]) }
Copy link
Contributor

@iko1 iko1 Jan 16, 2026

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Consider adding logging in case of failures

Copy link
Contributor Author

@jglogan jglogan Jan 16, 2026 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's a good idea but I think we'd want to build that into our test support code. We put together the integration tests and related support code early on and there's plenty of room for refinement. It's out of scope for this PR as we just need to get tests working again, but consider filing an issue describing the enhancement, and then knock out a PR!

do {
try doPull(imageName: alpine, args: ["--max-concurrent-downloads", "1"])
let imagePresent = try isImagePresent(targetImage: alpine)
Expand All @@ -42,6 +47,7 @@ class TestCLINoParallelCases: CLITest {
@Test func testImageManyConcurrentDownloads() throws {
// removing this image during parallel tests breaks stuff!
_ = try? run(arguments: ["image", "rm", alpine])
defer { _ = try? run(arguments: ["image", "rm", "--all"]) }
do {
try doPull(imageName: alpine, args: ["--max-concurrent-downloads", "64"])
let imagePresent = try isImagePresent(targetImage: alpine)
Expand All @@ -54,6 +60,7 @@ class TestCLINoParallelCases: CLITest {

@Test func testImagePruneNoImages() throws {
// Prune with no images should succeed
_ = try? run(arguments: ["image", "rm", "--all"])
let (_, output, error, status) = try run(arguments: ["image", "prune"])
if status != 0 {
throw CLIError.executionFailed("image prune failed: \(error)")
Expand All @@ -64,6 +71,8 @@ class TestCLINoParallelCases: CLITest {

@Test func testImagePruneUnusedImages() throws {
// 1. Pull the images
_ = try? run(arguments: ["image", "rm", "--all"])
defer { _ = try? run(arguments: ["image", "rm", "--all"]) }
try doPull(imageName: alpine)
try doPull(imageName: busybox)

Expand Down Expand Up @@ -93,6 +102,10 @@ class TestCLINoParallelCases: CLITest {
let containerName = "\(name)_container"

// 1. Pull the images
_ = try? run(arguments: ["image", "rm", "--all"])
defer { _ = try? run(arguments: ["image", "rm", "--all"]) }
_ = try? run(arguments: ["rm", "--all", "--force"])
defer { _ = try? run(arguments: ["rm", "--all", "--force"]) }
try doPull(imageName: alpine)
try doPull(imageName: busybox)

Expand Down