From a531e0303d4c40633638782c26168ae74dd1b310 Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Mon, 13 Apr 2026 16:55:57 -0400 Subject: [PATCH 1/2] Relax tests that use `testcontainers` a bit. --- gradle/java_no_deps.gradle | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/gradle/java_no_deps.gradle b/gradle/java_no_deps.gradle index 4d446e4a610..91d7525762b 100644 --- a/gradle/java_no_deps.gradle +++ b/gradle/java_no_deps.gradle @@ -121,9 +121,9 @@ def tracerJavaExtension = extensions.create(TracerJavaExtension.NAME, TracerJava -// Only run one testcontainers test at a time +// Number of testcontainers test at a time ext.testcontainersLimit = gradle.sharedServices.registerIfAbsent("testcontainersLimit", BuildService) { - maxParallelUsages = 1 + maxParallelUsages = 2 } // Task for tests that want to run forked in their own separate JVM From bf81dc2144625c028852fb15deb0909cbb56eee4 Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Mon, 13 Apr 2026 17:58:46 -0400 Subject: [PATCH 2/2] Attempt to apply slots per tasks, not projects. That should give better parallel execution. --- .../gradle/plugin/ci/CIJobsExtensions.kt | 91 +++++++++++-------- .../kotlin/dd-trace-java.ci-jobs.gradle.kts | 10 -- 2 files changed, 55 insertions(+), 46 deletions(-) diff --git a/buildSrc/src/main/kotlin/datadog/gradle/plugin/ci/CIJobsExtensions.kt b/buildSrc/src/main/kotlin/datadog/gradle/plugin/ci/CIJobsExtensions.kt index 329467f9dcb..06580f5c928 100644 --- a/buildSrc/src/main/kotlin/datadog/gradle/plugin/ci/CIJobsExtensions.kt +++ b/buildSrc/src/main/kotlin/datadog/gradle/plugin/ci/CIJobsExtensions.kt @@ -3,22 +3,19 @@ package datadog.gradle.plugin.ci import org.gradle.api.Project import org.gradle.api.Task import org.gradle.api.provider.Provider +import org.gradle.api.tasks.testing.Test import org.gradle.kotlin.dsl.extra import kotlin.math.abs -/** - * Determines if the current project is in the selected slot. - * - * The "slot" property should be provided in the format "X/Y", where X is the selected slot (1-based) - * and Y is the total number of slots. - * - * If the "slot" property is not provided, all projects are considered to be in the selected slot. - */ -val Project.isInSelectedSlot: Provider - get() = rootProject.providers.gradleProperty("slot").map { slot -> +private fun selectedSlotFor( + rootProject: Project, + identityPath: String, + kind: String, +): Provider = + rootProject.providers.gradleProperty("slot").map { slot -> val parts = slot.split("/") if (parts.size != 2) { - project.logger.warn("Invalid slot format '{}', expected 'X/Y'. Treating all projects as selected.", slot) + rootProject.logger.warn("Invalid slot format '{}', expected 'X/Y'. Treating all {}s as selected.", slot, kind) return@map true } @@ -26,31 +23,43 @@ val Project.isInSelectedSlot: Provider val totalSlots = parts[1].toIntOrNull() if (selectedSlot == null || totalSlots == null || totalSlots <= 0) { - project.logger.warn("Invalid slot values '{}', expected numeric 'X/Y' with Y > 0. Treating all projects as selected.", slot) + rootProject.logger.warn( + "Invalid slot values '{}', expected numeric 'X/Y' with Y > 0. Treating all {}s as selected.", + slot, + kind, + ) return@map true } - // Distribution numbers when running on rootProject.allprojects indicates - // bucket sizes are reasonably balanced: - // - // * size 4 distribution: {2=146, 0=143, 1=157, 3=145} - // * size 6 distribution: {4=100, 0=92, 3=97, 2=97, 1=108, 5=97} - // * size 8 distribution: {2=62, 4=72, 0=71, 5=70, 7=78, 6=84, 1=87, 3=67} - // * size 10 distribution: {8=62, 0=65, 5=70, 9=59, 3=54, 1=56, 6=63, 4=47, 2=52, 7=63} - // * size 12 distribution: {10=55, 0=47, 4=45, 9=46, 8=51, 3=51, 2=46, 1=59, 5=52, 7=49, 11=45, 6=45} - val projectSlot = abs(project.path.hashCode() % totalSlots) + 1 // Convert to 1-based - - project.logger.info( - "Project {} assigned to slot {}/{}, active slot is {}", - project.path, - projectSlot, - totalSlots, - selectedSlot, - ) - - projectSlot == selectedSlot + val slotForIdentity = abs(identityPath.hashCode() % totalSlots) + 1 // Convert to 1-based + slotForIdentity == selectedSlot }.orElse(true) +/** + * Determines if the current project is in the selected slot. + * + * The "slot" property should be provided in the format "X/Y", where X is the selected slot (1-based) + * and Y is the total number of slots. + * + * If the "slot" property is not provided, all projects are considered to be in the selected slot. + */ +val Project.isInSelectedSlot: Provider + get() = selectedSlotFor(rootProject, path, "project") + +val Task.isInSelectedSlot: Provider + get() = selectedSlotFor(project.rootProject, path, "task") + +private fun Project.aggregateTestTasksFor(subproject: Project, aggregateTaskName: String): List = + when (aggregateTaskName) { + "allTests" -> subproject.tasks.withType(Test::class.java).matching { testTask -> + !testTask.name.contains("latest", ignoreCase = true) && testTask.name != "traceAgentTest" + }.toList() + "allLatestDepTests" -> subproject.tasks.withType(Test::class.java).matching { testTask -> + testTask.name.contains("latest", ignoreCase = true) + }.toList() + else -> emptyList() + } + /** * Returns the task's path, given affected projects, if this task or its dependencies are affected by git changes. */ @@ -91,22 +100,26 @@ private fun Project.createRootTask( forceCoverage: Boolean ) { val coverage = forceCoverage || rootProject.providers.gradleProperty("checkCoverage").isPresent + val taskLevelSlotting = !coverage && (subProjTaskName == "allTests" || subProjTaskName == "allLatestDepTests") tasks.register(rootTaskName) { subprojects.forEach { subproject -> if ( - subproject.isInSelectedSlot.get() && includePrefixes.any { subproject.path.startsWith(it) } && !excludePrefixes.any { subproject.path.startsWith(it) } ) { - val testTask = subproject.tasks.findByName(subProjTaskName) + if (!taskLevelSlotting && !subproject.isInSelectedSlot.get()) { + return@forEach + } + + val aggregateTask = subproject.tasks.findByName(subProjTaskName) var isAffected = true - if (testTask != null) { + if (aggregateTask != null) { val useGitChanges = rootProject.extra.get("useGitChanges") as Boolean if (useGitChanges) { @Suppress("UNCHECKED_CAST") val affectedProjects = rootProject.extra.get("affectedProjects") as Map> - val affectedTaskPath = findAffectedTaskPath(testTask, affectedProjects) + val affectedTaskPath = findAffectedTaskPath(aggregateTask, affectedProjects) if (affectedTaskPath != null) { logger.warn("Selecting ${subproject.path}:$subProjTaskName (affected by $affectedTaskPath)") } else { @@ -115,7 +128,13 @@ private fun Project.createRootTask( } } if (isAffected) { - dependsOn(testTask) + if (taskLevelSlotting) { + dependsOn(aggregateTestTasksFor(subproject, subProjTaskName).filter { testTask -> + testTask.isInSelectedSlot.get() + }) + } else { + dependsOn(aggregateTask) + } } } diff --git a/buildSrc/src/main/kotlin/dd-trace-java.ci-jobs.gradle.kts b/buildSrc/src/main/kotlin/dd-trace-java.ci-jobs.gradle.kts index 4868f4f769b..e27c9431db3 100644 --- a/buildSrc/src/main/kotlin/dd-trace-java.ci-jobs.gradle.kts +++ b/buildSrc/src/main/kotlin/dd-trace-java.ci-jobs.gradle.kts @@ -1,5 +1,4 @@ import datadog.gradle.plugin.ci.isInSelectedSlot -import org.gradle.api.tasks.testing.Test import java.io.File /* @@ -14,15 +13,6 @@ if (project != rootProject) { logger.error("This plugin has been applied on a non-root project: ${project.path}") } -allprojects { - // Enable tests only on the selected slot (if -Pslot=n/t is provided) - tasks.withType().configureEach { - onlyIf("Project is in selected slot") { - project.isInSelectedSlot.get() - } - } -} - fun relativeToGitRoot(f: File): File { return rootProject.projectDir.toPath().relativize(f.absoluteFile.toPath()).toFile() }