From c4ad0ed8db0fe2ba944b749111612b54c28177e0 Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Fri, 8 May 2026 13:11:48 -0400 Subject: [PATCH 01/10] Support GitLab test execution on Linux arm64. --- .gitlab-ci.yml | 193 ++++++++++++++++++ build.gradle.kts | 19 ++ .../datadog/gradle/plugin/HostPlatform.kt | 21 ++ .../msgpack/MsgPackWriterTest.java | 3 +- .../datadog/environment/OperatingSystem.java | 10 + .../java/datadog/json/JsonReaderTest.java | 3 +- .../BuildIdExtractorIntegrationTest.java | 3 +- .../controller/EnvironmentChecker.java | 8 +- .../aerospike4/AerospikeBaseTest.groovy | 20 +- .../src/test/groovy/PubSubTest.groovy | 8 +- .../RemoteJDBCInstrumentationTest.groovy | 23 ++- .../instrumentation/protobuf-3.0/build.gradle | 27 ++- .../smoketest/AbstractSmokeTest.groovy | 7 + .../smoketest/WebSphereJmxSmokeTest.groovy | 7 + 14 files changed, 317 insertions(+), 35 deletions(-) create mode 100644 buildSrc/src/main/kotlin/datadog/gradle/plugin/HostPlatform.kt diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 369aa9b69c7..df5e8f5014a 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -38,6 +38,7 @@ stages: - benchmarks - macrobenchmarks - tests + - tests-arm64 - test-summary - exploration-tests - ci-visibility-tests @@ -54,6 +55,8 @@ variables: GRADLE_VERSION: "8.14.4" # must match gradle-wrapper.properties MAVEN_REPOSITORY_PROXY: "https://depot-read-api-java.us1.ddbuild.io/magicmirror/magicmirror/@current/" GRADLE_PLUGIN_PROXY: "https://depot-read-api-java.us1.ddbuild.io/magicmirror/magicmirror/@current/" + ARM64_BUILDER_IMAGE_REPO: "registry.ddbuild.io/images/mirror/dd-trace-java-docker-build" + ARM64_BUILDER_IMAGE_TAG: "alexeyk_arm64-test-arm64-base" BUILDER_IMAGE_REPO: "registry.ddbuild.io/images/mirror/dd-trace-java-docker-build" # images are pinned in images/mirror.lock.yaml in the DataDog/images repo BUILDER_IMAGE_VERSION_PREFIX: "ci-" # use either an empty string (e.g. "") for latest images or a version followed by a hyphen (e.g. "ci-" or "123_merge-") REPO_NOTIFICATION_CHANNEL: "#apm-java-escalations" @@ -683,6 +686,123 @@ muzzle-dep-report: - scheduler_failure - data_integrity_failure +.test_job_arm64: + image: ${ARM64_BUILDER_IMAGE_REPO}:${ARM64_BUILDER_IMAGE_TAG} + tags: [ "docker-in-docker:arm64" ] + stage: tests-arm64 + needs: [] + variables: + # arm64 docker image installs only these JVMs; override the global default regex. + DEFAULT_TEST_JVMS: /^(8|11|17|21|25)$/ + GRADLE_PARAMS: "-PskipFlakyTests" + GRADLE_WORKERS: 6 + GRADLE_MEMORY_MIN: 1G + GRADLE_MEMORY_MAX: 4G + # Sized for the arm64 microvm runner pool (smaller than amd64). + # Observed peak on equivalent amd64 workload: ~14.5 GB / ~1.5 vCPU avg. + # Bump back up if jobs OOM or we see CPU throttling in cgroup-info. + KUBERNETES_CPU_REQUEST: 6 + KUBERNETES_MEMORY_REQUEST: 12Gi + KUBERNETES_MEMORY_LIMIT: 16Gi + TESTCONTAINERS_CHECKS_DISABLE: "true" + TESTCONTAINERS_RYUK_DISABLED: "true" + TESTCONTAINERS_HUB_IMAGE_NAME_PREFIX: "registry.ddbuild.io/images/mirror/" + JETTY_AVAILABLE_PROCESSORS: 4 + GIT_SUBMODULE_STRATEGY: normal + GIT_SUBMODULE_DEPTH: 1 + rules: + - if: $testJvm =~ $DEFAULT_TEST_JVMS + when: manual + allow_failure: true + cache: + - key: dependency-$CACHE_TYPE + paths: + - .gradle/wrapper + - .gradle/caches + - .gradle/notifications + - .mvn/caches + policy: pull + fallback_keys: + - dependency-base + - dependency-lib + unprotect: true + before_script: + - git config --global --add safe.directory "$CI_PROJECT_DIR" + - export ORG_GRADLE_PROJECT_mavenRepositoryProxy=$MAVEN_REPOSITORY_PROXY + - export ORG_GRADLE_PROJECT_gradlePluginProxy=$GRADLE_PLUGIN_PROXY + - | + JAVA_HOMES=$(env | grep -E '^JAVA_[A-Z0-9_]+_HOME=' | sed 's/=.*//' | paste -sd,) + cat >> gradle.properties <()) } + // Temurin 11/21 on Linux arm64 SIGSEGVs in SystemDictionary::define_instance_class + // during CDS shared-class restore. GRADLE_OPTS / org.gradle.jvmargs only reaches the + // Gradle daemon, so every forked JVM (Test executor, JavaCompile worker daemon when + // the toolchain differs from the daemon's JDK) must disable CDS on its own. + val isLinuxArm64 = HostPlatform.isLinuxArm64() + tasks.configureEach { if (this is JavaForkOptions) { maxHeapSize = System.getProperty("datadog.forkedMaxHeapSize") @@ -91,6 +98,18 @@ allprojects { "-XX:+HeapDumpOnOutOfMemoryError", "-XX:HeapDumpPath=/tmp" ) + if (isLinuxArm64) { + jvmArgs("-Xshare:off") + } + } + } + + if (isLinuxArm64) { + tasks.withType().configureEach { + options.forkOptions.jvmArgs?.add("-Xshare:off") + } + tasks.withType().configureEach { + groovyOptions.forkOptions.jvmArgs?.add("-Xshare:off") } } } diff --git a/buildSrc/src/main/kotlin/datadog/gradle/plugin/HostPlatform.kt b/buildSrc/src/main/kotlin/datadog/gradle/plugin/HostPlatform.kt new file mode 100644 index 00000000000..40837194ecd --- /dev/null +++ b/buildSrc/src/main/kotlin/datadog/gradle/plugin/HostPlatform.kt @@ -0,0 +1,21 @@ +package datadog.gradle.plugin + +import java.util.Locale + +object HostPlatform { + @JvmStatic + fun isLinuxArm64(): Boolean = isExpectedOs("linux") && isArm64() + + @JvmStatic + fun isMacArm64(): Boolean = isExpectedOs("mac") && isArm64() + + private fun isExpectedOs(expectedOs: String): Boolean { + val osName = System.getProperty("os.name", "").lowercase(Locale.ROOT) + return osName.contains(expectedOs) + } + + private fun isArm64(): Boolean { + val osArch = System.getProperty("os.arch", "").lowercase(Locale.ROOT) + return osArch.contains("aarch64") || osArch.contains("arm64") + } +} diff --git a/communication/src/test/java/datadog/communication/serialization/msgpack/MsgPackWriterTest.java b/communication/src/test/java/datadog/communication/serialization/msgpack/MsgPackWriterTest.java index 4a3f773fdc5..fca302dd3d1 100644 --- a/communication/src/test/java/datadog/communication/serialization/msgpack/MsgPackWriterTest.java +++ b/communication/src/test/java/datadog/communication/serialization/msgpack/MsgPackWriterTest.java @@ -33,8 +33,7 @@ import org.msgpack.core.MessageUnpacker; public class MsgPackWriterTest { - // Explicit escapes for non-ASCII chars to make test independent of container settings. - private static final String NON_ASCII_STRING = "foob\u00E1r_\u263a"; // foobár_☺ + private static final String NON_ASCII_STRING = "foobár_☺"; private static final byte[] NON_ASCII_BYTES = NON_ASCII_STRING.getBytes(UTF_8); private static final int NON_ASCII_BUFFER_CAPACITY = NON_ASCII_BYTES.length + 1; diff --git a/components/environment/src/main/java/datadog/environment/OperatingSystem.java b/components/environment/src/main/java/datadog/environment/OperatingSystem.java index 1d24d7680c9..686bfca442d 100644 --- a/components/environment/src/main/java/datadog/environment/OperatingSystem.java +++ b/components/environment/src/main/java/datadog/environment/OperatingSystem.java @@ -1,5 +1,6 @@ package datadog.environment; +import static datadog.environment.OperatingSystem.Architecture.ARM64; import static datadog.environment.OperatingSystem.Type.LINUX; import static datadog.environment.OperatingSystem.Type.MACOS; import static datadog.environment.OperatingSystem.Type.WINDOWS; @@ -25,6 +26,15 @@ public final class OperatingSystem { private OperatingSystem() {} + /** + * Checks whether the architecture is arm64. + * + * @return @{@code true} if architecture is arm64, {@code false} otherwise. + */ + public static boolean isArm64() { + return ARCHITECTURE == ARM64; + } + /** * Checks whether the operating system is Linux based. * diff --git a/components/json/src/test/java/datadog/json/JsonReaderTest.java b/components/json/src/test/java/datadog/json/JsonReaderTest.java index 15a635c148a..e716a625103 100644 --- a/components/json/src/test/java/datadog/json/JsonReaderTest.java +++ b/components/json/src/test/java/datadog/json/JsonReaderTest.java @@ -215,8 +215,7 @@ void testStringEscaping() { assertEquals("\n", reader.nextString()); assertEquals("\r", reader.nextString()); assertEquals("\t", reader.nextString()); - // Explicit escape for non-ASCII `É` to make test independent of container settings. - assertEquals("\u00C9", reader.nextString()); + assertEquals("É", reader.nextString()); reader.endArray(); } catch (IOException e) { fail("Failed to read escaped JSON strings", e); diff --git a/dd-java-agent/agent-crashtracking/src/test/java/datadog/crashtracking/buildid/BuildIdExtractorIntegrationTest.java b/dd-java-agent/agent-crashtracking/src/test/java/datadog/crashtracking/buildid/BuildIdExtractorIntegrationTest.java index a0cf46ea89b..da13cbca0b5 100644 --- a/dd-java-agent/agent-crashtracking/src/test/java/datadog/crashtracking/buildid/BuildIdExtractorIntegrationTest.java +++ b/dd-java-agent/agent-crashtracking/src/test/java/datadog/crashtracking/buildid/BuildIdExtractorIntegrationTest.java @@ -1,6 +1,5 @@ package datadog.crashtracking.buildid; -import static datadog.environment.OperatingSystem.Architecture.ARM64; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -138,7 +137,7 @@ private static Stream elfBinaries() { @MethodSource("elfBinaries") void testElfBuildIdExtraction(String containerPath, String description) throws Exception { // TODO: check if arm64 can be supported too. - assumeFalse(OperatingSystem.architecture() == ARM64, "Skipping for arm64"); + assumeFalse(OperatingSystem.isArm64(), "Skipping for arm64"); Path localBinary = copyFromContainer(linuxContainer, containerPath); ElfBuildIdExtractor extractor = new ElfBuildIdExtractor(); diff --git a/dd-java-agent/agent-profiling/profiling-controller/src/main/java/com/datadog/profiling/controller/EnvironmentChecker.java b/dd-java-agent/agent-profiling/profiling-controller/src/main/java/com/datadog/profiling/controller/EnvironmentChecker.java index 62ca9ea29fe..62beaeba443 100644 --- a/dd-java-agent/agent-profiling/profiling-controller/src/main/java/com/datadog/profiling/controller/EnvironmentChecker.java +++ b/dd-java-agent/agent-profiling/profiling-controller/src/main/java/com/datadog/profiling/controller/EnvironmentChecker.java @@ -1,7 +1,5 @@ package com.datadog.profiling.controller; -import static datadog.environment.OperatingSystem.Architecture.ARM64; - import datadog.environment.JavaVirtualMachine; import datadog.environment.OperatingSystem; import datadog.environment.SystemProperties; @@ -243,11 +241,7 @@ private static boolean extractSoFromJar(Path target, StringBuilder sb) throws Ex .filter(e -> e.getName().contains("libjavaProfiler.so")) .filter( e -> - e.getName() - .contains( - OperatingSystem.architecture() == ARM64 - ? "/linux-arm64/" - : "/linux-x64/") + e.getName().contains(OperatingSystem.isArm64() ? "/linux-arm64/" : "/linux-x64/") && (!OperatingSystem.isMusl() || e.getName().contains("-musl"))) .findFirst() .map( diff --git a/dd-java-agent/instrumentation/aerospike-4.0/src/test/groovy/datadog/trace/instrumentation/aerospike4/AerospikeBaseTest.groovy b/dd-java-agent/instrumentation/aerospike-4.0/src/test/groovy/datadog/trace/instrumentation/aerospike4/AerospikeBaseTest.groovy index 57848379357..1c86d404784 100644 --- a/dd-java-agent/instrumentation/aerospike-4.0/src/test/groovy/datadog/trace/instrumentation/aerospike4/AerospikeBaseTest.groovy +++ b/dd-java-agent/instrumentation/aerospike-4.0/src/test/groovy/datadog/trace/instrumentation/aerospike4/AerospikeBaseTest.groovy @@ -1,6 +1,10 @@ package datadog.trace.instrumentation.aerospike4 +import static datadog.trace.agent.test.utils.PortUtils.waitForPortToOpen +import static java.util.concurrent.TimeUnit.SECONDS +import static org.testcontainers.containers.wait.strategy.Wait.forLogMessage +import com.github.dockerjava.api.model.Ulimit import datadog.trace.agent.test.asserts.TraceAssert import datadog.trace.agent.test.naming.VersionedNamingTestBase import datadog.trace.api.DDSpanTypes @@ -9,10 +13,6 @@ import datadog.trace.core.DDSpan import org.testcontainers.containers.GenericContainer import spock.lang.Shared -import static datadog.trace.agent.test.utils.PortUtils.waitForPortToOpen -import static java.util.concurrent.TimeUnit.SECONDS -import static org.testcontainers.containers.wait.strategy.Wait.forLogMessage - abstract class AerospikeBaseTest extends VersionedNamingTestBase { @Shared @@ -25,8 +25,14 @@ abstract class AerospikeBaseTest extends VersionedNamingTestBase { int aerospikePort = 3000 def setup() throws Exception { - aerospike = new GenericContainer('aerospike:5.5.0.9') + // arm64 supported since ce-6.2.0.2 + aerospike = new GenericContainer('aerospike:ce-6.2.0.2') .withExposedPorts(3000) + // proto-fd-max default is 15000, but container default is 1024. + // see: https://aerospike.com/docs/database/reference/config#service__proto-fd-max + .withCreateContainerCmdModifier({ cmd -> + cmd.getHostConfig().withUlimits([new Ulimit('nofile', 15000L, 15000L)] as Ulimit[]) + }) .waitingFor(forLogMessage(".*heartbeat-received.*\\n", 1)) aerospike.start() @@ -37,9 +43,7 @@ abstract class AerospikeBaseTest extends VersionedNamingTestBase { } def cleanup() throws Exception { - if (aerospike) { - aerospike.stop() - } + aerospike?.stop() } def aerospikeSpan(TraceAssert trace, int index, String methodName, Object parentSpan = null) { diff --git a/dd-java-agent/instrumentation/google-pubsub-1.116/src/test/groovy/PubSubTest.groovy b/dd-java-agent/instrumentation/google-pubsub-1.116/src/test/groovy/PubSubTest.groovy index 289822df042..07189f86b37 100644 --- a/dd-java-agent/instrumentation/google-pubsub-1.116/src/test/groovy/PubSubTest.groovy +++ b/dd-java-agent/instrumentation/google-pubsub-1.116/src/test/groovy/PubSubTest.groovy @@ -1,4 +1,3 @@ - import static datadog.trace.agent.test.utils.TraceUtils.basicSpan import com.google.api.gax.core.NoCredentialsProvider @@ -34,13 +33,12 @@ import datadog.trace.core.datastreams.StatsGroup import datadog.trace.instrumentation.grpc.client.GrpcClientDecorator import io.grpc.ManagedChannel import io.grpc.ManagedChannelBuilder +import java.nio.charset.StandardCharsets +import java.util.concurrent.CountDownLatch import org.testcontainers.containers.PubSubEmulatorContainer import org.testcontainers.utility.DockerImageName import spock.lang.Shared -import java.nio.charset.StandardCharsets -import java.util.concurrent.CountDownLatch - abstract class PubSubTest extends VersionedNamingTestBase { private static final String PROJECT_ID = "dd-trace-java" @@ -97,7 +95,7 @@ abstract class PubSubTest extends VersionedNamingTestBase { } def setupSpec() { - emulator = new PubSubEmulatorContainer(DockerImageName.parse("gcr.io/google.com/cloudsdktool/cloud-sdk:495.0.0-emulators")) + emulator = new PubSubEmulatorContainer(DockerImageName.parse("gcr.io/google.com/cloudsdktool/google-cloud-cli:495.0.0-emulators")) emulator.start() channel = ManagedChannelBuilder.forTarget(emulator.getEmulatorEndpoint()).usePlaintext().build() transportChannelProvider = FixedTransportChannelProvider.create(GrpcTransportChannel.create(channel)) diff --git a/dd-java-agent/instrumentation/jdbc/src/test/groovy/RemoteJDBCInstrumentationTest.groovy b/dd-java-agent/instrumentation/jdbc/src/test/groovy/RemoteJDBCInstrumentationTest.groovy index 616b38a456d..4fb72863371 100644 --- a/dd-java-agent/instrumentation/jdbc/src/test/groovy/RemoteJDBCInstrumentationTest.groovy +++ b/dd-java-agent/instrumentation/jdbc/src/test/groovy/RemoteJDBCInstrumentationTest.groovy @@ -2,11 +2,13 @@ import static datadog.trace.agent.test.utils.TraceUtils.basicSpan import static datadog.trace.agent.test.utils.TraceUtils.runUnderTrace import static datadog.trace.api.config.TraceInstrumentationConfig.DB_CLIENT_HOST_SPLIT_BY_INSTANCE import static datadog.trace.api.config.TraceInstrumentationConfig.DB_DBM_TRACE_PREPARED_STATEMENTS +import static org.junit.jupiter.api.Assumptions.assumeTrue import com.mchange.v2.c3p0.ComboPooledDataSource import com.microsoft.sqlserver.jdbc.SQLServerException import com.zaxxer.hikari.HikariConfig import com.zaxxer.hikari.HikariDataSource +import datadog.environment.OperatingSystem import datadog.trace.agent.test.naming.VersionedNamingTestBase import datadog.trace.agent.test.utils.PortUtils import datadog.trace.api.Config @@ -90,6 +92,10 @@ abstract class RemoteJDBCInstrumentationTest extends VersionedNamingTestBase { @Shared private Map> cpDatasources = new HashMap<>() + private static boolean dockerImageSupported(String db) { + return !(db == SQLSERVER && OperatingSystem.isArm64() && OperatingSystem.isLinux()) + } + def peerConnectionProps(String db){ def props = new Properties() props.setProperty("user", jdbcUserNames.get(db)) @@ -158,6 +164,11 @@ abstract class RemoteJDBCInstrumentationTest extends VersionedNamingTestBase { def createDS(String connectionPoolName, String dbType, String jdbcUrl) { DataSource ds = null + + if (!dockerImageSupported(dbType)) { + return ds + } + if (connectionPoolName == "tomcat") { ds = createTomcatDS(dbType, jdbcUrl) } @@ -196,10 +207,12 @@ abstract class RemoteJDBCInstrumentationTest extends VersionedNamingTestBase { jdbcUrls.put(MYSQL, "${mysql.getJdbcUrl()}") // SQLSERVER - sqlserver = new MSSQLServerContainer(MSSQLServerContainer.IMAGE).acceptLicense().withPassword(jdbcPasswords.get(SQLSERVER)) - sqlserver.start() - PortUtils.waitForPortToOpen(sqlserver.getHost(), sqlserver.getMappedPort(MSSQLServerContainer.MS_SQL_SERVER_PORT), 5, TimeUnit.SECONDS) - jdbcUrls.put(SQLSERVER, "${sqlserver.getJdbcUrl()};DatabaseName=${dbName.get(SQLSERVER)}") + if (dockerImageSupported(SQLSERVER)) { + sqlserver = new MSSQLServerContainer(MSSQLServerContainer.IMAGE).acceptLicense().withPassword(jdbcPasswords.get(SQLSERVER)) + sqlserver.start() + PortUtils.waitForPortToOpen(sqlserver.getHost(), sqlserver.getMappedPort(MSSQLServerContainer.MS_SQL_SERVER_PORT), 5, TimeUnit.SECONDS) + jdbcUrls.put(SQLSERVER, "${sqlserver.getJdbcUrl()};DatabaseName=${dbName.get(SQLSERVER)}") + } // ORACLE // Earlier Oracle version images (oracle-xe) don't work on arm64 @@ -1026,6 +1039,8 @@ abstract class RemoteJDBCInstrumentationTest extends VersionedNamingTestBase { } Connection setupConnection(String pool, String db) { + assumeTrue(dockerImageSupported(db)) + def conn = pool ? cpDatasources.get(pool).get(db).getConnection() : connectTo(db, peerConnectionProps(db)) // Clear any traces that pool or db can emmit on connection creation. diff --git a/dd-java-agent/instrumentation/protobuf-3.0/build.gradle b/dd-java-agent/instrumentation/protobuf-3.0/build.gradle index e262abdd272..173eaf3f490 100644 --- a/dd-java-agent/instrumentation/protobuf-3.0/build.gradle +++ b/dd-java-agent/instrumentation/protobuf-3.0/build.gradle @@ -1,3 +1,5 @@ +import datadog.gradle.plugin.HostPlatform + plugins { id 'com.google.protobuf' version '0.9.4' } @@ -14,13 +16,11 @@ muzzle { } } + protobuf { protoc { - def os = System.getProperty("os.name").toLowerCase() - def arch = System.getProperty("os.arch").toLowerCase() - - // There is no m1 support for protoc 3.0.0, so require Rosetta - if (os.contains("mac") && arch.contains("aarch64")) { + // There is no macOS arm64 support for protoc 3.0.0, so require Rosetta. + if (HostPlatform.isMacArm64()) { artifact = "com.google.protobuf:protoc:3.0.0:osx-x86_64" } else { artifact = "com.google.protobuf:protoc:3.0.0" @@ -28,6 +28,23 @@ protobuf { } } +// protobuf supports arm64 linux since 3.12.0, but it is not source-compatible with 3.0.0 +if (HostPlatform.isLinuxArm64()) { + tasks.matching { + it.name in [ + "extractTestProto", + "generateTestProto", + "compileTestJava", + "compileTestGroovy", + "processTestResources", + "testClasses", + "test" + ] + }.configureEach { + enabled = false + } +} + dependencies { compileOnly group: 'com.google.protobuf', name: 'protobuf-java', version: '3.0.0' diff --git a/dd-smoke-tests/src/main/groovy/datadog/smoketest/AbstractSmokeTest.groovy b/dd-smoke-tests/src/main/groovy/datadog/smoketest/AbstractSmokeTest.groovy index 25aba9b9660..0d740d17e41 100644 --- a/dd-smoke-tests/src/main/groovy/datadog/smoketest/AbstractSmokeTest.groovy +++ b/dd-smoke-tests/src/main/groovy/datadog/smoketest/AbstractSmokeTest.groovy @@ -252,6 +252,13 @@ abstract class AbstractSmokeTest extends ProcessManager { // Unlike crash tracking smoke test, keep the default delay; otherwise, otherwise other tests will fail // ret += "-Ddd.dogstatsd.start-delay=0" } + + // Disable CDS on Linux arm64: Temurin 11.0.31 / 21.0.10 hit a SIGSEGV during + // shared-class restore (ClassLoaderData::add_handle, Klass::class_loader) + // before any user code runs. + if (OperatingSystem.isArm64() && OperatingSystem.isLinux()) { + ret += "-Xshare:off" + } ret as String[] } diff --git a/dd-smoke-tests/websphere-jmx/src/test/groovy/datadog/smoketest/WebSphereJmxSmokeTest.groovy b/dd-smoke-tests/websphere-jmx/src/test/groovy/datadog/smoketest/WebSphereJmxSmokeTest.groovy index 28b72c111c7..471716b203d 100644 --- a/dd-smoke-tests/websphere-jmx/src/test/groovy/datadog/smoketest/WebSphereJmxSmokeTest.groovy +++ b/dd-smoke-tests/websphere-jmx/src/test/groovy/datadog/smoketest/WebSphereJmxSmokeTest.groovy @@ -1,6 +1,7 @@ package datadog.smoketest +import datadog.environment.OperatingSystem import java.time.Duration import java.util.concurrent.ArrayBlockingQueue import java.util.concurrent.BlockingQueue @@ -11,6 +12,7 @@ import org.testcontainers.containers.GenericContainer import org.testcontainers.containers.output.Slf4jLogConsumer import org.testcontainers.containers.wait.strategy.Wait import org.testcontainers.utility.MountableFile +import spock.lang.IgnoreIf import spock.lang.Shared /** @@ -22,6 +24,11 @@ import spock.lang.Shared * * Note that the websphere related metrics will only arrive if our instrumentation is applied. */ +// IBM does not publish an arm64 build of icr.io/appcafe/websphere-traditional, and the +// arm64 CI runner has no amd64 emulation configured, so the container fails to start. +@IgnoreIf({ + OperatingSystem.isArm64() && OperatingSystem.isLinux() +}) class WebSphereJmxSmokeTest extends AbstractSmokeTest { private static final Logger LOG = LoggerFactory.getLogger(WebSphereJmxSmokeTest) From 2901752e24add2d0206a7b6892535cdb0f1e034f Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Fri, 8 May 2026 14:39:25 -0400 Subject: [PATCH 02/10] Fixed netty on arm64 --- .../java-concurrent-1.8/build.gradle | 8 ++++++++ .../NettyExecutorInstrumentationTest.groovy | 14 ++++++-------- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/dd-java-agent/instrumentation/java/java-concurrent/java-concurrent-1.8/build.gradle b/dd-java-agent/instrumentation/java/java-concurrent/java-concurrent-1.8/build.gradle index ee85bef77fa..5097bfb89ab 100644 --- a/dd-java-agent/instrumentation/java/java-concurrent/java-concurrent-1.8/build.gradle +++ b/dd-java-agent/instrumentation/java/java-concurrent/java-concurrent-1.8/build.gradle @@ -1,3 +1,5 @@ +import datadog.gradle.plugin.HostPlatform + muzzle { pass { coreJdk() @@ -17,6 +19,12 @@ dependencies { testImplementation libs.guava testImplementation group: 'io.netty', name: 'netty-all', version: '4.1.9.Final' testImplementation group: 'org.apache.tomcat.embed', name: 'tomcat-embed-core', version: '7.0.0' + + if (HostPlatform.isLinuxArm64()) { + // netty-all 4.1.9 only ships the linux-x86_64 epoll native; pull the aarch64 one explicitly. + testImplementation group: 'io.netty', name: 'netty-transport-native-epoll', version: '4.1.108.Final', classifier: 'linux-aarch_64' + } + // Tomcat 10.1.+ seems to require Java 11. Limit to fix build. // TODO: Tomcat 10.0.10 has a copy of the JSR166 ThreadPoolExecutor so it needs special instrumentation latestDepTestImplementation group: 'org.apache.tomcat.embed', name: 'tomcat-embed-core', version: '10.0.8' diff --git a/dd-java-agent/instrumentation/java/java-concurrent/java-concurrent-1.8/src/test/groovy/executor/NettyExecutorInstrumentationTest.groovy b/dd-java-agent/instrumentation/java/java-concurrent/java-concurrent-1.8/src/test/groovy/executor/NettyExecutorInstrumentationTest.groovy index fc944416d94..d06fcdc6487 100644 --- a/dd-java-agent/instrumentation/java/java-concurrent/java-concurrent-1.8/src/test/groovy/executor/NettyExecutorInstrumentationTest.groovy +++ b/dd-java-agent/instrumentation/java/java-concurrent/java-concurrent-1.8/src/test/groovy/executor/NettyExecutorInstrumentationTest.groovy @@ -1,5 +1,8 @@ package executor +import static org.junit.jupiter.api.Assumptions.assumeTrue + +import datadog.environment.OperatingSystem import datadog.trace.agent.test.InstrumentationSpecification import datadog.trace.api.Trace import datadog.trace.core.DDSpan @@ -8,22 +11,17 @@ import io.netty.channel.epoll.EpollEventLoopGroup import io.netty.channel.local.LocalEventLoopGroup import io.netty.channel.nio.NioEventLoopGroup import io.netty.util.concurrent.DefaultEventExecutorGroup -import runnable.JavaAsyncChild -import spock.lang.Shared - import java.lang.reflect.InvocationTargetException import java.util.concurrent.Callable import java.util.concurrent.Future import java.util.concurrent.RejectedExecutionException import java.util.concurrent.TimeUnit - -import static org.junit.jupiter.api.Assumptions.assumeTrue +import runnable.JavaAsyncChild +import spock.lang.Shared class NettyExecutorInstrumentationTest extends InstrumentationSpecification { - @Shared - boolean isLinux = System.getProperty("os.name").toLowerCase().contains("linux") - + boolean isLinux = OperatingSystem.isLinux() @Shared EpollEventLoopGroup epollEventLoopGroup = isLinux ? new EpollEventLoopGroup(4) : null @Shared From 3506190b6c82e23034d83e1d761c3b9891742869 Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Fri, 8 May 2026 15:05:46 -0400 Subject: [PATCH 03/10] Fixed Java crashes on arm64 Linux during CDS --- .../plugin/instrument/InstrumentPostProcessingAction.kt | 5 +++++ .../kotlin/datadog/gradle/plugin/muzzle/tasks/MuzzleTask.kt | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/buildSrc/src/main/kotlin/datadog/gradle/plugin/instrument/InstrumentPostProcessingAction.kt b/buildSrc/src/main/kotlin/datadog/gradle/plugin/instrument/InstrumentPostProcessingAction.kt index b8c3e300afd..3f70bac7dec 100644 --- a/buildSrc/src/main/kotlin/datadog/gradle/plugin/instrument/InstrumentPostProcessingAction.kt +++ b/buildSrc/src/main/kotlin/datadog/gradle/plugin/instrument/InstrumentPostProcessingAction.kt @@ -1,5 +1,6 @@ package datadog.gradle.plugin.instrument +import datadog.gradle.plugin.HostPlatform import datadog.gradle.plugin.instrument.BuildTimeInstrumentationPlugin.Companion.BUILD_TIME_INSTRUMENTATION_PLUGIN_CONFIGURATION import org.gradle.api.Action import org.gradle.api.Project @@ -73,6 +74,10 @@ abstract class InstrumentPostProcessingAction @Inject constructor( return workerExecutor.processIsolation { forkOptions { setExecutable(javaLauncher.executablePath.asFile.absolutePath) + if (HostPlatform.isLinuxArm64()) { + // Temurin on the arm64 Linux can crash during CDS shared-class restore; + jvmArgs("-Xshare:off") + } } } } diff --git a/buildSrc/src/main/kotlin/datadog/gradle/plugin/muzzle/tasks/MuzzleTask.kt b/buildSrc/src/main/kotlin/datadog/gradle/plugin/muzzle/tasks/MuzzleTask.kt index 2d5d830ea3b..08d42673298 100644 --- a/buildSrc/src/main/kotlin/datadog/gradle/plugin/muzzle/tasks/MuzzleTask.kt +++ b/buildSrc/src/main/kotlin/datadog/gradle/plugin/muzzle/tasks/MuzzleTask.kt @@ -1,5 +1,6 @@ package datadog.gradle.plugin.muzzle.tasks +import datadog.gradle.plugin.HostPlatform import datadog.gradle.plugin.muzzle.MuzzleAction import datadog.gradle.plugin.muzzle.MuzzleDirective import datadog.gradle.plugin.muzzle.MuzzleExtension @@ -102,6 +103,10 @@ abstract class MuzzleTask @Inject constructor( if(javaLauncher.metadata.languageVersion > JavaLanguageVersion.of(9)) { jvmArgs("--add-opens=java.base/java.lang=ALL-UNNAMED") } + if (HostPlatform.isLinuxArm64()) { + // Temurin on the arm64 Linux can crash during CDS shared-class restore; + jvmArgs("-Xshare:off") + } executable(javaLauncher.executablePath) } } From bf4f3fb3ee2b053866b8d4d442edf61381f2aadc Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Fri, 8 May 2026 15:11:41 -0400 Subject: [PATCH 04/10] Fixed RestletTest on arm64 Linux --- .../src/latestDepTest/groovy/RestletTest.groovy | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/dd-java-agent/instrumentation/restlet-2.2/src/latestDepTest/groovy/RestletTest.groovy b/dd-java-agent/instrumentation/restlet-2.2/src/latestDepTest/groovy/RestletTest.groovy index 3217e53394d..2c308efeb29 100644 --- a/dd-java-agent/instrumentation/restlet-2.2/src/latestDepTest/groovy/RestletTest.groovy +++ b/dd-java-agent/instrumentation/restlet-2.2/src/latestDepTest/groovy/RestletTest.groovy @@ -1,3 +1,6 @@ +import static datadog.environment.OperatingSystem.isArm64 +import static datadog.environment.OperatingSystem.isLinux + import org.restlet.Request import org.restlet.Response import org.restlet.data.Header @@ -6,6 +9,12 @@ import org.restlet.util.Series class RestletTest extends RestletTestBase { + @Override + boolean testParallelRequest() { + // TODO: Parallel processing is failing on Linux arm64. + return !(isLinux() && isArm64()) + } + @Override protected Filter createHeaderFilter() { return new ResponseHeaderFilter() From 75eaaf7579fbdf43672645706674bafeeb4713e5 Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Fri, 8 May 2026 15:23:48 -0400 Subject: [PATCH 05/10] Fixed debugger on Linux arm64 --- .../smoketest/ProcessBuilderHelper.java | 27 ++++++++++++------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/dd-smoke-tests/debugger-integration-tests/src/test/java/datadog/smoketest/ProcessBuilderHelper.java b/dd-smoke-tests/debugger-integration-tests/src/test/java/datadog/smoketest/ProcessBuilderHelper.java index 5faf29bbaa5..7947a8badf8 100644 --- a/dd-smoke-tests/debugger-integration-tests/src/test/java/datadog/smoketest/ProcessBuilderHelper.java +++ b/dd-smoke-tests/debugger-integration-tests/src/test/java/datadog/smoketest/ProcessBuilderHelper.java @@ -1,5 +1,6 @@ package datadog.smoketest; +import datadog.environment.OperatingSystem; import java.io.File; import java.nio.file.Path; import java.util.ArrayList; @@ -43,16 +44,22 @@ public static ProcessBuilder createProcessBuilder( + System.nanoTime(); List baseCommand = - Arrays.asList( - javaPath(), - // "-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=localhost:5006", - "-Xmx" + System.getProperty("datadog.forkedMaxHeapSize", "1024M"), - "-Xms" + System.getProperty("datadog.forkedMinHeapSize", "64M"), - "-javaagent:" + agentShadowJar(), - "-XX:ErrorFile=/tmp/hs_err_pid%p.log", - "-Ddd.env=smoketest", - "-Ddd.version=99", - "-Djava.util.prefs.userRoot=" + prefsDir); + new ArrayList<>( + Arrays.asList( + javaPath(), + // "-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=localhost:5006", + "-Xmx" + System.getProperty("datadog.forkedMaxHeapSize", "1024M"), + "-Xms" + System.getProperty("datadog.forkedMinHeapSize", "64M"), + "-javaagent:" + agentShadowJar(), + "-XX:ErrorFile=/tmp/hs_err_pid%p.log", + "-Ddd.env=smoketest", + "-Ddd.version=99", + "-Djava.util.prefs.userRoot=" + prefsDir)); + + if (OperatingSystem.isLinux() && OperatingSystem.isArm64()) { + // Temurin on the arm64 Linux can crash during CDS shared-class restore; + baseCommand.add(1, "-Xshare:off"); + } List command = new ArrayList<>(); command.addAll(baseCommand); From 50c71e5a1a561209ad1247b689350b8013fa88e2 Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Fri, 8 May 2026 15:41:15 -0400 Subject: [PATCH 06/10] Fixed ByteBuddy issue on arm64: `Could not self-attach to current VM`. --- .../dd-trace-java.test-jvm-constraints.gradle.kts | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/buildSrc/src/main/kotlin/dd-trace-java.test-jvm-constraints.gradle.kts b/buildSrc/src/main/kotlin/dd-trace-java.test-jvm-constraints.gradle.kts index aa4724183b0..dfe3cfa52c2 100644 --- a/buildSrc/src/main/kotlin/dd-trace-java.test-jvm-constraints.gradle.kts +++ b/buildSrc/src/main/kotlin/dd-trace-java.test-jvm-constraints.gradle.kts @@ -1,3 +1,4 @@ +import datadog.gradle.plugin.HostPlatform import datadog.gradle.plugin.testJvmConstraints.ProvideJvmArgsOnJvmLauncherVersion import datadog.gradle.plugin.testJvmConstraints.TestJvmConstraintsExtension import datadog.gradle.plugin.testJvmConstraints.TestJvmConstraintsExtension.Companion.TEST_JVM_CONSTRAINTS @@ -79,6 +80,16 @@ private fun Test.configureTestJvm(extension: TestJvmConstraintsExtension) { ), extension.allowReflectiveAccessToJdk ) + + // Fix for arm64 Linux ByteBuddy error: + // "Could not self-attach to current VM using external process" + // Why it is needed on arm64: probably arm64 GitLab runners has stricter rules applied. + if (HostPlatform.isLinuxArm64()) { + conditionalJvmArgs( + JavaVersion.VERSION_1_9, + listOf("-Djdk.attach.allowAttachSelf=true") + ) + } } // Jacoco plugin is not applied on every project From 71c1bb05ab5a6959758a1381391b9e13f3b1db13 Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Fri, 8 May 2026 15:42:18 -0400 Subject: [PATCH 07/10] Optimize K8s resources. --- .gitlab-ci.yml | 33 ++++++++++++++++++++++++--------- 1 file changed, 24 insertions(+), 9 deletions(-) diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index df5e8f5014a..147d26c3772 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -168,6 +168,24 @@ default: echo -e "${TEXT_BOLD}${TEXT_YELLOW} Containers:${TEXT_CLEAR} https://app.datadoghq.com/containers?${TIME_PARAMS}query=image_name%3A%2A%2Fdatadog%2Fdd-trace-java-docker-build%20AND%20pod_name%3A${POD_NAME}&live=false" echo -e "${TEXT_BOLD}${TEXT_YELLOW} Processes:${TEXT_CLEAR} https://app.datadoghq.com/process?${TIME_PARAMS}query=image_name%3A%2A%2Fdatadog%2Fdd-trace-java-docker-build%20AND%20pod_name%3A${POD_NAME}&live=false" +.tier_m: + variables: &tier_m_variables + GRADLE_WORKERS: 4 + GRADLE_MEMORY_MIN: 1G + GRADLE_MEMORY_MAX: 5G + KUBERNETES_CPU_REQUEST: 6 + KUBERNETES_MEMORY_REQUEST: 16Gi + KUBERNETES_MEMORY_LIMIT: 16Gi + +.tier_l: + variables: &tier_l_variables + GRADLE_WORKERS: 6 + GRADLE_MEMORY_MIN: 1G + GRADLE_MEMORY_MAX: 6G + KUBERNETES_CPU_REQUEST: 10 + KUBERNETES_MEMORY_REQUEST: 20Gi + KUBERNETES_MEMORY_LIMIT: 20Gi + .gitlab_base_ref_params: &gitlab_base_ref_params - | export GIT_BASE_REF=$(.gitlab/find-gh-base-ref.sh) @@ -692,18 +710,10 @@ muzzle-dep-report: stage: tests-arm64 needs: [] variables: + <<: *tier_m_variables # arm64 docker image installs only these JVMs; override the global default regex. DEFAULT_TEST_JVMS: /^(8|11|17|21|25)$/ GRADLE_PARAMS: "-PskipFlakyTests" - GRADLE_WORKERS: 6 - GRADLE_MEMORY_MIN: 1G - GRADLE_MEMORY_MAX: 4G - # Sized for the arm64 microvm runner pool (smaller than amd64). - # Observed peak on equivalent amd64 workload: ~14.5 GB / ~1.5 vCPU avg. - # Bump back up if jobs OOM or we see CPU throttling in cgroup-info. - KUBERNETES_CPU_REQUEST: 6 - KUBERNETES_MEMORY_REQUEST: 12Gi - KUBERNETES_MEMORY_LIMIT: 16Gi TESTCONTAINERS_CHECKS_DISABLE: "true" TESTCONTAINERS_RYUK_DISABLED: "true" TESTCONTAINERS_HUB_IMAGE_NAME_PREFIX: "registry.ddbuild.io/images/mirror/" @@ -869,6 +879,7 @@ test_inst: test_inst_arm64: extends: .test_job_arm64_with_test_agent variables: + <<: *tier_l_variables GRADLE_TARGET: ":instrumentationTest" CACHE_TYPE: "inst" parallel: @@ -889,6 +900,7 @@ test_inst_latest: test_inst_latest_arm64: extends: .test_job_arm64_with_test_agent variables: + <<: *tier_l_variables GRADLE_TARGET: ":instrumentationLatestDepTest" CACHE_TYPE: "latestdep" parallel: @@ -977,6 +989,7 @@ test_smoke: test_smoke_arm64: extends: .test_job_arm64 variables: + <<: *tier_l_variables GRADLE_TARGET: "stageMainDist :smokeTest" GRADLE_PARAMS: "-PskipFlakyTests" CACHE_TYPE: "smoke" @@ -1003,6 +1016,7 @@ test_ssi_smoke: test_ssi_smoke_arm64: extends: .test_job_arm64 variables: + <<: *tier_l_variables GRADLE_TARGET: "stageMainDist :smokeTest" CACHE_TYPE: "smoke" DD_INJECT_FORCE: "true" @@ -1025,6 +1039,7 @@ test_smoke_graalvm: test_smoke_graalvm_arm64: extends: .test_job_arm64 variables: + <<: *tier_l_variables GRADLE_TARGET: "stageMainDist :dd-smoke-test:spring-boot-3.0-native:test" CACHE_TYPE: "smoke" CI_NO_SPLIT: "true" From 79715861961950aafdcdd8581f2a64f3c2860220 Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Fri, 8 May 2026 18:44:14 -0400 Subject: [PATCH 08/10] Fixed netty on arm64 --- .../java-concurrent-1.8/build.gradle | 11 ++----- .../java-concurrent-1.8/gradle.lockfile | 32 ++++++++++++++++++- .../RunnableFutureInstrumentation.java | 9 ++++++ 3 files changed, 42 insertions(+), 10 deletions(-) diff --git a/dd-java-agent/instrumentation/java/java-concurrent/java-concurrent-1.8/build.gradle b/dd-java-agent/instrumentation/java/java-concurrent/java-concurrent-1.8/build.gradle index 5097bfb89ab..98ba8af8c3a 100644 --- a/dd-java-agent/instrumentation/java/java-concurrent/java-concurrent-1.8/build.gradle +++ b/dd-java-agent/instrumentation/java/java-concurrent/java-concurrent-1.8/build.gradle @@ -1,5 +1,3 @@ -import datadog.gradle.plugin.HostPlatform - muzzle { pass { coreJdk() @@ -17,16 +15,11 @@ addTestSuiteForDir('latestDepTest', 'test') dependencies { testImplementation project(':dd-java-agent:instrumentation:datadog:tracing:trace-annotation') testImplementation libs.guava - testImplementation group: 'io.netty', name: 'netty-all', version: '4.1.9.Final' + // netty-all ships with the linux arm64 libs only since 4.1.50+ + testImplementation group: 'io.netty', name: 'netty-all', version: '4.1.50.Final' testImplementation group: 'org.apache.tomcat.embed', name: 'tomcat-embed-core', version: '7.0.0' - if (HostPlatform.isLinuxArm64()) { - // netty-all 4.1.9 only ships the linux-x86_64 epoll native; pull the aarch64 one explicitly. - testImplementation group: 'io.netty', name: 'netty-transport-native-epoll', version: '4.1.108.Final', classifier: 'linux-aarch_64' - } - // Tomcat 10.1.+ seems to require Java 11. Limit to fix build. // TODO: Tomcat 10.0.10 has a copy of the JSR166 ThreadPoolExecutor so it needs special instrumentation latestDepTestImplementation group: 'org.apache.tomcat.embed', name: 'tomcat-embed-core', version: '10.0.8' } - diff --git a/dd-java-agent/instrumentation/java/java-concurrent/java-concurrent-1.8/gradle.lockfile b/dd-java-agent/instrumentation/java/java-concurrent/java-concurrent-1.8/gradle.lockfile index e918d10d18f..fc190d59b60 100644 --- a/dd-java-agent/instrumentation/java/java-concurrent/java-concurrent-1.8/gradle.lockfile +++ b/dd-java-agent/instrumentation/java/java-concurrent/java-concurrent-1.8/gradle.lockfile @@ -47,7 +47,37 @@ commons-io:commons-io:2.11.0=latestDepTestCompileClasspath,latestDepTestRuntimeC commons-io:commons-io:2.20.0=spotbugs de.thetaphi:forbiddenapis:3.10=compileClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath io.leangen.geantyref:geantyref:1.3.16=latestDepTestRuntimeClasspath,testRuntimeClasspath -io.netty:netty-all:4.1.9.Final=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +io.netty:netty-all:4.1.133.Final=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +io.netty:netty-buffer:4.1.133.Final=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +io.netty:netty-codec-dns:4.1.133.Final=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +io.netty:netty-codec-haproxy:4.1.133.Final=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +io.netty:netty-codec-http2:4.1.133.Final=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +io.netty:netty-codec-http:4.1.133.Final=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +io.netty:netty-codec-memcache:4.1.133.Final=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +io.netty:netty-codec-mqtt:4.1.133.Final=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +io.netty:netty-codec-redis:4.1.133.Final=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +io.netty:netty-codec-smtp:4.1.133.Final=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +io.netty:netty-codec-socks:4.1.133.Final=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +io.netty:netty-codec-stomp:4.1.133.Final=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +io.netty:netty-codec-xml:4.1.133.Final=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +io.netty:netty-codec:4.1.133.Final=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +io.netty:netty-common:4.1.133.Final=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +io.netty:netty-handler-proxy:4.1.133.Final=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +io.netty:netty-handler-ssl-ocsp:4.1.133.Final=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +io.netty:netty-handler:4.1.133.Final=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +io.netty:netty-resolver-dns-classes-macos:4.1.133.Final=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +io.netty:netty-resolver-dns-native-macos:4.1.133.Final=latestDepTestRuntimeClasspath,testRuntimeClasspath +io.netty:netty-resolver-dns:4.1.133.Final=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +io.netty:netty-resolver:4.1.133.Final=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +io.netty:netty-transport-classes-epoll:4.1.133.Final=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +io.netty:netty-transport-classes-kqueue:4.1.133.Final=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +io.netty:netty-transport-native-epoll:4.1.133.Final=latestDepTestRuntimeClasspath,testRuntimeClasspath +io.netty:netty-transport-native-kqueue:4.1.133.Final=latestDepTestRuntimeClasspath,testRuntimeClasspath +io.netty:netty-transport-native-unix-common:4.1.133.Final=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +io.netty:netty-transport-rxtx:4.1.133.Final=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +io.netty:netty-transport-sctp:4.1.133.Final=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +io.netty:netty-transport-udt:4.1.133.Final=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +io.netty:netty-transport:4.1.133.Final=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath io.sqreen:libsqreen:17.3.0=latestDepTestRuntimeClasspath,testRuntimeClasspath javax.servlet:javax.servlet-api:3.1.0=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath jaxen:jaxen:2.0.0=spotbugs diff --git a/dd-java-agent/instrumentation/java/java-concurrent/java-concurrent-1.8/src/main/java/datadog/trace/instrumentation/java/concurrent/runnable/RunnableFutureInstrumentation.java b/dd-java-agent/instrumentation/java/java-concurrent/java-concurrent-1.8/src/main/java/datadog/trace/instrumentation/java/concurrent/runnable/RunnableFutureInstrumentation.java index a783258c421..e069b09341d 100644 --- a/dd-java-agent/instrumentation/java/java-concurrent/java-concurrent-1.8/src/main/java/datadog/trace/instrumentation/java/concurrent/runnable/RunnableFutureInstrumentation.java +++ b/dd-java-agent/instrumentation/java/java-concurrent/java-concurrent-1.8/src/main/java/datadog/trace/instrumentation/java/concurrent/runnable/RunnableFutureInstrumentation.java @@ -30,7 +30,9 @@ import java.util.Collection; import java.util.Map; import java.util.concurrent.Callable; +import java.util.concurrent.Delayed; import java.util.concurrent.RunnableFuture; +import java.util.concurrent.TimeUnit; import net.bytebuddy.asm.Advice; import net.bytebuddy.description.type.TypeDescription; import net.bytebuddy.matcher.ElementMatcher; @@ -147,6 +149,13 @@ public static void captureScope(@Advice.This RunnableFuture task) { public static final class Run { @Advice.OnMethodEnter public static AgentScope activate(@Advice.This RunnableFuture task) { + // Newer Netty versions may execute a delayed ScheduledFutureTask once only to move it into + // the scheduled queue. Keep the continuation for the run that actually invokes the task. + if (task instanceof Delayed + && task.getClass().getName().endsWith(".netty.util.concurrent.ScheduledFutureTask") + && ((Delayed) task).getDelay(TimeUnit.NANOSECONDS) > 0) { + return null; + } return startTaskScope(InstrumentationContext.get(RunnableFuture.class, State.class), task); } From 0384ca29af6ef289cbe88966de21c09773f17607 Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Fri, 8 May 2026 18:57:29 -0400 Subject: [PATCH 09/10] Fix for play gradle plugin on arm64 --- dd-smoke-tests/play-2.4/build.gradle | 40 ++++++++++++++++++++++++++++ 1 file changed, 40 insertions(+) diff --git a/dd-smoke-tests/play-2.4/build.gradle b/dd-smoke-tests/play-2.4/build.gradle index 1a0de08b58b..4645fd91e74 100644 --- a/dd-smoke-tests/play-2.4/build.gradle +++ b/dd-smoke-tests/play-2.4/build.gradle @@ -1,3 +1,9 @@ +import datadog.gradle.plugin.HostPlatform +import org.gradle.playframework.tasks.RoutesCompile +import org.gradle.playframework.tasks.internal.RoutesCompileWorkAction +import org.gradle.playframework.tools.internal.routes.DefaultRoutesCompileSpec +import org.gradle.playframework.tools.internal.routes.RoutesCompilerFactory + plugins { id 'org.gradle.playframework' } @@ -93,6 +99,40 @@ tasks.withType(AbstractCopyTask).configureEach { duplicatesStrategy = DuplicatesStrategy.EXCLUDE } +if (HostPlatform.isLinuxArm64()) { + tasks.named("compilePlayRoutes", RoutesCompile) { RoutesCompile task -> + // org.gradle.playframework.tasks.RoutesCompile does not expose worker fork options, but its + // worker JVM needs CDS disabled on Linux arm64 like the other forked JVMs in this build. + def workerExecutorField = RoutesCompile.getDeclaredField("workerExecutor") + workerExecutorField.accessible = true + + task.actions.clear() + task.doLast { + def spec = new DefaultRoutesCompileSpec( + task.source.files, + task.outputDirectory.get().asFile, + task.javaProject, + task.namespaceReverseRouter.get(), + task.generateReverseRoutes.get(), + task.injectedRoutesGenerator.get(), + task.additionalImports.get(), + projectDir + ) + + def workerExecutor = workerExecutorField.get(task) + workerExecutor.processIsolation { workerSpec -> + workerSpec.forkOptions { options -> + options.jvmArgs("-XX:MaxMetaspaceSize=256m", "-Xshare:off") + } + workerSpec.classpath.from(task.routesCompilerClasspath) + }.submit(RoutesCompileWorkAction) { parameters -> + parameters.compiler.set(RoutesCompilerFactory.create(task.platform.get())) + parameters.spec.set(spec) + } + } + } +} + spotless { java { target "**/*.java" From 5dca385ab7ec54b6a157cfb0e771d4eb782ecdd1 Mon Sep 17 00:00:00 2001 From: Alexey Kuznetsov Date: Fri, 8 May 2026 21:35:22 -0400 Subject: [PATCH 10/10] Revert changes and simply skip netty test on arm64 linux. --- .../java-concurrent-1.8/build.gradle | 4 +-- .../java-concurrent-1.8/gradle.lockfile | 32 +------------------ .../RunnableFutureInstrumentation.java | 9 ------ .../NettyExecutorInstrumentationTest.groovy | 7 ++++ 4 files changed, 9 insertions(+), 43 deletions(-) diff --git a/dd-java-agent/instrumentation/java/java-concurrent/java-concurrent-1.8/build.gradle b/dd-java-agent/instrumentation/java/java-concurrent/java-concurrent-1.8/build.gradle index 98ba8af8c3a..83caaeb1324 100644 --- a/dd-java-agent/instrumentation/java/java-concurrent/java-concurrent-1.8/build.gradle +++ b/dd-java-agent/instrumentation/java/java-concurrent/java-concurrent-1.8/build.gradle @@ -15,10 +15,8 @@ addTestSuiteForDir('latestDepTest', 'test') dependencies { testImplementation project(':dd-java-agent:instrumentation:datadog:tracing:trace-annotation') testImplementation libs.guava - // netty-all ships with the linux arm64 libs only since 4.1.50+ - testImplementation group: 'io.netty', name: 'netty-all', version: '4.1.50.Final' + testImplementation group: 'io.netty', name: 'netty-all', version: '4.1.9.Final' testImplementation group: 'org.apache.tomcat.embed', name: 'tomcat-embed-core', version: '7.0.0' - // Tomcat 10.1.+ seems to require Java 11. Limit to fix build. // TODO: Tomcat 10.0.10 has a copy of the JSR166 ThreadPoolExecutor so it needs special instrumentation latestDepTestImplementation group: 'org.apache.tomcat.embed', name: 'tomcat-embed-core', version: '10.0.8' diff --git a/dd-java-agent/instrumentation/java/java-concurrent/java-concurrent-1.8/gradle.lockfile b/dd-java-agent/instrumentation/java/java-concurrent/java-concurrent-1.8/gradle.lockfile index fc190d59b60..e918d10d18f 100644 --- a/dd-java-agent/instrumentation/java/java-concurrent/java-concurrent-1.8/gradle.lockfile +++ b/dd-java-agent/instrumentation/java/java-concurrent/java-concurrent-1.8/gradle.lockfile @@ -47,37 +47,7 @@ commons-io:commons-io:2.11.0=latestDepTestCompileClasspath,latestDepTestRuntimeC commons-io:commons-io:2.20.0=spotbugs de.thetaphi:forbiddenapis:3.10=compileClasspath,latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath io.leangen.geantyref:geantyref:1.3.16=latestDepTestRuntimeClasspath,testRuntimeClasspath -io.netty:netty-all:4.1.133.Final=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -io.netty:netty-buffer:4.1.133.Final=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -io.netty:netty-codec-dns:4.1.133.Final=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -io.netty:netty-codec-haproxy:4.1.133.Final=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -io.netty:netty-codec-http2:4.1.133.Final=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -io.netty:netty-codec-http:4.1.133.Final=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -io.netty:netty-codec-memcache:4.1.133.Final=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -io.netty:netty-codec-mqtt:4.1.133.Final=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -io.netty:netty-codec-redis:4.1.133.Final=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -io.netty:netty-codec-smtp:4.1.133.Final=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -io.netty:netty-codec-socks:4.1.133.Final=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -io.netty:netty-codec-stomp:4.1.133.Final=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -io.netty:netty-codec-xml:4.1.133.Final=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -io.netty:netty-codec:4.1.133.Final=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -io.netty:netty-common:4.1.133.Final=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -io.netty:netty-handler-proxy:4.1.133.Final=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -io.netty:netty-handler-ssl-ocsp:4.1.133.Final=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -io.netty:netty-handler:4.1.133.Final=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -io.netty:netty-resolver-dns-classes-macos:4.1.133.Final=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -io.netty:netty-resolver-dns-native-macos:4.1.133.Final=latestDepTestRuntimeClasspath,testRuntimeClasspath -io.netty:netty-resolver-dns:4.1.133.Final=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -io.netty:netty-resolver:4.1.133.Final=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -io.netty:netty-transport-classes-epoll:4.1.133.Final=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -io.netty:netty-transport-classes-kqueue:4.1.133.Final=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -io.netty:netty-transport-native-epoll:4.1.133.Final=latestDepTestRuntimeClasspath,testRuntimeClasspath -io.netty:netty-transport-native-kqueue:4.1.133.Final=latestDepTestRuntimeClasspath,testRuntimeClasspath -io.netty:netty-transport-native-unix-common:4.1.133.Final=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -io.netty:netty-transport-rxtx:4.1.133.Final=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -io.netty:netty-transport-sctp:4.1.133.Final=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -io.netty:netty-transport-udt:4.1.133.Final=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath -io.netty:netty-transport:4.1.133.Final=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath +io.netty:netty-all:4.1.9.Final=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath io.sqreen:libsqreen:17.3.0=latestDepTestRuntimeClasspath,testRuntimeClasspath javax.servlet:javax.servlet-api:3.1.0=latestDepTestCompileClasspath,latestDepTestRuntimeClasspath,testCompileClasspath,testRuntimeClasspath jaxen:jaxen:2.0.0=spotbugs diff --git a/dd-java-agent/instrumentation/java/java-concurrent/java-concurrent-1.8/src/main/java/datadog/trace/instrumentation/java/concurrent/runnable/RunnableFutureInstrumentation.java b/dd-java-agent/instrumentation/java/java-concurrent/java-concurrent-1.8/src/main/java/datadog/trace/instrumentation/java/concurrent/runnable/RunnableFutureInstrumentation.java index e069b09341d..a783258c421 100644 --- a/dd-java-agent/instrumentation/java/java-concurrent/java-concurrent-1.8/src/main/java/datadog/trace/instrumentation/java/concurrent/runnable/RunnableFutureInstrumentation.java +++ b/dd-java-agent/instrumentation/java/java-concurrent/java-concurrent-1.8/src/main/java/datadog/trace/instrumentation/java/concurrent/runnable/RunnableFutureInstrumentation.java @@ -30,9 +30,7 @@ import java.util.Collection; import java.util.Map; import java.util.concurrent.Callable; -import java.util.concurrent.Delayed; import java.util.concurrent.RunnableFuture; -import java.util.concurrent.TimeUnit; import net.bytebuddy.asm.Advice; import net.bytebuddy.description.type.TypeDescription; import net.bytebuddy.matcher.ElementMatcher; @@ -149,13 +147,6 @@ public static void captureScope(@Advice.This RunnableFuture task) { public static final class Run { @Advice.OnMethodEnter public static AgentScope activate(@Advice.This RunnableFuture task) { - // Newer Netty versions may execute a delayed ScheduledFutureTask once only to move it into - // the scheduled queue. Keep the continuation for the run that actually invokes the task. - if (task instanceof Delayed - && task.getClass().getName().endsWith(".netty.util.concurrent.ScheduledFutureTask") - && ((Delayed) task).getDelay(TimeUnit.NANOSECONDS) > 0) { - return null; - } return startTaskScope(InstrumentationContext.get(RunnableFuture.class, State.class), task); } diff --git a/dd-java-agent/instrumentation/java/java-concurrent/java-concurrent-1.8/src/test/groovy/executor/NettyExecutorInstrumentationTest.groovy b/dd-java-agent/instrumentation/java/java-concurrent/java-concurrent-1.8/src/test/groovy/executor/NettyExecutorInstrumentationTest.groovy index d06fcdc6487..48b8267c5b8 100644 --- a/dd-java-agent/instrumentation/java/java-concurrent/java-concurrent-1.8/src/test/groovy/executor/NettyExecutorInstrumentationTest.groovy +++ b/dd-java-agent/instrumentation/java/java-concurrent/java-concurrent-1.8/src/test/groovy/executor/NettyExecutorInstrumentationTest.groovy @@ -17,11 +17,18 @@ import java.util.concurrent.Future import java.util.concurrent.RejectedExecutionException import java.util.concurrent.TimeUnit import runnable.JavaAsyncChild +import spock.lang.IgnoreIf import spock.lang.Shared +// TODO: netty-all 4.1.9 only ships linux-x86_64 epoll native libraries. +@IgnoreIf({ + OperatingSystem.isLinux() && OperatingSystem.isArm64() +}) class NettyExecutorInstrumentationTest extends InstrumentationSpecification { + @Shared boolean isLinux = OperatingSystem.isLinux() + @Shared EpollEventLoopGroup epollEventLoopGroup = isLinux ? new EpollEventLoopGroup(4) : null @Shared