diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 5290d93abd0..5c05e7fe9f5 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -36,6 +36,7 @@ stages: - shared-pipeline - benchmarks - tests + - tests-arm64 - test-summary - exploration-tests - ci-visibility-tests @@ -52,6 +53,8 @@ variables: GRADLE_VERSION: "8.14.4" # must match gradle-wrapper.properties MAVEN_REPOSITORY_PROXY: "https://depot-read-api-java.us1.ddbuild.io/magicmirror/magicmirror/@current/" GRADLE_PLUGIN_PROXY: "https://depot-read-api-java.us1.ddbuild.io/magicmirror/magicmirror/@current/" + ARM64_BUILDER_IMAGE_REPO: "registry.ddbuild.io/images/mirror/dd-trace-java-docker-build" + ARM64_BUILDER_IMAGE_TAG: "alexeyk_arm64-test-arm64-base" BUILDER_IMAGE_REPO: "registry.ddbuild.io/images/mirror/dd-trace-java-docker-build" # images are pinned in images/mirror.lock.yaml in the DataDog/images repo BUILDER_IMAGE_VERSION_PREFIX: "ci-" # use either an empty string (e.g. "") for latest images or a version followed by a hyphen (e.g. "ci-" or "123_merge-") REPO_NOTIFICATION_CHANNEL: "#apm-java-escalations" @@ -163,6 +166,24 @@ default: echo -e "${TEXT_BOLD}${TEXT_YELLOW} Containers:${TEXT_CLEAR} https://app.datadoghq.com/containers?${TIME_PARAMS}query=image_name%3A%2A%2Fdatadog%2Fdd-trace-java-docker-build%20AND%20pod_name%3A${POD_NAME}&live=false" echo -e "${TEXT_BOLD}${TEXT_YELLOW} Processes:${TEXT_CLEAR} https://app.datadoghq.com/process?${TIME_PARAMS}query=image_name%3A%2A%2Fdatadog%2Fdd-trace-java-docker-build%20AND%20pod_name%3A${POD_NAME}&live=false" +.tier_m: + variables: &tier_m_variables + GRADLE_WORKERS: 4 + GRADLE_MEMORY_MIN: 1G + GRADLE_MEMORY_MAX: 5G + KUBERNETES_CPU_REQUEST: 6 + KUBERNETES_MEMORY_REQUEST: 16Gi + KUBERNETES_MEMORY_LIMIT: 16Gi + +.tier_l: + variables: &tier_l_variables + GRADLE_WORKERS: 6 + GRADLE_MEMORY_MIN: 1G + GRADLE_MEMORY_MAX: 6G + KUBERNETES_CPU_REQUEST: 10 + KUBERNETES_MEMORY_REQUEST: 20Gi + KUBERNETES_MEMORY_LIMIT: 20Gi + .gitlab_base_ref_params: &gitlab_base_ref_params - | export GIT_BASE_REF=$(.gitlab/find-gh-base-ref.sh) @@ -681,6 +702,115 @@ muzzle-dep-report: - scheduler_failure - data_integrity_failure +.test_job_arm64: + image: ${ARM64_BUILDER_IMAGE_REPO}:${ARM64_BUILDER_IMAGE_TAG} + tags: [ "docker-in-docker:arm64" ] + stage: tests-arm64 + needs: [] + variables: + <<: *tier_m_variables + # arm64 docker image installs only these JVMs; override the global default regex. + DEFAULT_TEST_JVMS: /^(8|11|17|21|25)$/ + GRADLE_PARAMS: "-PskipFlakyTests" + TESTCONTAINERS_CHECKS_DISABLE: "true" + TESTCONTAINERS_RYUK_DISABLED: "true" + TESTCONTAINERS_HUB_IMAGE_NAME_PREFIX: "registry.ddbuild.io/images/mirror/" + JETTY_AVAILABLE_PROCESSORS: 4 + GIT_SUBMODULE_STRATEGY: normal + GIT_SUBMODULE_DEPTH: 1 + rules: + - if: $testJvm =~ $DEFAULT_TEST_JVMS + when: manual + allow_failure: true + cache: + - key: dependency-$CACHE_TYPE + paths: + - .gradle/wrapper + - .gradle/caches + - .gradle/notifications + - .mvn/caches + policy: pull + fallback_keys: + - dependency-base + - dependency-lib + unprotect: true + before_script: + - git config --global --add safe.directory "$CI_PROJECT_DIR" + - export ORG_GRADLE_PROJECT_mavenRepositoryProxy=$MAVEN_REPOSITORY_PROXY + - export ORG_GRADLE_PROJECT_gradlePluginProxy=$GRADLE_PLUGIN_PROXY + - | + JAVA_HOMES=$(env | grep -E '^JAVA_[A-Z0-9_]+_HOME=' | sed 's/=.*//' | paste -sd,) + cat >> gradle.properties <()) } + // Temurin 11/21 on Linux arm64 SIGSEGVs in SystemDictionary::define_instance_class + // during CDS shared-class restore. GRADLE_OPTS / org.gradle.jvmargs only reaches the + // Gradle daemon, so every forked JVM (Test executor, JavaCompile worker daemon when + // the toolchain differs from the daemon's JDK) must disable CDS on its own. + val isLinuxArm64 = HostPlatform.isLinuxArm64() + tasks.configureEach { if (this is JavaForkOptions) { maxHeapSize = System.getProperty("datadog.forkedMaxHeapSize") @@ -91,6 +98,18 @@ allprojects { "-XX:+HeapDumpOnOutOfMemoryError", "-XX:HeapDumpPath=/tmp" ) + if (isLinuxArm64) { + jvmArgs("-Xshare:off") + } + } + } + + if (isLinuxArm64) { + tasks.withType().configureEach { + options.forkOptions.jvmArgs?.add("-Xshare:off") + } + tasks.withType().configureEach { + groovyOptions.forkOptions.jvmArgs?.add("-Xshare:off") } } } diff --git a/buildSrc/src/main/kotlin/datadog/gradle/plugin/HostPlatform.kt b/buildSrc/src/main/kotlin/datadog/gradle/plugin/HostPlatform.kt new file mode 100644 index 00000000000..40837194ecd --- /dev/null +++ b/buildSrc/src/main/kotlin/datadog/gradle/plugin/HostPlatform.kt @@ -0,0 +1,21 @@ +package datadog.gradle.plugin + +import java.util.Locale + +object HostPlatform { + @JvmStatic + fun isLinuxArm64(): Boolean = isExpectedOs("linux") && isArm64() + + @JvmStatic + fun isMacArm64(): Boolean = isExpectedOs("mac") && isArm64() + + private fun isExpectedOs(expectedOs: String): Boolean { + val osName = System.getProperty("os.name", "").lowercase(Locale.ROOT) + return osName.contains(expectedOs) + } + + private fun isArm64(): Boolean { + val osArch = System.getProperty("os.arch", "").lowercase(Locale.ROOT) + return osArch.contains("aarch64") || osArch.contains("arm64") + } +} diff --git a/buildSrc/src/main/kotlin/datadog/gradle/plugin/instrument/InstrumentPostProcessingAction.kt b/buildSrc/src/main/kotlin/datadog/gradle/plugin/instrument/InstrumentPostProcessingAction.kt index b8c3e300afd..3f70bac7dec 100644 --- a/buildSrc/src/main/kotlin/datadog/gradle/plugin/instrument/InstrumentPostProcessingAction.kt +++ b/buildSrc/src/main/kotlin/datadog/gradle/plugin/instrument/InstrumentPostProcessingAction.kt @@ -1,5 +1,6 @@ package datadog.gradle.plugin.instrument +import datadog.gradle.plugin.HostPlatform import datadog.gradle.plugin.instrument.BuildTimeInstrumentationPlugin.Companion.BUILD_TIME_INSTRUMENTATION_PLUGIN_CONFIGURATION import org.gradle.api.Action import org.gradle.api.Project @@ -73,6 +74,10 @@ abstract class InstrumentPostProcessingAction @Inject constructor( return workerExecutor.processIsolation { forkOptions { setExecutable(javaLauncher.executablePath.asFile.absolutePath) + if (HostPlatform.isLinuxArm64()) { + // Temurin on the arm64 Linux can crash during CDS shared-class restore; + jvmArgs("-Xshare:off") + } } } } diff --git a/buildSrc/src/main/kotlin/datadog/gradle/plugin/muzzle/tasks/MuzzleTask.kt b/buildSrc/src/main/kotlin/datadog/gradle/plugin/muzzle/tasks/MuzzleTask.kt index 2d5d830ea3b..08d42673298 100644 --- a/buildSrc/src/main/kotlin/datadog/gradle/plugin/muzzle/tasks/MuzzleTask.kt +++ b/buildSrc/src/main/kotlin/datadog/gradle/plugin/muzzle/tasks/MuzzleTask.kt @@ -1,5 +1,6 @@ package datadog.gradle.plugin.muzzle.tasks +import datadog.gradle.plugin.HostPlatform import datadog.gradle.plugin.muzzle.MuzzleAction import datadog.gradle.plugin.muzzle.MuzzleDirective import datadog.gradle.plugin.muzzle.MuzzleExtension @@ -102,6 +103,10 @@ abstract class MuzzleTask @Inject constructor( if(javaLauncher.metadata.languageVersion > JavaLanguageVersion.of(9)) { jvmArgs("--add-opens=java.base/java.lang=ALL-UNNAMED") } + if (HostPlatform.isLinuxArm64()) { + // Temurin on the arm64 Linux can crash during CDS shared-class restore; + jvmArgs("-Xshare:off") + } executable(javaLauncher.executablePath) } } diff --git a/buildSrc/src/main/kotlin/dd-trace-java.test-jvm-constraints.gradle.kts b/buildSrc/src/main/kotlin/dd-trace-java.test-jvm-constraints.gradle.kts index aa4724183b0..dfe3cfa52c2 100644 --- a/buildSrc/src/main/kotlin/dd-trace-java.test-jvm-constraints.gradle.kts +++ b/buildSrc/src/main/kotlin/dd-trace-java.test-jvm-constraints.gradle.kts @@ -1,3 +1,4 @@ +import datadog.gradle.plugin.HostPlatform import datadog.gradle.plugin.testJvmConstraints.ProvideJvmArgsOnJvmLauncherVersion import datadog.gradle.plugin.testJvmConstraints.TestJvmConstraintsExtension import datadog.gradle.plugin.testJvmConstraints.TestJvmConstraintsExtension.Companion.TEST_JVM_CONSTRAINTS @@ -79,6 +80,16 @@ private fun Test.configureTestJvm(extension: TestJvmConstraintsExtension) { ), extension.allowReflectiveAccessToJdk ) + + // Fix for arm64 Linux ByteBuddy error: + // "Could not self-attach to current VM using external process" + // Why it is needed on arm64: probably arm64 GitLab runners has stricter rules applied. + if (HostPlatform.isLinuxArm64()) { + conditionalJvmArgs( + JavaVersion.VERSION_1_9, + listOf("-Djdk.attach.allowAttachSelf=true") + ) + } } // Jacoco plugin is not applied on every project diff --git a/communication/src/test/java/datadog/communication/serialization/msgpack/MsgPackWriterTest.java b/communication/src/test/java/datadog/communication/serialization/msgpack/MsgPackWriterTest.java index 4a3f773fdc5..fca302dd3d1 100644 --- a/communication/src/test/java/datadog/communication/serialization/msgpack/MsgPackWriterTest.java +++ b/communication/src/test/java/datadog/communication/serialization/msgpack/MsgPackWriterTest.java @@ -33,8 +33,7 @@ import org.msgpack.core.MessageUnpacker; public class MsgPackWriterTest { - // Explicit escapes for non-ASCII chars to make test independent of container settings. - private static final String NON_ASCII_STRING = "foob\u00E1r_\u263a"; // foobár_☺ + private static final String NON_ASCII_STRING = "foobár_☺"; private static final byte[] NON_ASCII_BYTES = NON_ASCII_STRING.getBytes(UTF_8); private static final int NON_ASCII_BUFFER_CAPACITY = NON_ASCII_BYTES.length + 1; diff --git a/components/environment/src/main/java/datadog/environment/OperatingSystem.java b/components/environment/src/main/java/datadog/environment/OperatingSystem.java index 1d24d7680c9..686bfca442d 100644 --- a/components/environment/src/main/java/datadog/environment/OperatingSystem.java +++ b/components/environment/src/main/java/datadog/environment/OperatingSystem.java @@ -1,5 +1,6 @@ package datadog.environment; +import static datadog.environment.OperatingSystem.Architecture.ARM64; import static datadog.environment.OperatingSystem.Type.LINUX; import static datadog.environment.OperatingSystem.Type.MACOS; import static datadog.environment.OperatingSystem.Type.WINDOWS; @@ -25,6 +26,15 @@ public final class OperatingSystem { private OperatingSystem() {} + /** + * Checks whether the architecture is arm64. + * + * @return @{@code true} if architecture is arm64, {@code false} otherwise. + */ + public static boolean isArm64() { + return ARCHITECTURE == ARM64; + } + /** * Checks whether the operating system is Linux based. * diff --git a/components/json/src/test/java/datadog/json/JsonReaderTest.java b/components/json/src/test/java/datadog/json/JsonReaderTest.java index 15a635c148a..e716a625103 100644 --- a/components/json/src/test/java/datadog/json/JsonReaderTest.java +++ b/components/json/src/test/java/datadog/json/JsonReaderTest.java @@ -215,8 +215,7 @@ void testStringEscaping() { assertEquals("\n", reader.nextString()); assertEquals("\r", reader.nextString()); assertEquals("\t", reader.nextString()); - // Explicit escape for non-ASCII `É` to make test independent of container settings. - assertEquals("\u00C9", reader.nextString()); + assertEquals("É", reader.nextString()); reader.endArray(); } catch (IOException e) { fail("Failed to read escaped JSON strings", e); diff --git a/dd-java-agent/agent-crashtracking/src/test/java/datadog/crashtracking/buildid/BuildIdExtractorIntegrationTest.java b/dd-java-agent/agent-crashtracking/src/test/java/datadog/crashtracking/buildid/BuildIdExtractorIntegrationTest.java index a0cf46ea89b..da13cbca0b5 100644 --- a/dd-java-agent/agent-crashtracking/src/test/java/datadog/crashtracking/buildid/BuildIdExtractorIntegrationTest.java +++ b/dd-java-agent/agent-crashtracking/src/test/java/datadog/crashtracking/buildid/BuildIdExtractorIntegrationTest.java @@ -1,6 +1,5 @@ package datadog.crashtracking.buildid; -import static datadog.environment.OperatingSystem.Architecture.ARM64; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertNotNull; import static org.junit.jupiter.api.Assertions.assertTrue; @@ -138,7 +137,7 @@ private static Stream elfBinaries() { @MethodSource("elfBinaries") void testElfBuildIdExtraction(String containerPath, String description) throws Exception { // TODO: check if arm64 can be supported too. - assumeFalse(OperatingSystem.architecture() == ARM64, "Skipping for arm64"); + assumeFalse(OperatingSystem.isArm64(), "Skipping for arm64"); Path localBinary = copyFromContainer(linuxContainer, containerPath); ElfBuildIdExtractor extractor = new ElfBuildIdExtractor(); diff --git a/dd-java-agent/agent-profiling/profiling-controller/src/main/java/com/datadog/profiling/controller/EnvironmentChecker.java b/dd-java-agent/agent-profiling/profiling-controller/src/main/java/com/datadog/profiling/controller/EnvironmentChecker.java index 62ca9ea29fe..62beaeba443 100644 --- a/dd-java-agent/agent-profiling/profiling-controller/src/main/java/com/datadog/profiling/controller/EnvironmentChecker.java +++ b/dd-java-agent/agent-profiling/profiling-controller/src/main/java/com/datadog/profiling/controller/EnvironmentChecker.java @@ -1,7 +1,5 @@ package com.datadog.profiling.controller; -import static datadog.environment.OperatingSystem.Architecture.ARM64; - import datadog.environment.JavaVirtualMachine; import datadog.environment.OperatingSystem; import datadog.environment.SystemProperties; @@ -243,11 +241,7 @@ private static boolean extractSoFromJar(Path target, StringBuilder sb) throws Ex .filter(e -> e.getName().contains("libjavaProfiler.so")) .filter( e -> - e.getName() - .contains( - OperatingSystem.architecture() == ARM64 - ? "/linux-arm64/" - : "/linux-x64/") + e.getName().contains(OperatingSystem.isArm64() ? "/linux-arm64/" : "/linux-x64/") && (!OperatingSystem.isMusl() || e.getName().contains("-musl"))) .findFirst() .map( diff --git a/dd-java-agent/instrumentation/aerospike-4.0/src/test/groovy/datadog/trace/instrumentation/aerospike4/AerospikeBaseTest.groovy b/dd-java-agent/instrumentation/aerospike-4.0/src/test/groovy/datadog/trace/instrumentation/aerospike4/AerospikeBaseTest.groovy index 57848379357..1c86d404784 100644 --- a/dd-java-agent/instrumentation/aerospike-4.0/src/test/groovy/datadog/trace/instrumentation/aerospike4/AerospikeBaseTest.groovy +++ b/dd-java-agent/instrumentation/aerospike-4.0/src/test/groovy/datadog/trace/instrumentation/aerospike4/AerospikeBaseTest.groovy @@ -1,6 +1,10 @@ package datadog.trace.instrumentation.aerospike4 +import static datadog.trace.agent.test.utils.PortUtils.waitForPortToOpen +import static java.util.concurrent.TimeUnit.SECONDS +import static org.testcontainers.containers.wait.strategy.Wait.forLogMessage +import com.github.dockerjava.api.model.Ulimit import datadog.trace.agent.test.asserts.TraceAssert import datadog.trace.agent.test.naming.VersionedNamingTestBase import datadog.trace.api.DDSpanTypes @@ -9,10 +13,6 @@ import datadog.trace.core.DDSpan import org.testcontainers.containers.GenericContainer import spock.lang.Shared -import static datadog.trace.agent.test.utils.PortUtils.waitForPortToOpen -import static java.util.concurrent.TimeUnit.SECONDS -import static org.testcontainers.containers.wait.strategy.Wait.forLogMessage - abstract class AerospikeBaseTest extends VersionedNamingTestBase { @Shared @@ -25,8 +25,14 @@ abstract class AerospikeBaseTest extends VersionedNamingTestBase { int aerospikePort = 3000 def setup() throws Exception { - aerospike = new GenericContainer('aerospike:5.5.0.9') + // arm64 supported since ce-6.2.0.2 + aerospike = new GenericContainer('aerospike:ce-6.2.0.2') .withExposedPorts(3000) + // proto-fd-max default is 15000, but container default is 1024. + // see: https://aerospike.com/docs/database/reference/config#service__proto-fd-max + .withCreateContainerCmdModifier({ cmd -> + cmd.getHostConfig().withUlimits([new Ulimit('nofile', 15000L, 15000L)] as Ulimit[]) + }) .waitingFor(forLogMessage(".*heartbeat-received.*\\n", 1)) aerospike.start() @@ -37,9 +43,7 @@ abstract class AerospikeBaseTest extends VersionedNamingTestBase { } def cleanup() throws Exception { - if (aerospike) { - aerospike.stop() - } + aerospike?.stop() } def aerospikeSpan(TraceAssert trace, int index, String methodName, Object parentSpan = null) { diff --git a/dd-java-agent/instrumentation/google-pubsub-1.116/src/test/groovy/PubSubTest.groovy b/dd-java-agent/instrumentation/google-pubsub-1.116/src/test/groovy/PubSubTest.groovy index 289822df042..07189f86b37 100644 --- a/dd-java-agent/instrumentation/google-pubsub-1.116/src/test/groovy/PubSubTest.groovy +++ b/dd-java-agent/instrumentation/google-pubsub-1.116/src/test/groovy/PubSubTest.groovy @@ -1,4 +1,3 @@ - import static datadog.trace.agent.test.utils.TraceUtils.basicSpan import com.google.api.gax.core.NoCredentialsProvider @@ -34,13 +33,12 @@ import datadog.trace.core.datastreams.StatsGroup import datadog.trace.instrumentation.grpc.client.GrpcClientDecorator import io.grpc.ManagedChannel import io.grpc.ManagedChannelBuilder +import java.nio.charset.StandardCharsets +import java.util.concurrent.CountDownLatch import org.testcontainers.containers.PubSubEmulatorContainer import org.testcontainers.utility.DockerImageName import spock.lang.Shared -import java.nio.charset.StandardCharsets -import java.util.concurrent.CountDownLatch - abstract class PubSubTest extends VersionedNamingTestBase { private static final String PROJECT_ID = "dd-trace-java" @@ -97,7 +95,7 @@ abstract class PubSubTest extends VersionedNamingTestBase { } def setupSpec() { - emulator = new PubSubEmulatorContainer(DockerImageName.parse("gcr.io/google.com/cloudsdktool/cloud-sdk:495.0.0-emulators")) + emulator = new PubSubEmulatorContainer(DockerImageName.parse("gcr.io/google.com/cloudsdktool/google-cloud-cli:495.0.0-emulators")) emulator.start() channel = ManagedChannelBuilder.forTarget(emulator.getEmulatorEndpoint()).usePlaintext().build() transportChannelProvider = FixedTransportChannelProvider.create(GrpcTransportChannel.create(channel)) diff --git a/dd-java-agent/instrumentation/java/java-concurrent/java-concurrent-1.8/build.gradle b/dd-java-agent/instrumentation/java/java-concurrent/java-concurrent-1.8/build.gradle index ee85bef77fa..83caaeb1324 100644 --- a/dd-java-agent/instrumentation/java/java-concurrent/java-concurrent-1.8/build.gradle +++ b/dd-java-agent/instrumentation/java/java-concurrent/java-concurrent-1.8/build.gradle @@ -21,4 +21,3 @@ dependencies { // TODO: Tomcat 10.0.10 has a copy of the JSR166 ThreadPoolExecutor so it needs special instrumentation latestDepTestImplementation group: 'org.apache.tomcat.embed', name: 'tomcat-embed-core', version: '10.0.8' } - diff --git a/dd-java-agent/instrumentation/java/java-concurrent/java-concurrent-1.8/src/test/groovy/executor/NettyExecutorInstrumentationTest.groovy b/dd-java-agent/instrumentation/java/java-concurrent/java-concurrent-1.8/src/test/groovy/executor/NettyExecutorInstrumentationTest.groovy index fc944416d94..48b8267c5b8 100644 --- a/dd-java-agent/instrumentation/java/java-concurrent/java-concurrent-1.8/src/test/groovy/executor/NettyExecutorInstrumentationTest.groovy +++ b/dd-java-agent/instrumentation/java/java-concurrent/java-concurrent-1.8/src/test/groovy/executor/NettyExecutorInstrumentationTest.groovy @@ -1,5 +1,8 @@ package executor +import static org.junit.jupiter.api.Assumptions.assumeTrue + +import datadog.environment.OperatingSystem import datadog.trace.agent.test.InstrumentationSpecification import datadog.trace.api.Trace import datadog.trace.core.DDSpan @@ -8,21 +11,23 @@ import io.netty.channel.epoll.EpollEventLoopGroup import io.netty.channel.local.LocalEventLoopGroup import io.netty.channel.nio.NioEventLoopGroup import io.netty.util.concurrent.DefaultEventExecutorGroup -import runnable.JavaAsyncChild -import spock.lang.Shared - import java.lang.reflect.InvocationTargetException import java.util.concurrent.Callable import java.util.concurrent.Future import java.util.concurrent.RejectedExecutionException import java.util.concurrent.TimeUnit +import runnable.JavaAsyncChild +import spock.lang.IgnoreIf +import spock.lang.Shared -import static org.junit.jupiter.api.Assumptions.assumeTrue - +// TODO: netty-all 4.1.9 only ships linux-x86_64 epoll native libraries. +@IgnoreIf({ + OperatingSystem.isLinux() && OperatingSystem.isArm64() +}) class NettyExecutorInstrumentationTest extends InstrumentationSpecification { @Shared - boolean isLinux = System.getProperty("os.name").toLowerCase().contains("linux") + boolean isLinux = OperatingSystem.isLinux() @Shared EpollEventLoopGroup epollEventLoopGroup = isLinux ? new EpollEventLoopGroup(4) : null diff --git a/dd-java-agent/instrumentation/jdbc/src/test/groovy/RemoteJDBCInstrumentationTest.groovy b/dd-java-agent/instrumentation/jdbc/src/test/groovy/RemoteJDBCInstrumentationTest.groovy index 616b38a456d..4fb72863371 100644 --- a/dd-java-agent/instrumentation/jdbc/src/test/groovy/RemoteJDBCInstrumentationTest.groovy +++ b/dd-java-agent/instrumentation/jdbc/src/test/groovy/RemoteJDBCInstrumentationTest.groovy @@ -2,11 +2,13 @@ import static datadog.trace.agent.test.utils.TraceUtils.basicSpan import static datadog.trace.agent.test.utils.TraceUtils.runUnderTrace import static datadog.trace.api.config.TraceInstrumentationConfig.DB_CLIENT_HOST_SPLIT_BY_INSTANCE import static datadog.trace.api.config.TraceInstrumentationConfig.DB_DBM_TRACE_PREPARED_STATEMENTS +import static org.junit.jupiter.api.Assumptions.assumeTrue import com.mchange.v2.c3p0.ComboPooledDataSource import com.microsoft.sqlserver.jdbc.SQLServerException import com.zaxxer.hikari.HikariConfig import com.zaxxer.hikari.HikariDataSource +import datadog.environment.OperatingSystem import datadog.trace.agent.test.naming.VersionedNamingTestBase import datadog.trace.agent.test.utils.PortUtils import datadog.trace.api.Config @@ -90,6 +92,10 @@ abstract class RemoteJDBCInstrumentationTest extends VersionedNamingTestBase { @Shared private Map> cpDatasources = new HashMap<>() + private static boolean dockerImageSupported(String db) { + return !(db == SQLSERVER && OperatingSystem.isArm64() && OperatingSystem.isLinux()) + } + def peerConnectionProps(String db){ def props = new Properties() props.setProperty("user", jdbcUserNames.get(db)) @@ -158,6 +164,11 @@ abstract class RemoteJDBCInstrumentationTest extends VersionedNamingTestBase { def createDS(String connectionPoolName, String dbType, String jdbcUrl) { DataSource ds = null + + if (!dockerImageSupported(dbType)) { + return ds + } + if (connectionPoolName == "tomcat") { ds = createTomcatDS(dbType, jdbcUrl) } @@ -196,10 +207,12 @@ abstract class RemoteJDBCInstrumentationTest extends VersionedNamingTestBase { jdbcUrls.put(MYSQL, "${mysql.getJdbcUrl()}") // SQLSERVER - sqlserver = new MSSQLServerContainer(MSSQLServerContainer.IMAGE).acceptLicense().withPassword(jdbcPasswords.get(SQLSERVER)) - sqlserver.start() - PortUtils.waitForPortToOpen(sqlserver.getHost(), sqlserver.getMappedPort(MSSQLServerContainer.MS_SQL_SERVER_PORT), 5, TimeUnit.SECONDS) - jdbcUrls.put(SQLSERVER, "${sqlserver.getJdbcUrl()};DatabaseName=${dbName.get(SQLSERVER)}") + if (dockerImageSupported(SQLSERVER)) { + sqlserver = new MSSQLServerContainer(MSSQLServerContainer.IMAGE).acceptLicense().withPassword(jdbcPasswords.get(SQLSERVER)) + sqlserver.start() + PortUtils.waitForPortToOpen(sqlserver.getHost(), sqlserver.getMappedPort(MSSQLServerContainer.MS_SQL_SERVER_PORT), 5, TimeUnit.SECONDS) + jdbcUrls.put(SQLSERVER, "${sqlserver.getJdbcUrl()};DatabaseName=${dbName.get(SQLSERVER)}") + } // ORACLE // Earlier Oracle version images (oracle-xe) don't work on arm64 @@ -1026,6 +1039,8 @@ abstract class RemoteJDBCInstrumentationTest extends VersionedNamingTestBase { } Connection setupConnection(String pool, String db) { + assumeTrue(dockerImageSupported(db)) + def conn = pool ? cpDatasources.get(pool).get(db).getConnection() : connectTo(db, peerConnectionProps(db)) // Clear any traces that pool or db can emmit on connection creation. diff --git a/dd-java-agent/instrumentation/protobuf-3.0/build.gradle b/dd-java-agent/instrumentation/protobuf-3.0/build.gradle index d411c3b12d3..34fd1e22816 100644 --- a/dd-java-agent/instrumentation/protobuf-3.0/build.gradle +++ b/dd-java-agent/instrumentation/protobuf-3.0/build.gradle @@ -1,3 +1,5 @@ +import datadog.gradle.plugin.HostPlatform + plugins { id 'com.google.protobuf' version '0.10.0' } @@ -14,13 +16,11 @@ muzzle { } } + protobuf { protoc { - def os = System.getProperty("os.name").toLowerCase() - def arch = System.getProperty("os.arch").toLowerCase() - - // There is no m1 support for protoc 3.0.0, so require Rosetta - if (os.contains("mac") && arch.contains("aarch64")) { + // There is no macOS arm64 support for protoc 3.0.0, so require Rosetta. + if (HostPlatform.isMacArm64()) { artifact = "com.google.protobuf:protoc:3.0.0:osx-x86_64" } else { artifact = "com.google.protobuf:protoc:3.0.0" @@ -28,6 +28,23 @@ protobuf { } } +// protobuf supports arm64 linux since 3.12.0, but it is not source-compatible with 3.0.0 +if (HostPlatform.isLinuxArm64()) { + tasks.matching { + it.name in [ + "extractTestProto", + "generateTestProto", + "compileTestJava", + "compileTestGroovy", + "processTestResources", + "testClasses", + "test" + ] + }.configureEach { + enabled = false + } +} + dependencies { compileOnly group: 'com.google.protobuf', name: 'protobuf-java', version: '3.0.0' diff --git a/dd-java-agent/instrumentation/restlet-2.2/src/latestDepTest/groovy/RestletTest.groovy b/dd-java-agent/instrumentation/restlet-2.2/src/latestDepTest/groovy/RestletTest.groovy index 3217e53394d..2c308efeb29 100644 --- a/dd-java-agent/instrumentation/restlet-2.2/src/latestDepTest/groovy/RestletTest.groovy +++ b/dd-java-agent/instrumentation/restlet-2.2/src/latestDepTest/groovy/RestletTest.groovy @@ -1,3 +1,6 @@ +import static datadog.environment.OperatingSystem.isArm64 +import static datadog.environment.OperatingSystem.isLinux + import org.restlet.Request import org.restlet.Response import org.restlet.data.Header @@ -6,6 +9,12 @@ import org.restlet.util.Series class RestletTest extends RestletTestBase { + @Override + boolean testParallelRequest() { + // TODO: Parallel processing is failing on Linux arm64. + return !(isLinux() && isArm64()) + } + @Override protected Filter createHeaderFilter() { return new ResponseHeaderFilter() diff --git a/dd-smoke-tests/debugger-integration-tests/src/test/java/datadog/smoketest/ProcessBuilderHelper.java b/dd-smoke-tests/debugger-integration-tests/src/test/java/datadog/smoketest/ProcessBuilderHelper.java index 5faf29bbaa5..7947a8badf8 100644 --- a/dd-smoke-tests/debugger-integration-tests/src/test/java/datadog/smoketest/ProcessBuilderHelper.java +++ b/dd-smoke-tests/debugger-integration-tests/src/test/java/datadog/smoketest/ProcessBuilderHelper.java @@ -1,5 +1,6 @@ package datadog.smoketest; +import datadog.environment.OperatingSystem; import java.io.File; import java.nio.file.Path; import java.util.ArrayList; @@ -43,16 +44,22 @@ public static ProcessBuilder createProcessBuilder( + System.nanoTime(); List baseCommand = - Arrays.asList( - javaPath(), - // "-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=localhost:5006", - "-Xmx" + System.getProperty("datadog.forkedMaxHeapSize", "1024M"), - "-Xms" + System.getProperty("datadog.forkedMinHeapSize", "64M"), - "-javaagent:" + agentShadowJar(), - "-XX:ErrorFile=/tmp/hs_err_pid%p.log", - "-Ddd.env=smoketest", - "-Ddd.version=99", - "-Djava.util.prefs.userRoot=" + prefsDir); + new ArrayList<>( + Arrays.asList( + javaPath(), + // "-agentlib:jdwp=transport=dt_socket,server=y,suspend=y,address=localhost:5006", + "-Xmx" + System.getProperty("datadog.forkedMaxHeapSize", "1024M"), + "-Xms" + System.getProperty("datadog.forkedMinHeapSize", "64M"), + "-javaagent:" + agentShadowJar(), + "-XX:ErrorFile=/tmp/hs_err_pid%p.log", + "-Ddd.env=smoketest", + "-Ddd.version=99", + "-Djava.util.prefs.userRoot=" + prefsDir)); + + if (OperatingSystem.isLinux() && OperatingSystem.isArm64()) { + // Temurin on the arm64 Linux can crash during CDS shared-class restore; + baseCommand.add(1, "-Xshare:off"); + } List command = new ArrayList<>(); command.addAll(baseCommand); diff --git a/dd-smoke-tests/play-2.4/build.gradle b/dd-smoke-tests/play-2.4/build.gradle index 1a0de08b58b..4645fd91e74 100644 --- a/dd-smoke-tests/play-2.4/build.gradle +++ b/dd-smoke-tests/play-2.4/build.gradle @@ -1,3 +1,9 @@ +import datadog.gradle.plugin.HostPlatform +import org.gradle.playframework.tasks.RoutesCompile +import org.gradle.playframework.tasks.internal.RoutesCompileWorkAction +import org.gradle.playframework.tools.internal.routes.DefaultRoutesCompileSpec +import org.gradle.playframework.tools.internal.routes.RoutesCompilerFactory + plugins { id 'org.gradle.playframework' } @@ -93,6 +99,40 @@ tasks.withType(AbstractCopyTask).configureEach { duplicatesStrategy = DuplicatesStrategy.EXCLUDE } +if (HostPlatform.isLinuxArm64()) { + tasks.named("compilePlayRoutes", RoutesCompile) { RoutesCompile task -> + // org.gradle.playframework.tasks.RoutesCompile does not expose worker fork options, but its + // worker JVM needs CDS disabled on Linux arm64 like the other forked JVMs in this build. + def workerExecutorField = RoutesCompile.getDeclaredField("workerExecutor") + workerExecutorField.accessible = true + + task.actions.clear() + task.doLast { + def spec = new DefaultRoutesCompileSpec( + task.source.files, + task.outputDirectory.get().asFile, + task.javaProject, + task.namespaceReverseRouter.get(), + task.generateReverseRoutes.get(), + task.injectedRoutesGenerator.get(), + task.additionalImports.get(), + projectDir + ) + + def workerExecutor = workerExecutorField.get(task) + workerExecutor.processIsolation { workerSpec -> + workerSpec.forkOptions { options -> + options.jvmArgs("-XX:MaxMetaspaceSize=256m", "-Xshare:off") + } + workerSpec.classpath.from(task.routesCompilerClasspath) + }.submit(RoutesCompileWorkAction) { parameters -> + parameters.compiler.set(RoutesCompilerFactory.create(task.platform.get())) + parameters.spec.set(spec) + } + } + } +} + spotless { java { target "**/*.java" diff --git a/dd-smoke-tests/src/main/groovy/datadog/smoketest/AbstractSmokeTest.groovy b/dd-smoke-tests/src/main/groovy/datadog/smoketest/AbstractSmokeTest.groovy index 25aba9b9660..0d740d17e41 100644 --- a/dd-smoke-tests/src/main/groovy/datadog/smoketest/AbstractSmokeTest.groovy +++ b/dd-smoke-tests/src/main/groovy/datadog/smoketest/AbstractSmokeTest.groovy @@ -252,6 +252,13 @@ abstract class AbstractSmokeTest extends ProcessManager { // Unlike crash tracking smoke test, keep the default delay; otherwise, otherwise other tests will fail // ret += "-Ddd.dogstatsd.start-delay=0" } + + // Disable CDS on Linux arm64: Temurin 11.0.31 / 21.0.10 hit a SIGSEGV during + // shared-class restore (ClassLoaderData::add_handle, Klass::class_loader) + // before any user code runs. + if (OperatingSystem.isArm64() && OperatingSystem.isLinux()) { + ret += "-Xshare:off" + } ret as String[] } diff --git a/dd-smoke-tests/websphere-jmx/src/test/groovy/datadog/smoketest/WebSphereJmxSmokeTest.groovy b/dd-smoke-tests/websphere-jmx/src/test/groovy/datadog/smoketest/WebSphereJmxSmokeTest.groovy index 28b72c111c7..471716b203d 100644 --- a/dd-smoke-tests/websphere-jmx/src/test/groovy/datadog/smoketest/WebSphereJmxSmokeTest.groovy +++ b/dd-smoke-tests/websphere-jmx/src/test/groovy/datadog/smoketest/WebSphereJmxSmokeTest.groovy @@ -1,6 +1,7 @@ package datadog.smoketest +import datadog.environment.OperatingSystem import java.time.Duration import java.util.concurrent.ArrayBlockingQueue import java.util.concurrent.BlockingQueue @@ -11,6 +12,7 @@ import org.testcontainers.containers.GenericContainer import org.testcontainers.containers.output.Slf4jLogConsumer import org.testcontainers.containers.wait.strategy.Wait import org.testcontainers.utility.MountableFile +import spock.lang.IgnoreIf import spock.lang.Shared /** @@ -22,6 +24,11 @@ import spock.lang.Shared * * Note that the websphere related metrics will only arrive if our instrumentation is applied. */ +// IBM does not publish an arm64 build of icr.io/appcafe/websphere-traditional, and the +// arm64 CI runner has no amd64 emulation configured, so the container fails to start. +@IgnoreIf({ + OperatingSystem.isArm64() && OperatingSystem.isLinux() +}) class WebSphereJmxSmokeTest extends AbstractSmokeTest { private static final Logger LOG = LoggerFactory.getLogger(WebSphereJmxSmokeTest)