Add metrics OpenTelemetry instrumentation entry mechanism (#9908)

* trace autoinstrumentaton should still match all versions after 1.4
* Add otel metrics enable config but doesn't use it
* Introduce instrumentation scope abstraction
* Cache Otel meters by instrumentation scope (mandatory name, optional version/schema)
* Align OtelTracerProvider with meter approach
* Add activation test for OpenTelemetry 1.47 instrumentation (metrics support)
* Review feedback

---------

Co-authored-by: Stuart McCulloch <stuart.mcculloch@datadoghq.com>
Co-authored-by: Bruce Bujon <PerfectSlayer@users.noreply.github.com>

Author: 3 people

dd-java-agent/agent-otel/otel-shim/src/main/java/datadog/opentelemetry/shim/OtelInstrumentationScope.java

@@ -0,0 +1,39 @@
+package datadog.opentelemetry.shim;
+
+import java.util.Objects;
+import javax.annotation.Nullable;
+
+/** Instrumentation scopes have a mandatory name, optional version, and optional schema URL. */
+public final class OtelInstrumentationScope {
+
+  private final String scopeName;
+  @Nullable private final String scopeVersion;
+  @Nullable private final String schemaUrl;
+
+  public OtelInstrumentationScope(
+      String scopeName, @Nullable String scopeVersion, @Nullable String schemaUrl) {
+    this.scopeName = scopeName;
+    this.scopeVersion = scopeVersion;
+    this.schemaUrl = schemaUrl;
+  }
+
+  @Override
+  public boolean equals(Object o) {
+    if (!(o instanceof OtelInstrumentationScope)) {
+      return false;
+    }
+
+    OtelInstrumentationScope that = (OtelInstrumentationScope) o;
+    return scopeName.equals(that.scopeName)
+        && Objects.equals(scopeVersion, that.scopeVersion)
+        && Objects.equals(schemaUrl, that.schemaUrl);
+  }
+
+  @Override
+  public int hashCode() {
+    int result = scopeName.hashCode();
+    result = 31 * result + Objects.hashCode(scopeVersion);
+    result = 31 * result + Objects.hashCode(schemaUrl);
+    return result;
+  }
+}

dd-java-agent/agent-otel/otel-shim/src/main/java/datadog/opentelemetry/shim/metrics/OtelMeter.java

@@ -0,0 +1,45 @@
+package datadog.opentelemetry.shim.metrics;
+
+import datadog.opentelemetry.shim.OtelInstrumentationScope;
+import io.opentelemetry.api.metrics.BatchCallback;
+import io.opentelemetry.api.metrics.DoubleGaugeBuilder;
+import io.opentelemetry.api.metrics.DoubleHistogramBuilder;
+import io.opentelemetry.api.metrics.LongCounterBuilder;
+import io.opentelemetry.api.metrics.LongUpDownCounterBuilder;
+import io.opentelemetry.api.metrics.Meter;
+import io.opentelemetry.api.metrics.ObservableMeasurement;
+import javax.annotation.ParametersAreNonnullByDefault;
+
+@ParametersAreNonnullByDefault
+final class OtelMeter implements Meter {
+
+  OtelMeter(@SuppressWarnings("unused") OtelInstrumentationScope instrumentationScope) {}
+
+  @Override
+  public LongCounterBuilder counterBuilder(String instrumentName) {
+    throw new UnsupportedOperationException("counterBuilder is not yet supported");
+  }
+
+  @Override
+  public LongUpDownCounterBuilder upDownCounterBuilder(String instrumentName) {
+    throw new UnsupportedOperationException("upDownCounterBuilder is not yet supported");
+  }
+
+  @Override
+  public DoubleHistogramBuilder histogramBuilder(String instrumentName) {
+    throw new UnsupportedOperationException("histogramBuilder is not yet supported");
+  }
+
+  @Override
+  public DoubleGaugeBuilder gaugeBuilder(String instrumentName) {
+    throw new UnsupportedOperationException("gaugeBuilder is not yet supported");
+  }
+
+  @Override
+  public BatchCallback batchCallback(
+      Runnable callback,
+      ObservableMeasurement observableMeasurement,
+      ObservableMeasurement... additionalMeasurements) {
+    throw new UnsupportedOperationException("batchCallback is not yet supported");
+  }
+}

dd-java-agent/agent-otel/otel-shim/src/main/java/datadog/opentelemetry/shim/metrics/OtelMeterBuilder.java

@@ -0,0 +1,38 @@
+package datadog.opentelemetry.shim.metrics;
+
+import io.opentelemetry.api.metrics.Meter;
+import io.opentelemetry.api.metrics.MeterBuilder;
+import javax.annotation.Nullable;
+import javax.annotation.ParametersAreNonnullByDefault;
+
+@ParametersAreNonnullByDefault
+final class OtelMeterBuilder implements MeterBuilder {
+  private final OtelMeterProvider meterProvider;
+
+  private final String instrumentationScopeName;
+  @Nullable private String instrumentationScopeVersion;
+  @Nullable private String schemaUrl;
+
+  OtelMeterBuilder(OtelMeterProvider meterProvider, String instrumentationScopeName) {
+    this.meterProvider = meterProvider;
+    this.instrumentationScopeName = instrumentationScopeName;
+  }
+
+  @Override
+  public MeterBuilder setInstrumentationVersion(String instrumentationScopeVersion) {
+    this.instrumentationScopeVersion = instrumentationScopeVersion;
+    return this;
+  }
+
+  @Override
+  public MeterBuilder setSchemaUrl(String schemaUrl) {
+    this.schemaUrl = schemaUrl;
+    return this;
+  }
+
+  @Override
+  public Meter build() {
+    return meterProvider.getMeterShim(
+        instrumentationScopeName, instrumentationScopeVersion, schemaUrl);
+  }
+}

dd-java-agent/agent-otel/otel-shim/src/main/java/datadog/opentelemetry/shim/metrics/OtelMeterProvider.java

@@ -0,0 +1,48 @@
+package datadog.opentelemetry.shim.metrics;
+
+import datadog.opentelemetry.shim.OtelInstrumentationScope;
+import datadog.trace.util.Strings;
+import io.opentelemetry.api.metrics.Meter;
+import io.opentelemetry.api.metrics.MeterBuilder;
+import io.opentelemetry.api.metrics.MeterProvider;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import javax.annotation.Nullable;
+import javax.annotation.ParametersAreNonnullByDefault;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+@ParametersAreNonnullByDefault
+public final class OtelMeterProvider implements MeterProvider {
+  private static final Logger LOGGER = LoggerFactory.getLogger(OtelMeterProvider.class);
+  private static final String DEFAULT_METER_NAME = "unknown";
+
+  public static final MeterProvider INSTANCE = new OtelMeterProvider();
+
+  /** Meter shims, indexed by instrumentation scope. */
+  private final Map<OtelInstrumentationScope, OtelMeter> meters = new ConcurrentHashMap<>();
+
+  @Override
+  public Meter get(String instrumentationScopeName) {
+    return getMeterShim(instrumentationScopeName, null, null);
+  }
+
+  @Override
+  public MeterBuilder meterBuilder(String instrumentationScopeName) {
+    return new OtelMeterBuilder(this, instrumentationScopeName);
+  }
+
+  OtelMeter getMeterShim(
+      String instrumentationScopeName,
+      @Nullable String instrumentationScopeVersion,
+      @Nullable String schemaUrl) {
+    if (Strings.isBlank(instrumentationScopeName)) {
+      LOGGER.debug("Meter requested without instrumentation scope name.");
+      instrumentationScopeName = DEFAULT_METER_NAME;
+    }
+    return meters.computeIfAbsent(
+        new OtelInstrumentationScope(
+            instrumentationScopeName, instrumentationScopeVersion, schemaUrl),
+        OtelMeter::new);
+  }
+}

dd-java-agent/agent-otel/otel-shim/src/main/java/datadog/opentelemetry/shim/trace/OtelTracer.java

@@ -8,13 +8,12 @@
 import javax.annotation.ParametersAreNonnullByDefault;
 
 @ParametersAreNonnullByDefault
-class OtelTracer implements Tracer {
+final class OtelTracer implements Tracer {
   private static final String INSTRUMENTATION_NAME = otelInstrumentationName();
+
   private final AgentTracer.TracerAPI tracer;
-  private final String instrumentationScopeName;
 
-  public OtelTracer(String instrumentationScopeName) {
-    this.instrumentationScopeName = instrumentationScopeName;
+  OtelTracer(@SuppressWarnings("unused") String instrumentationScopeName) {
     this.tracer = AgentTracer.get();
   }
 

dd-java-agent/agent-otel/otel-shim/src/main/java/datadog/opentelemetry/shim/trace/OtelTracerBuilder.java

@@ -5,27 +5,29 @@
 import javax.annotation.ParametersAreNonnullByDefault;
 
 @ParametersAreNonnullByDefault
-class OtelTracerBuilder implements TracerBuilder {
+final class OtelTracerBuilder implements TracerBuilder {
+  private final OtelTracerProvider tracerProvider;
   private final String instrumentationScopeName;
 
-  public OtelTracerBuilder(String instrumentationScopeName) {
+  OtelTracerBuilder(OtelTracerProvider tracerProvider, String instrumentationScopeName) {
+    this.tracerProvider = tracerProvider;
     this.instrumentationScopeName = instrumentationScopeName;
   }
 
   @Override
-  public TracerBuilder setSchemaUrl(String schemaUrl) {
+  public TracerBuilder setInstrumentationVersion(String instrumentationScopeVersion) {
     // Not supported
     return this;
   }
 
   @Override
-  public TracerBuilder setInstrumentationVersion(String instrumentationScopeVersion) {
+  public TracerBuilder setSchemaUrl(String schemaUrl) {
     // Not supported
     return this;
   }
 
   @Override
   public Tracer build() {
-    return new OtelTracer(this.instrumentationScopeName);
+    return tracerProvider.getTracerShim(instrumentationScopeName);
   }
 }

dd-java-agent/agent-otel/otel-shim/src/main/java/datadog/opentelemetry/shim/trace/OtelTracerProvider.java

@@ -1,56 +1,47 @@
 package datadog.opentelemetry.shim.trace;
 
+import datadog.trace.util.Strings;
 import io.opentelemetry.api.trace.Tracer;
 import io.opentelemetry.api.trace.TracerBuilder;
 import io.opentelemetry.api.trace.TracerProvider;
-import java.util.HashMap;
 import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
 import javax.annotation.ParametersAreNonnullByDefault;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 
 @ParametersAreNonnullByDefault
-public class OtelTracerProvider implements TracerProvider {
+public final class OtelTracerProvider implements TracerProvider {
   private static final Logger LOGGER = LoggerFactory.getLogger(OtelTracerProvider.class);
   private static final String DEFAULT_TRACER_NAME = "";
-  public static final TracerProvider INSTANCE = new OtelTracerProvider();
 
-  /** Tracer instances, indexed by instrumentation scope name. */
-  private final Map<String, Tracer> tracers;
+  public static final TracerProvider INSTANCE = new OtelTracerProvider();
 
-  public OtelTracerProvider() {
-    this.tracers = new HashMap<>();
-  }
+  /** Tracer shims, indexed by instrumentation scope name. */
+  private final Map<String, OtelTracer> tracers = new ConcurrentHashMap<>();
 
   @Override
   public Tracer get(String instrumentationScopeName) {
-    Tracer tracer = this.tracers.get(instrumentationScopeName);
-    if (tracer == null) {
-      tracer = tracerBuilder(instrumentationScopeName).build();
-      this.tracers.put(instrumentationScopeName, tracer);
-    }
-    return tracer;
+    return getTracerShim(instrumentationScopeName);
   }
 
   @Override
-  public Tracer get(String instrumentationScopeName, String instrumentationScopeVersion) {
-    Tracer tracer = this.tracers.get(instrumentationScopeName);
-    if (tracer == null) {
-      tracer =
-          tracerBuilder(instrumentationScopeName)
-              .setInstrumentationVersion(instrumentationScopeVersion)
-              .build();
-      this.tracers.put(instrumentationScopeName, tracer);
-    }
-    return tracer;
+  public Tracer get(
+      String instrumentationScopeName,
+      @SuppressWarnings("unused") String instrumentationScopeVersion) {
+    return getTracerShim(instrumentationScopeName);
   }
 
   @Override
   public TracerBuilder tracerBuilder(String instrumentationScopeName) {
-    if (instrumentationScopeName.trim().isEmpty()) {
+    return new OtelTracerBuilder(this, instrumentationScopeName);
+  }
+
+  OtelTracer getTracerShim(String instrumentationScopeName) {
+    if (Strings.isBlank(instrumentationScopeName)) {
       LOGGER.debug("Tracer requested without instrumentation scope name.");
       instrumentationScopeName = DEFAULT_TRACER_NAME;
     }
-    return new OtelTracerBuilder(instrumentationScopeName);
+    return tracers.computeIfAbsent(instrumentationScopeName, OtelTracer::new);
   }
 }

dd-java-agent/instrumentation/opentelemetry/opentelemetry-1.47/build.gradle

@@ -0,0 +1,23 @@
+def openTelemetryVersion = '1.47.0'
+
+muzzle {
+  pass {
+    module = 'opentelemetry-api'
+    group = 'io.opentelemetry'
+    versions = "[$openTelemetryVersion,)"
+  }
+}
+
+apply from: "$rootDir/gradle/java.gradle"
+
+addTestSuiteForDir('latestDepTest', 'test')
+
+dependencies {
+  compileOnly group: 'io.opentelemetry', name: 'opentelemetry-api', version: openTelemetryVersion
+  compileOnly group: 'com.google.auto.value', name: 'auto-value-annotations', version: '1.6.6'
+
+  implementation project(':dd-java-agent:agent-otel:otel-shim')
+
+  testImplementation group: 'io.opentelemetry', name: 'opentelemetry-api', version: openTelemetryVersion
+  latestDepTestImplementation group: 'io.opentelemetry', name: 'opentelemetry-api', version: '1+'
+}