[Java] Custom Metrics E2E Test

.github/workflows/java-ec2-default-test.yml

@@ -251,9 +251,27 @@ jobs:
           --instance-id ${{ env.MAIN_SERVICE_INSTANCE_ID }}
           --rollup'
 
+      - name: Validate custom metrics
+        id: cwagent-metric-validation
+        if: (success() || steps.log-validation.outcome == 'failure' || steps.metric-validation.outcome == 'failure') && !cancelled()
+        run: ./gradlew validator:run --args='-c java/ec2/default/custom-metric-validation.yml
+          --testing-id ${{ env.TESTING_ID }}
+          --endpoint http://${{ env.MAIN_SERVICE_ENDPOINT }}
+          --remote-service-deployment-name ${{ env.REMOTE_SERVICE_IP }}:8080
+          --region ${{ inputs.aws-region }}
+          --account-id ${{ env.ACCOUNT_ID }}
+          --metric-namespace CWAgent
+          --log-group ${{ env.LOG_GROUP_NAME }}
+          --service-name sample-application-${{ env.TESTING_ID }}
+          --remote-service-name sample-remote-application-${{ env.TESTING_ID }}
+          --query-string ip=${{ env.REMOTE_SERVICE_IP }}&testingId=${{ env.TESTING_ID }}
+          --instance-ami ${{ env.EC2_INSTANCE_AMI }}
+          --instance-id ${{ env.MAIN_SERVICE_INSTANCE_ID }}
+          --rollup'
+
       - name: Validate generated traces
         id: trace-validation
-        if: (success() || steps.log-validation.outcome == 'failure' || steps.metric-validation.outcome == 'failure') && !cancelled()
+        if: (success() || steps.log-validation.outcome == 'failure' || steps.metric-validation.outcome == 'failure' || steps.cwagent-metric-validation.outcome == 'failure') && !cancelled()
         run: ./gradlew validator:run --args='-c java/ec2/default/trace-validation.yml
           --testing-id ${{ env.TESTING_ID }}
           --endpoint http://${{ env.MAIN_SERVICE_ENDPOINT }}
@@ -280,7 +298,7 @@ jobs:
         if: always()
         id: validation-result
         run: |
-          if [ "${{ steps.log-validation.outcome }}" = "success" ] && [ "${{ steps.metric-validation.outcome }}" = "success" ] && [ "${{ steps.trace-validation.outcome }}" = "success" ]; then
+          if [ "${{ steps.log-validation.outcome }}" = "success" ] && [ "${{ steps.metric-validation.outcome }}" = "success" ] && [ "${{ steps.cwagent-metric-validation.outcome }}" = "success" ] && [ "${{ steps.trace-validation.outcome }}" = "success" ]; then
             echo "validation-result=success" >> $GITHUB_OUTPUT
           else
             echo "validation-result=failure" >> $GITHUB_OUTPUT

sample-apps/java/springboot-main-service/build.gradle.kts

@@ -43,6 +43,10 @@ dependencies {
   implementation("org.springframework.boot:spring-boot-starter-web")
   implementation("org.springframework.boot:spring-boot-starter-logging")
   implementation("io.opentelemetry:opentelemetry-api:1.34.1")
+  implementation("io.opentelemetry:opentelemetry-sdk:1.34.1")
+  implementation("io.opentelemetry:opentelemetry-sdk-metrics:1.34.1")
+  implementation("io.opentelemetry:opentelemetry-exporter-otlp:1.34.1")
+  implementation("io.opentelemetry:opentelemetry-sdk-extension-autoconfigure:1.34.1")
   implementation("software.amazon.awssdk:s3")
   implementation("software.amazon.awssdk:sts")
   implementation("com.mysql:mysql-connector-j:8.4.0")

sample-apps/java/springboot-main-service/src/main/java/com/amazon/sampleapp/FrontendServiceController.java

@@ -45,6 +45,22 @@
 import org.springframework.web.bind.annotation.ResponseBody;
 import software.amazon.awssdk.services.s3.S3Client;
 import software.amazon.awssdk.services.s3.model.GetBucketLocationRequest;
+import io.opentelemetry.api.GlobalOpenTelemetry;
+import io.opentelemetry.api.metrics.Meter;
+import io.opentelemetry.api.metrics.LongCounter;
+import io.opentelemetry.api.common.Attributes;
+import io.opentelemetry.api.common.AttributeKey;
+import io.opentelemetry.exporter.otlp.http.metrics.OtlpHttpMetricExporter;
+import io.opentelemetry.sdk.metrics.export.MetricExporter;
+import io.opentelemetry.sdk.metrics.SdkMeterProvider;
+import io.opentelemetry.sdk.resources.Resource;
+import io.opentelemetry.sdk.metrics.export.MetricReader;
+import io.opentelemetry.sdk.metrics.export.PeriodicMetricReader;
+import io.opentelemetry.api.OpenTelemetry;
+import io.opentelemetry.sdk.OpenTelemetrySdk;
+import io.opentelemetry.api.metrics.DoubleHistogram;
+import io.opentelemetry.api.metrics.LongUpDownCounter;
+import java.time.Duration;
 
 @Controller
 public class FrontendServiceController {
@@ -76,10 +92,74 @@ private void runLocalRootClientCallRecurringService() { // run the service
     executorService.scheduleAtFixedRate(runnableTask, 100, 1000, TimeUnit.MILLISECONDS);
   }
 
+  // Agent-based metrics using GlobalOpenTelemetry
+  private final Meter agentMeter;
+  private final LongCounter agentBasedCounter;
+  private final DoubleHistogram agentBasedHistogram;
+  private final LongUpDownCounter agentBasedGauge;
+
+  // Pipeline-based metrics (conditionally initialized)
+  private final Meter customPipelineMeter;
+  private final LongCounter customPipelineCounter;
+  private final DoubleHistogram customPipelineHistogram;
+  private final LongUpDownCounter customPipelineGauge;
+
   @Autowired
   public FrontendServiceController(CloseableHttpClient httpClient, S3Client s3) {
     this.httpClient = httpClient;
     this.s3 = s3;
+
+    // Initialize agent-based metrics using GLOBAL OpenTelemetry (ADOT agent's configuration)
+    this.agentMeter = GlobalOpenTelemetry.get().getMeter("agent-meter");
+
+    this.agentBasedCounter = agentMeter.counterBuilder("agent_based_counter").build();
+    this.agentBasedHistogram = agentMeter.histogramBuilder("agent_based_histogram").build();
+    this.agentBasedGauge = agentMeter.upDownCounterBuilder("agent_based_gauge").build();
+
+    // Get environment variables
+    String serviceName = System.getenv("SERVICE_NAME");
+    String deploymentEnvironmentName = System.getenv("DEPLOYMENT_ENVIRONMENT_NAME");
+
+    // Only create pipeline if environment variables exist (matching Python logic)
+    if (serviceName != null && deploymentEnvironmentName != null && 
+        !serviceName.isEmpty() && !deploymentEnvironmentName.isEmpty()) {
+
+        Resource pipelineResource = Resource.getDefault().toBuilder()
+            .put("service.name", serviceName)
+            .put("deployment.environment.name", deploymentEnvironmentName)
+            .build();
+
+        MetricExporter pipelineMetricExporter = OtlpHttpMetricExporter.builder()
+            .setEndpoint("http://localhost:4318/v1/metrics")
+            .setTimeout(Duration.ofSeconds(10))
+            .build();
+
+        MetricReader pipelineMetricReader = PeriodicMetricReader.builder(pipelineMetricExporter)
+            .setInterval(Duration.ofSeconds(1))
+            .build();
+
+        SdkMeterProvider pipelineMeterProvider = SdkMeterProvider.builder()
+            .setResource(pipelineResource)
+            .registerMetricReader(pipelineMetricReader)
+            .build();
+
+        // Initialize pipeline metrics using SEPARATE SdkMeterProvider
+        this.customPipelineMeter = pipelineMeterProvider.get("pipeline-meter");
+
+        this.customPipelineCounter = customPipelineMeter.counterBuilder("custom_pipeline_counter").build();
+        this.customPipelineHistogram = customPipelineMeter.histogramBuilder("custom_pipeline_histogram").build();
+        this.customPipelineGauge = customPipelineMeter.upDownCounterBuilder("custom_pipeline_gauge").build();
+    } else {
+        // No pipeline metrics if environment variables missing
+        this.customPipelineMeter = null;
+        this.customPipelineCounter = null;
+        this.customPipelineHistogram = null;
+        this.customPipelineGauge = null;
+    }
+  }
+
+  private int random(int min, int max) {
+    return (int) (Math.random() * (max - min + 1)) + min;
   }
 
   @GetMapping("/")
@@ -92,6 +172,29 @@ public String healthcheck() {
   @GetMapping("/aws-sdk-call")
   @ResponseBody
   public String awssdkCall(@RequestParam(name = "testingId", required = false) String testingId) {
+
+    // Record agent-based metrics
+    int histogramValue = random(100,1000);
+    int gaugeValue = random(-10,10);
+
+    agentBasedCounter.add(1, Attributes.of(AttributeKey.stringKey("Operation"), "counter"));
+
+    agentBasedHistogram.record((double)histogramValue, Attributes.of(AttributeKey.stringKey("Operation"), "histogram"));
+
+    agentBasedGauge.add(gaugeValue, Attributes.of(AttributeKey.stringKey("Operation"), "gauge"));
+
+    // Only record pipeline metrics if pipeline exists (matching Python logic)
+    if (customPipelineCounter != null) {
+        int pipelineHistogramValue = random(100,1000);
+        int pipelineGaugeValue = random(-10,10);
+
+        customPipelineCounter.add(1, Attributes.of(AttributeKey.stringKey("Operation"), "pipeline_counter"));
+
+        customPipelineHistogram.record(pipelineHistogramValue, Attributes.of(AttributeKey.stringKey("Operation"), "pipeline_histogram"));
+
+        customPipelineGauge.add(pipelineGaugeValue, Attributes.of(AttributeKey.stringKey("Operation"), "pipeline_gauge"));
+    }
+
     String bucketName = "e2e-test-bucket-name";
     if (testingId != null) {
       bucketName += "-" + testingId;
@@ -186,4 +289,4 @@ private String getXrayTraceId() {
     String xrayTraceId = "1-" + traceId.substring(0, 8) + "-" + traceId.substring(8);
     return String.format("{\"traceId\": \"%s\"}", xrayTraceId);
   }
-}
+}

terraform/java/ec2/asg/main.tf

@@ -134,7 +134,7 @@ resource "aws_launch_configuration" "launch_configuration" {
     OTEL_AWS_APPLICATION_SIGNALS_EXPORTER_ENDPOINT=http://localhost:4316/v1/metrics \
     OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf \
     OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=http://localhost:4316/v1/traces \
-    OTEL_RESOURCE_ATTRIBUTES=service.name=sample-application-${var.test_id},aws.application_signals.metric_resource_keys=all_attributes \
+    OTEL_RESOURCE_ATTRIBUTES="service.name=sample-application-${var.test_id},Internal_Org=Financial,Business Unit=Payments,Region=us-east-1,aws.application_signals.metric_resource_keys=Business Unit&Region&Organization" \
     OTEL_INSTRUMENTATION_COMMON_EXPERIMENTAL_CONTROLLER_TELEMETRY_ENABLED=true \
     nohup java -jar -XX:+UseG1GC main-service.jar &> nohup.out &
 

terraform/java/ec2/default/amazon-cloudwatch-agent.json

@@ -10,7 +10,11 @@
   },
   "logs": {
     "metrics_collected": {
-      "application_signals": {}
+      "application_signals": {},
+      "otlp": {
+        "grpc_endpoint": "0.0.0.0:4317",
+        "http_endpoint": "0.0.0.0:4318"
+      }
     }
   }
 }

terraform/java/ec2/default/main.tf

@@ -122,6 +122,9 @@ resource "null_resource" "main_service_setup" {
         sudo yum install java-${var.language_version}-amazon-corretto -y
       fi
 
+      # enable ec2 instance connect for debug
+      sudo yum install ec2-instance-connect -y
+
       # Copy in CW Agent configuration
       agent_config='${replace(replace(file("./amazon-cloudwatch-agent.json"), "/\\s+/", ""), "$REGION", var.aws_region)}'
       echo $agent_config > amazon-cloudwatch-agent.json
@@ -137,16 +140,22 @@ resource "null_resource" "main_service_setup" {
       # Get and run the sample application with configuration
       aws s3 cp ${var.sample_app_jar} ./main-service.jar
 
+      export SERVICE_NAME='sample-application-${var.test_id}' 
+      export DEPLOYMENT_ENVIRONMENT_NAME='ec2:default' 
+
       JAVA_TOOL_OPTIONS=' -javaagent:/home/ec2-user/adot.jar' \
-      OTEL_METRICS_EXPORTER=none \
+      OTEL_METRICS_EXPORTER=otlp \
       OTEL_LOGS_EXPORT=none \
       OTEL_AWS_APPLICATION_SIGNALS_ENABLED=true \
       OTEL_AWS_APPLICATION_SIGNALS_EXPORTER_ENDPOINT=http://localhost:4316/v1/metrics \
-      OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf \
       OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=http://localhost:4316/v1/traces \
-      OTEL_RESOURCE_ATTRIBUTES="service.name=sample-application-${var.test_id},Internal_Org=Financial,Business Unit=Payments,Region=us-east-1,aws.application_signals.metric_resource_keys=Business Unit&Region&Organization" \
+      OTEL_EXPORTER_OTLP_METRICS_ENDPOINT=http://localhost:4318/v1/metrics \
       OTEL_INSTRUMENTATION_COMMON_EXPERIMENTAL_CONTROLLER_TELEMETRY_ENABLED=true \
-      nohup java -XX:+UseG1GC -jar main-service.jar &> nohup.out &
+      SERVICE_NAME='sample-application-${var.test_id}' \
+      DEPLOYMENT_ENVIRONMENT_NAME='ec2:default' \
+      OTEL_RESOURCE_ATTRIBUTES="service.name=$${SERVICE_NAME},deployment.environment.name=$${DEPLOYMENT_ENVIRONMENT_NAME},aws.application_signals.metric_resource_keys=all_attributes" \
+      AWS_REGION='${var.aws_region}' \
+      nohup java -Dotel.java.global-autoconfigure.enabled=true -XX:+UseG1GC -jar main-service.jar &> nohup.out &
 
       # The application needs time to come up and reach a steady state, this should not take longer than 30 seconds
       sleep 30

validator/src/main/java/com/amazon/aoc/fileconfigs/PredefinedExpectedTemplate.java

@@ -71,6 +71,9 @@ public enum PredefinedExpectedTemplate implements FileConfig {
   JAVA_EC2_DEFAULT_AWS_SDK_CALL_METRIC("/expected-data-template/java/ec2/default/aws-sdk-call-metric.mustache"),
   JAVA_EC2_DEFAULT_AWS_SDK_CALL_TRACE("/expected-data-template/java/ec2/default/aws-sdk-call-trace.mustache"),
 
+    /** Java EC2 Default Custom Metrics Test Case Validations */
+  JAVA_EC2_DEFAULT_AWS_OTEL_CUSTOM_METRIC("/expected-data-template/java/ec2/default/aws-otel-custom-metrics.mustache"),
+
   JAVA_EC2_DEFAULT_REMOTE_SERVICE_LOG("/expected-data-template/java/ec2/default/remote-service-log.mustache"),
   JAVA_EC2_DEFAULT_REMOTE_SERVICE_METRIC("/expected-data-template/java/ec2/default/remote-service-metric.mustache"),
   JAVA_EC2_DEFAULT_REMOTE_SERVICE_TRACE("/expected-data-template/java/ec2/default/remote-service-trace.mustache"),

validator/src/main/resources/expected-data-template/java/ec2/asg/aws-sdk-call-log.mustache

@@ -7,9 +7,8 @@
   "Version": "^1$",
   "Telemetry.Source": "^LocalRootSpan$",
   "Host": "^{{privateDnsName}}$",
-  "otel.resource.aws.application_signals.metric_resource_keys": "all_attributes",
-  "otel.resource.host.image.id": "^{{instanceAmi}}$",
-  "otel.resource.host.type": "^([a-z0-9]+\\.[a-z0-9]+)$"
+  "otel.resource.Business Unit": "Payments",
+  "otel.resource.Region": "us-east-1"
 },
 {
   "EC2.AutoScalingGroup": "^{{platformInfo}}$",
@@ -24,7 +23,6 @@
   "RemoteResourceType": "^AWS::S3::Bucket$",
   "Telemetry.Source": "^ClientSpan$",
   "Host": "^{{privateDnsName}}$",
-  "otel.resource.aws.application_signals.metric_resource_keys": "all_attributes",
-  "otel.resource.host.image.id": "^{{instanceAmi}}$",
-  "otel.resource.host.type": "^([a-z0-9]+\\.[a-z0-9]+)$"
+  "otel.resource.Business Unit": "Payments",
+  "otel.resource.Region": "us-east-1"
 }]

validator/src/main/resources/expected-data-template/java/ec2/asg/aws-sdk-call-trace.mustache

@@ -21,7 +21,10 @@
     "default": {
       "EC2.AutoScalingGroup": "^{{platformInfo}}$",
       "EC2.InstanceId": "^{{instanceId}}$",
-      "otel.resource.aws.application_signals.metric_resource_keys": "all_attributes",
+      "otel.resource.Internal_Org": "Financial",
+      "otel.resource.Business Unit": "Payments",
+      "otel.resource.Region": "us-east-1",
+      "otel.resource.aws.application_signals.metric_resource_keys": "Business Unit&Region&Organization",
       "otel.resource.ec2.tag.aws:autoscaling:groupName": "^{{platformInfo}}$",
       "otel.resource.host.id": "^{{instanceId}}$",
       "otel.resource.host.image.id": "^{{instanceAmi}}$",

validator/src/main/resources/expected-data-template/java/ec2/asg/client-call-log.mustache

@@ -7,9 +7,8 @@
   "Version": "^1$",
   "Telemetry.Source": "^LocalRootSpan$",
   "Host": "^{{privateDnsName}}$",
-  "otel.resource.aws.application_signals.metric_resource_keys": "all_attributes",
-  "otel.resource.host.image.id": "^{{instanceAmi}}$",
-  "otel.resource.host.type": "^([a-z0-9]+\\.[a-z0-9]+)$"
+  "otel.resource.Business Unit": "Payments",
+  "otel.resource.Region": "us-east-1"
 },
 {
   "EC2.AutoScalingGroup": "^{{platformInfo}}$",
@@ -22,7 +21,6 @@
   "RemoteOperation": "GET /",
   "Telemetry.Source": "^ClientSpan$",
   "Host": "^{{privateDnsName}}$",
-  "otel.resource.aws.application_signals.metric_resource_keys": "all_attributes",
-  "otel.resource.host.image.id": "^{{instanceAmi}}$",
-  "otel.resource.host.type": "^([a-z0-9]+\\.[a-z0-9]+)$"
+  "otel.resource.Business Unit": "Payments",
+  "otel.resource.Region": "us-east-1"
 }]

validator/src/main/resources/expected-data-template/java/ec2/asg/client-call-trace.mustache

@@ -7,7 +7,10 @@
   },
   "metadata": {
     "default": {
-      "otel.resource.aws.application_signals.metric_resource_keys": "all_attributes",
+      "otel.resource.Internal_Org": "Financial",
+      "otel.resource.Business Unit": "Payments",
+      "otel.resource.Region": "us-east-1",
+      "otel.resource.aws.application_signals.metric_resource_keys": "Business Unit&Region&Organization",
       "otel.resource.ec2.tag.aws:autoscaling:groupName": "^{{platformInfo}}$",
       "otel.resource.host.id": "^{{instanceId}}$",
       "otel.resource.host.image.id": "^{{instanceAmi}}$",

validator/src/main/resources/expected-data-template/java/ec2/asg/outgoing-http-call-log.mustache

@@ -7,9 +7,8 @@
   "Version": "^1$",
   "Telemetry.Source": "^LocalRootSpan$",
   "Host": "^{{privateDnsName}}$",
-  "otel.resource.aws.application_signals.metric_resource_keys": "all_attributes",
-  "otel.resource.host.image.id": "^{{instanceAmi}}$",
-  "otel.resource.host.type": "^([a-z0-9]+\\.[a-z0-9]+)$"
+  "otel.resource.Business Unit": "Payments",
+  "otel.resource.Region": "us-east-1"
 },
 {
   "EC2.AutoScalingGroup": "^{{platformInfo}}$",
@@ -22,7 +21,6 @@
   "RemoteOperation": "GET /",
   "Telemetry.Source": "^ClientSpan$",
   "Host": "^{{privateDnsName}}$",
-  "otel.resource.aws.application_signals.metric_resource_keys": "all_attributes",
-  "otel.resource.host.image.id": "^{{instanceAmi}}$",
-  "otel.resource.host.type": "^([a-z0-9]+\\.[a-z0-9]+)$"
+  "otel.resource.Business Unit": "Payments",
+  "otel.resource.Region": "us-east-1"
 }]

validator/src/main/resources/expected-data-template/java/ec2/asg/outgoing-http-call-trace.mustache

@@ -21,7 +21,10 @@
     "default": {
       "EC2.AutoScalingGroup": "^{{platformInfo}}$",
       "EC2.InstanceId": "^{{instanceId}}$",
-      "otel.resource.aws.application_signals.metric_resource_keys": "all_attributes",
+      "otel.resource.Internal_Org": "Financial",
+      "otel.resource.Business Unit": "Payments",
+      "otel.resource.Region": "us-east-1",
+      "otel.resource.aws.application_signals.metric_resource_keys": "Business Unit&Region&Organization",
       "otel.resource.ec2.tag.aws:autoscaling:groupName": "^{{platformInfo}}$",
       "otel.resource.host.id": "^{{instanceId}}$",
       "otel.resource.host.image.id": "^{{instanceAmi}}$",

validator/src/main/resources/expected-data-template/java/ec2/asg/remote-service-log.mustache

@@ -7,9 +7,8 @@
   "Version": "^1$",
   "Telemetry.Source": "^LocalRootSpan$",
   "Host": "^{{privateDnsName}}$",
-  "otel.resource.aws.application_signals.metric_resource_keys": "all_attributes",
-  "otel.resource.host.image.id": "^{{instanceAmi}}$",
-  "otel.resource.host.type": "^([a-z0-9]+\\.[a-z0-9]+)$"
+  "otel.resource.Business Unit": "Payments",
+  "otel.resource.Region": "us-east-1"
 },
 {
   "EC2.AutoScalingGroup": "^{{platformInfo}}$",
@@ -22,7 +21,6 @@
   "RemoteOperation": "GET /healthcheck",
   "Telemetry.Source": "^ClientSpan$",
   "Host": "^{{privateDnsName}}$",
-  "otel.resource.aws.application_signals.metric_resource_keys": "all_attributes",
-  "otel.resource.host.image.id": "^{{instanceAmi}}$",
-  "otel.resource.host.type": "^([a-z0-9]+\\.[a-z0-9]+)$"
+  "otel.resource.Business Unit": "Payments",
+  "otel.resource.Region": "us-east-1"
 }]

validator/src/main/resources/expected-data-template/java/ec2/asg/remote-service-trace.mustache

@@ -22,7 +22,10 @@
       "EC2.AutoScalingGroup": "^{{platformInfo}}$",
       "EC2.InstanceId": "^{{instanceId}}$",
       "PlatformType": "^AWS::EC2$",
-      "otel.resource.aws.application_signals.metric_resource_keys": "all_attributes",
+      "otel.resource.Internal_Org": "Financial",
+      "otel.resource.Business Unit": "Payments",
+      "otel.resource.Region": "us-east-1",
+      "otel.resource.aws.application_signals.metric_resource_keys": "Business Unit&Region&Organization",
       "otel.resource.ec2.tag.aws:autoscaling:groupName": "^{{platformInfo}}$",
       "otel.resource.host.id": "^{{instanceId}}$",
       "otel.resource.host.image.id": "^{{instanceAmi}}$",