Skip to content

Commit 6d1ff60

Browse files
authored
[Java] Custom Metrics E2E Test (#479)
1 parent 57724b3 commit 6d1ff60

25 files changed

+560
-69
lines changed

.github/workflows/java-ec2-default-test.yml

Lines changed: 20 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -251,9 +251,27 @@ jobs:
251251
--instance-id ${{ env.MAIN_SERVICE_INSTANCE_ID }}
252252
--rollup'
253253

254+
- name: Validate custom metrics
255+
id: cwagent-metric-validation
256+
if: (success() || steps.log-validation.outcome == 'failure' || steps.metric-validation.outcome == 'failure') && !cancelled()
257+
run: ./gradlew validator:run --args='-c java/ec2/default/custom-metric-validation.yml
258+
--testing-id ${{ env.TESTING_ID }}
259+
--endpoint http://${{ env.MAIN_SERVICE_ENDPOINT }}
260+
--remote-service-deployment-name ${{ env.REMOTE_SERVICE_IP }}:8080
261+
--region ${{ inputs.aws-region }}
262+
--account-id ${{ env.ACCOUNT_ID }}
263+
--metric-namespace CWAgent
264+
--log-group ${{ env.LOG_GROUP_NAME }}
265+
--service-name sample-application-${{ env.TESTING_ID }}
266+
--remote-service-name sample-remote-application-${{ env.TESTING_ID }}
267+
--query-string ip=${{ env.REMOTE_SERVICE_IP }}&testingId=${{ env.TESTING_ID }}
268+
--instance-ami ${{ env.EC2_INSTANCE_AMI }}
269+
--instance-id ${{ env.MAIN_SERVICE_INSTANCE_ID }}
270+
--rollup'
271+
254272
- name: Validate generated traces
255273
id: trace-validation
256-
if: (success() || steps.log-validation.outcome == 'failure' || steps.metric-validation.outcome == 'failure') && !cancelled()
274+
if: (success() || steps.log-validation.outcome == 'failure' || steps.metric-validation.outcome == 'failure' || steps.cwagent-metric-validation.outcome == 'failure') && !cancelled()
257275
run: ./gradlew validator:run --args='-c java/ec2/default/trace-validation.yml
258276
--testing-id ${{ env.TESTING_ID }}
259277
--endpoint http://${{ env.MAIN_SERVICE_ENDPOINT }}
@@ -280,7 +298,7 @@ jobs:
280298
if: always()
281299
id: validation-result
282300
run: |
283-
if [ "${{ steps.log-validation.outcome }}" = "success" ] && [ "${{ steps.metric-validation.outcome }}" = "success" ] && [ "${{ steps.trace-validation.outcome }}" = "success" ]; then
301+
if [ "${{ steps.log-validation.outcome }}" = "success" ] && [ "${{ steps.metric-validation.outcome }}" = "success" ] && [ "${{ steps.cwagent-metric-validation.outcome }}" = "success" ] && [ "${{ steps.trace-validation.outcome }}" = "success" ]; then
284302
echo "validation-result=success" >> $GITHUB_OUTPUT
285303
else
286304
echo "validation-result=failure" >> $GITHUB_OUTPUT

sample-apps/java/springboot-main-service/build.gradle.kts

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,10 @@ dependencies {
4343
implementation("org.springframework.boot:spring-boot-starter-web")
4444
implementation("org.springframework.boot:spring-boot-starter-logging")
4545
implementation("io.opentelemetry:opentelemetry-api:1.34.1")
46+
implementation("io.opentelemetry:opentelemetry-sdk:1.34.1")
47+
implementation("io.opentelemetry:opentelemetry-sdk-metrics:1.34.1")
48+
implementation("io.opentelemetry:opentelemetry-exporter-otlp:1.34.1")
49+
implementation("io.opentelemetry:opentelemetry-sdk-extension-autoconfigure:1.34.1")
4650
implementation("software.amazon.awssdk:s3")
4751
implementation("software.amazon.awssdk:sts")
4852
implementation("com.mysql:mysql-connector-j:8.4.0")

sample-apps/java/springboot-main-service/src/main/java/com/amazon/sampleapp/FrontendServiceController.java

Lines changed: 104 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,22 @@
4545
import org.springframework.web.bind.annotation.ResponseBody;
4646
import software.amazon.awssdk.services.s3.S3Client;
4747
import software.amazon.awssdk.services.s3.model.GetBucketLocationRequest;
48+
import io.opentelemetry.api.GlobalOpenTelemetry;
49+
import io.opentelemetry.api.metrics.Meter;
50+
import io.opentelemetry.api.metrics.LongCounter;
51+
import io.opentelemetry.api.common.Attributes;
52+
import io.opentelemetry.api.common.AttributeKey;
53+
import io.opentelemetry.exporter.otlp.http.metrics.OtlpHttpMetricExporter;
54+
import io.opentelemetry.sdk.metrics.export.MetricExporter;
55+
import io.opentelemetry.sdk.metrics.SdkMeterProvider;
56+
import io.opentelemetry.sdk.resources.Resource;
57+
import io.opentelemetry.sdk.metrics.export.MetricReader;
58+
import io.opentelemetry.sdk.metrics.export.PeriodicMetricReader;
59+
import io.opentelemetry.api.OpenTelemetry;
60+
import io.opentelemetry.sdk.OpenTelemetrySdk;
61+
import io.opentelemetry.api.metrics.DoubleHistogram;
62+
import io.opentelemetry.api.metrics.LongUpDownCounter;
63+
import java.time.Duration;
4864

4965
@Controller
5066
public class FrontendServiceController {
@@ -76,10 +92,74 @@ private void runLocalRootClientCallRecurringService() { // run the service
7692
executorService.scheduleAtFixedRate(runnableTask, 100, 1000, TimeUnit.MILLISECONDS);
7793
}
7894

95+
// Agent-based metrics using GlobalOpenTelemetry
96+
private final Meter agentMeter;
97+
private final LongCounter agentBasedCounter;
98+
private final DoubleHistogram agentBasedHistogram;
99+
private final LongUpDownCounter agentBasedGauge;
100+
101+
// Pipeline-based metrics (conditionally initialized)
102+
private final Meter customPipelineMeter;
103+
private final LongCounter customPipelineCounter;
104+
private final DoubleHistogram customPipelineHistogram;
105+
private final LongUpDownCounter customPipelineGauge;
106+
79107
@Autowired
80108
public FrontendServiceController(CloseableHttpClient httpClient, S3Client s3) {
81109
this.httpClient = httpClient;
82110
this.s3 = s3;
111+
112+
// Initialize agent-based metrics using GLOBAL OpenTelemetry (ADOT agent's configuration)
113+
this.agentMeter = GlobalOpenTelemetry.get().getMeter("agent-meter");
114+
115+
this.agentBasedCounter = agentMeter.counterBuilder("agent_based_counter").build();
116+
this.agentBasedHistogram = agentMeter.histogramBuilder("agent_based_histogram").build();
117+
this.agentBasedGauge = agentMeter.upDownCounterBuilder("agent_based_gauge").build();
118+
119+
// Get environment variables
120+
String serviceName = System.getenv("SERVICE_NAME");
121+
String deploymentEnvironmentName = System.getenv("DEPLOYMENT_ENVIRONMENT_NAME");
122+
123+
// Only create pipeline if environment variables exist (matching Python logic)
124+
if (serviceName != null && deploymentEnvironmentName != null &&
125+
!serviceName.isEmpty() && !deploymentEnvironmentName.isEmpty()) {
126+
127+
Resource pipelineResource = Resource.getDefault().toBuilder()
128+
.put("service.name", serviceName)
129+
.put("deployment.environment.name", deploymentEnvironmentName)
130+
.build();
131+
132+
MetricExporter pipelineMetricExporter = OtlpHttpMetricExporter.builder()
133+
.setEndpoint("http://localhost:4318/v1/metrics")
134+
.setTimeout(Duration.ofSeconds(10))
135+
.build();
136+
137+
MetricReader pipelineMetricReader = PeriodicMetricReader.builder(pipelineMetricExporter)
138+
.setInterval(Duration.ofSeconds(1))
139+
.build();
140+
141+
SdkMeterProvider pipelineMeterProvider = SdkMeterProvider.builder()
142+
.setResource(pipelineResource)
143+
.registerMetricReader(pipelineMetricReader)
144+
.build();
145+
146+
// Initialize pipeline metrics using SEPARATE SdkMeterProvider
147+
this.customPipelineMeter = pipelineMeterProvider.get("pipeline-meter");
148+
149+
this.customPipelineCounter = customPipelineMeter.counterBuilder("custom_pipeline_counter").build();
150+
this.customPipelineHistogram = customPipelineMeter.histogramBuilder("custom_pipeline_histogram").build();
151+
this.customPipelineGauge = customPipelineMeter.upDownCounterBuilder("custom_pipeline_gauge").build();
152+
} else {
153+
// No pipeline metrics if environment variables missing
154+
this.customPipelineMeter = null;
155+
this.customPipelineCounter = null;
156+
this.customPipelineHistogram = null;
157+
this.customPipelineGauge = null;
158+
}
159+
}
160+
161+
private int random(int min, int max) {
162+
return (int) (Math.random() * (max - min + 1)) + min;
83163
}
84164

85165
@GetMapping("/")
@@ -92,6 +172,29 @@ public String healthcheck() {
92172
@GetMapping("/aws-sdk-call")
93173
@ResponseBody
94174
public String awssdkCall(@RequestParam(name = "testingId", required = false) String testingId) {
175+
176+
// Record agent-based metrics
177+
int histogramValue = random(100,1000);
178+
int gaugeValue = random(-10,10);
179+
180+
agentBasedCounter.add(1, Attributes.of(AttributeKey.stringKey("Operation"), "counter"));
181+
182+
agentBasedHistogram.record((double)histogramValue, Attributes.of(AttributeKey.stringKey("Operation"), "histogram"));
183+
184+
agentBasedGauge.add(gaugeValue, Attributes.of(AttributeKey.stringKey("Operation"), "gauge"));
185+
186+
// Only record pipeline metrics if pipeline exists (matching Python logic)
187+
if (customPipelineCounter != null) {
188+
int pipelineHistogramValue = random(100,1000);
189+
int pipelineGaugeValue = random(-10,10);
190+
191+
customPipelineCounter.add(1, Attributes.of(AttributeKey.stringKey("Operation"), "pipeline_counter"));
192+
193+
customPipelineHistogram.record(pipelineHistogramValue, Attributes.of(AttributeKey.stringKey("Operation"), "pipeline_histogram"));
194+
195+
customPipelineGauge.add(pipelineGaugeValue, Attributes.of(AttributeKey.stringKey("Operation"), "pipeline_gauge"));
196+
}
197+
95198
String bucketName = "e2e-test-bucket-name";
96199
if (testingId != null) {
97200
bucketName += "-" + testingId;
@@ -186,4 +289,4 @@ private String getXrayTraceId() {
186289
String xrayTraceId = "1-" + traceId.substring(0, 8) + "-" + traceId.substring(8);
187290
return String.format("{\"traceId\": \"%s\"}", xrayTraceId);
188291
}
189-
}
292+
}

terraform/java/ec2/asg/main.tf

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -134,7 +134,7 @@ resource "aws_launch_configuration" "launch_configuration" {
134134
OTEL_AWS_APPLICATION_SIGNALS_EXPORTER_ENDPOINT=http://localhost:4316/v1/metrics \
135135
OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf \
136136
OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=http://localhost:4316/v1/traces \
137-
OTEL_RESOURCE_ATTRIBUTES=service.name=sample-application-${var.test_id},aws.application_signals.metric_resource_keys=all_attributes \
137+
OTEL_RESOURCE_ATTRIBUTES="service.name=sample-application-${var.test_id},Internal_Org=Financial,Business Unit=Payments,Region=us-east-1,aws.application_signals.metric_resource_keys=Business Unit&Region&Organization" \
138138
OTEL_INSTRUMENTATION_COMMON_EXPERIMENTAL_CONTROLLER_TELEMETRY_ENABLED=true \
139139
nohup java -jar -XX:+UseG1GC main-service.jar &> nohup.out &
140140

terraform/java/ec2/default/amazon-cloudwatch-agent.json

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,11 @@
1010
},
1111
"logs": {
1212
"metrics_collected": {
13-
"application_signals": {}
13+
"application_signals": {},
14+
"otlp": {
15+
"grpc_endpoint": "0.0.0.0:4317",
16+
"http_endpoint": "0.0.0.0:4318"
17+
}
1418
}
1519
}
1620
}

terraform/java/ec2/default/main.tf

Lines changed: 13 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -122,6 +122,9 @@ resource "null_resource" "main_service_setup" {
122122
sudo yum install java-${var.language_version}-amazon-corretto -y
123123
fi
124124
125+
# enable ec2 instance connect for debug
126+
sudo yum install ec2-instance-connect -y
127+
125128
# Copy in CW Agent configuration
126129
agent_config='${replace(replace(file("./amazon-cloudwatch-agent.json"), "/\\s+/", ""), "$REGION", var.aws_region)}'
127130
echo $agent_config > amazon-cloudwatch-agent.json
@@ -137,16 +140,22 @@ resource "null_resource" "main_service_setup" {
137140
# Get and run the sample application with configuration
138141
aws s3 cp ${var.sample_app_jar} ./main-service.jar
139142
143+
export SERVICE_NAME='sample-application-${var.test_id}'
144+
export DEPLOYMENT_ENVIRONMENT_NAME='ec2:default'
145+
140146
JAVA_TOOL_OPTIONS=' -javaagent:/home/ec2-user/adot.jar' \
141-
OTEL_METRICS_EXPORTER=none \
147+
OTEL_METRICS_EXPORTER=otlp \
142148
OTEL_LOGS_EXPORT=none \
143149
OTEL_AWS_APPLICATION_SIGNALS_ENABLED=true \
144150
OTEL_AWS_APPLICATION_SIGNALS_EXPORTER_ENDPOINT=http://localhost:4316/v1/metrics \
145-
OTEL_EXPORTER_OTLP_PROTOCOL=http/protobuf \
146151
OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=http://localhost:4316/v1/traces \
147-
OTEL_RESOURCE_ATTRIBUTES="service.name=sample-application-${var.test_id},Internal_Org=Financial,Business Unit=Payments,Region=us-east-1,aws.application_signals.metric_resource_keys=Business Unit&Region&Organization" \
152+
OTEL_EXPORTER_OTLP_METRICS_ENDPOINT=http://localhost:4318/v1/metrics \
148153
OTEL_INSTRUMENTATION_COMMON_EXPERIMENTAL_CONTROLLER_TELEMETRY_ENABLED=true \
149-
nohup java -XX:+UseG1GC -jar main-service.jar &> nohup.out &
154+
SERVICE_NAME='sample-application-${var.test_id}' \
155+
DEPLOYMENT_ENVIRONMENT_NAME='ec2:default' \
156+
OTEL_RESOURCE_ATTRIBUTES="service.name=$${SERVICE_NAME},deployment.environment.name=$${DEPLOYMENT_ENVIRONMENT_NAME},aws.application_signals.metric_resource_keys=all_attributes" \
157+
AWS_REGION='${var.aws_region}' \
158+
nohup java -Dotel.java.global-autoconfigure.enabled=true -XX:+UseG1GC -jar main-service.jar &> nohup.out &
150159
151160
# The application needs time to come up and reach a steady state, this should not take longer than 30 seconds
152161
sleep 30

validator/src/main/java/com/amazon/aoc/fileconfigs/PredefinedExpectedTemplate.java

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -71,6 +71,9 @@ public enum PredefinedExpectedTemplate implements FileConfig {
7171
JAVA_EC2_DEFAULT_AWS_SDK_CALL_METRIC("/expected-data-template/java/ec2/default/aws-sdk-call-metric.mustache"),
7272
JAVA_EC2_DEFAULT_AWS_SDK_CALL_TRACE("/expected-data-template/java/ec2/default/aws-sdk-call-trace.mustache"),
7373

74+
/** Java EC2 Default Custom Metrics Test Case Validations */
75+
JAVA_EC2_DEFAULT_AWS_OTEL_CUSTOM_METRIC("/expected-data-template/java/ec2/default/aws-otel-custom-metrics.mustache"),
76+
7477
JAVA_EC2_DEFAULT_REMOTE_SERVICE_LOG("/expected-data-template/java/ec2/default/remote-service-log.mustache"),
7578
JAVA_EC2_DEFAULT_REMOTE_SERVICE_METRIC("/expected-data-template/java/ec2/default/remote-service-metric.mustache"),
7679
JAVA_EC2_DEFAULT_REMOTE_SERVICE_TRACE("/expected-data-template/java/ec2/default/remote-service-trace.mustache"),

validator/src/main/resources/expected-data-template/java/ec2/asg/aws-sdk-call-log.mustache

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,8 @@
77
"Version": "^1$",
88
"Telemetry.Source": "^LocalRootSpan$",
99
"Host": "^{{privateDnsName}}$",
10-
"otel.resource.aws.application_signals.metric_resource_keys": "all_attributes",
11-
"otel.resource.host.image.id": "^{{instanceAmi}}$",
12-
"otel.resource.host.type": "^([a-z0-9]+\\.[a-z0-9]+)$"
10+
"otel.resource.Business Unit": "Payments",
11+
"otel.resource.Region": "us-east-1"
1312
},
1413
{
1514
"EC2.AutoScalingGroup": "^{{platformInfo}}$",
@@ -24,7 +23,6 @@
2423
"RemoteResourceType": "^AWS::S3::Bucket$",
2524
"Telemetry.Source": "^ClientSpan$",
2625
"Host": "^{{privateDnsName}}$",
27-
"otel.resource.aws.application_signals.metric_resource_keys": "all_attributes",
28-
"otel.resource.host.image.id": "^{{instanceAmi}}$",
29-
"otel.resource.host.type": "^([a-z0-9]+\\.[a-z0-9]+)$"
26+
"otel.resource.Business Unit": "Payments",
27+
"otel.resource.Region": "us-east-1"
3028
}]

validator/src/main/resources/expected-data-template/java/ec2/asg/aws-sdk-call-trace.mustache

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,10 @@
2121
"default": {
2222
"EC2.AutoScalingGroup": "^{{platformInfo}}$",
2323
"EC2.InstanceId": "^{{instanceId}}$",
24-
"otel.resource.aws.application_signals.metric_resource_keys": "all_attributes",
24+
"otel.resource.Internal_Org": "Financial",
25+
"otel.resource.Business Unit": "Payments",
26+
"otel.resource.Region": "us-east-1",
27+
"otel.resource.aws.application_signals.metric_resource_keys": "Business Unit&Region&Organization",
2528
"otel.resource.ec2.tag.aws:autoscaling:groupName": "^{{platformInfo}}$",
2629
"otel.resource.host.id": "^{{instanceId}}$",
2730
"otel.resource.host.image.id": "^{{instanceAmi}}$",

validator/src/main/resources/expected-data-template/java/ec2/asg/client-call-log.mustache

Lines changed: 4 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -7,9 +7,8 @@
77
"Version": "^1$",
88
"Telemetry.Source": "^LocalRootSpan$",
99
"Host": "^{{privateDnsName}}$",
10-
"otel.resource.aws.application_signals.metric_resource_keys": "all_attributes",
11-
"otel.resource.host.image.id": "^{{instanceAmi}}$",
12-
"otel.resource.host.type": "^([a-z0-9]+\\.[a-z0-9]+)$"
10+
"otel.resource.Business Unit": "Payments",
11+
"otel.resource.Region": "us-east-1"
1312
},
1413
{
1514
"EC2.AutoScalingGroup": "^{{platformInfo}}$",
@@ -22,7 +21,6 @@
2221
"RemoteOperation": "GET /",
2322
"Telemetry.Source": "^ClientSpan$",
2423
"Host": "^{{privateDnsName}}$",
25-
"otel.resource.aws.application_signals.metric_resource_keys": "all_attributes",
26-
"otel.resource.host.image.id": "^{{instanceAmi}}$",
27-
"otel.resource.host.type": "^([a-z0-9]+\\.[a-z0-9]+)$"
24+
"otel.resource.Business Unit": "Payments",
25+
"otel.resource.Region": "us-east-1"
2826
}]

0 commit comments

Comments
 (0)