Skip to content

Commit 3b6549a

Browse files
dagnirdavidh44
dagnir
and
davidh44
authored
Add VirtualThread HTTP benchmarks (#6602)
* Add VirtualThread HTTP benchmarks Add virtual thread benchmarks for supported HTTP clients. Note that these "benchmarks" act more as verification that virtual threads don't get pinned with various HTTP clients rather than a direct measurement of performance gains or improvements as a result of using virtual threads over platform threads. This commit also refactors existing Apache4 and Apache5Benchmark slightly. * Checkstyle fixes * Sonar fixes * Update test/http-client-benchmarks/src/main/java/software/amazon/awssdk/benchmark/core/ObjectSize.java Co-authored-by: David Ho <70000000+davidh44@users.noreply.github.com> * Update test/http-client-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apache5/Apache5Benchmark.java Co-authored-by: David Ho <70000000+davidh44@users.noreply.github.com> * Update test/http-client-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apache5/Apache5Benchmark.java Co-authored-by: David Ho <70000000+davidh44@users.noreply.github.com> * Update test/http-client-benchmarks/src/main/java/software/amazon/awssdk/benchmark/apache5/Apache5Benchmark.java Co-authored-by: David Ho <70000000+davidh44@users.noreply.github.com> --------- Co-authored-by: David Ho <70000000+davidh44@users.noreply.github.com>
1 parent bcf6e5c commit 3b6549a

File tree

13 files changed

+858
-424
lines changed

13 files changed

+858
-424
lines changed

‎test/http-client-benchmarks/pom.xml‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -74,6 +74,24 @@
7474
<version>${awsjavasdk.version}-PREVIEW</version>
7575
</dependency>
7676

77+
<dependency>
78+
<groupId>software.amazon.awssdk</groupId>
79+
<artifactId>url-connection-client</artifactId>
80+
<version>${awsjavasdk.version}</version>
81+
</dependency>
82+
83+
<dependency>
84+
<groupId>software.amazon.awssdk</groupId>
85+
<artifactId>aws-crt-client</artifactId>
86+
<version>${awsjavasdk.version}</version>
87+
</dependency>
88+
89+
<dependency>
90+
<groupId>software.amazon.awssdk</groupId>
91+
<artifactId>netty-nio-client</artifactId>
92+
<version>${awsjavasdk.version}</version>
93+
</dependency>
94+
7795
<dependency>
7896
<groupId>org.apache.logging.log4j</groupId>
7997
<artifactId>log4j-api</artifactId>

‎test/http-client-benchmarks/src/main/java/software/amazon/awssdk/benchmark/AsyncVirtualThreadBenchmark.java‎

Lines changed: 209 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,209 @@
1+
/*
2+
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License").
5+
* You may not use this file except in compliance with the License.
6+
* A copy of the License is located at
7+
*
8+
* http://aws.amazon.com/apache2.0
9+
*
10+
* or in the "license" file accompanying this file. This file is distributed
11+
* on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
12+
* express or implied. See the License for the specific language governing
13+
* permissions and limitations under the License.
14+
*/
15+
16+
package software.amazon.awssdk.benchmark;
17+
18+
import static software.amazon.awssdk.benchmark.apache5.utility.BenchmarkUtilities.isJava21OrHigher;
19+
20+
import java.io.IOException;
21+
import java.io.PrintStream;
22+
import java.lang.reflect.InvocationTargetException;
23+
import java.lang.reflect.Method;
24+
import java.nio.file.Files;
25+
import java.nio.file.Path;
26+
import java.nio.file.Paths;
27+
import java.nio.file.StandardOpenOption;
28+
import java.util.UUID;
29+
import java.util.concurrent.ExecutionException;
30+
import java.util.concurrent.ExecutorService;
31+
import java.util.concurrent.Executors;
32+
import java.util.concurrent.TimeUnit;
33+
import org.openjdk.jmh.annotations.Benchmark;
34+
import org.openjdk.jmh.annotations.Fork;
35+
import org.openjdk.jmh.annotations.Level;
36+
import org.openjdk.jmh.annotations.Param;
37+
import org.openjdk.jmh.annotations.Scope;
38+
import org.openjdk.jmh.annotations.Setup;
39+
import org.openjdk.jmh.annotations.State;
40+
import org.openjdk.jmh.annotations.TearDown;
41+
import org.openjdk.jmh.infra.Blackhole;
42+
import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider;
43+
import software.amazon.awssdk.benchmark.core.ObjectSize;
44+
import software.amazon.awssdk.benchmark.core.S3BenchmarkHelper;
45+
import software.amazon.awssdk.core.ResponseInputStream;
46+
import software.amazon.awssdk.core.async.AsyncResponseTransformer;
47+
import software.amazon.awssdk.http.SdkHttpConfigurationOption;
48+
import software.amazon.awssdk.http.async.SdkAsyncHttpClient;
49+
import software.amazon.awssdk.http.crt.AwsCrtAsyncHttpClient;
50+
import software.amazon.awssdk.http.nio.netty.NettyNioAsyncHttpClient;
51+
import software.amazon.awssdk.regions.Region;
52+
import software.amazon.awssdk.services.s3.S3AsyncClient;
53+
import software.amazon.awssdk.services.s3.model.GetObjectResponse;
54+
import software.amazon.awssdk.services.s3.model.PutObjectResponse;
55+
import software.amazon.awssdk.utils.AttributeMap;
56+
import software.amazon.awssdk.utils.IoUtils;
57+
import software.amazon.awssdk.utils.JavaSystemSetting;
58+
59+
/**
60+
* Async http client benchmark using virtual threads. This class requires Java 21+.
61+
*/
62+
@Fork(jvmArgsAppend = "-Djdk.tracePinnedThreads=full")
63+
@State(Scope.Benchmark)
64+
public class AsyncVirtualThreadBenchmark {
65+
// We redirect standard out to a file for the -Djdk.tracePinnedThreads=full option. When virtual threads become pinned,
66+
// the JDK will print out the stacktrace through standard out. However, because JMH runs benchmarks in a forked JVM
67+
// (unless you specify -f 0, which is not recommended by JMH), that output is lost. Redirect standard out to a file so
68+
// that any time a thread is pinned, the stack trace is written to the file instead,which can be inspected after the
69+
// benchmark run.
70+
static {
71+
try {
72+
Path tmp = Paths.get(AsyncVirtualThreadBenchmark.class.getSimpleName() + "-stdout-" + UUID.randomUUID() + ".log");
73+
PrintStream fileOut = new PrintStream(
74+
Files.newOutputStream(tmp, StandardOpenOption.APPEND, StandardOpenOption.CREATE));
75+
System.setOut(fileOut);
76+
} catch (IOException e) {
77+
throw new RuntimeException("Unable to create STDOUT file", e);
78+
}
79+
}
80+
81+
public enum Client {
82+
Netty,
83+
Crt
84+
}
85+
86+
@Param("50")
87+
private int maxConnections;
88+
89+
@Param("SMALL")
90+
private ObjectSize objectSize;
91+
92+
@Param({"Netty", "Crt"})
93+
private Client client;
94+
95+
private S3AsyncClient s3AsyncClient;
96+
private S3BenchmarkHelper benchmark;
97+
private ExecutorService virtualThreadExecutor;
98+
private String putKeyPrefix;
99+
100+
@Setup(Level.Trial)
101+
public void setup() {
102+
if (!isJava21OrHigher()) {
103+
throw new UnsupportedOperationException(
104+
"Virtual threads require Java 21 or higher. Current version: " + JavaSystemSetting.JAVA_VERSION);
105+
}
106+
107+
SdkAsyncHttpClient.Builder<?> httpClientBuilder = httpClientBuilder();
108+
109+
s3AsyncClient = S3AsyncClient.builder()
110+
.region(Region.US_WEST_2)
111+
.credentialsProvider(DefaultCredentialsProvider.create())
112+
.httpClient(configure(httpClientBuilder))
113+
.build();
114+
115+
String benchmarkName = AsyncVirtualThreadBenchmark.class.getSimpleName();
116+
117+
benchmark = new S3BenchmarkHelper(benchmarkName, s3AsyncClient);
118+
benchmark.setup();
119+
120+
virtualThreadExecutor = createVirtualThreadExecutor();
121+
122+
putKeyPrefix = benchmarkName + "-";
123+
}
124+
125+
private SdkAsyncHttpClient configure(SdkAsyncHttpClient.Builder<?> builder) {
126+
AttributeMap config = AttributeMap.builder()
127+
.put(SdkHttpConfigurationOption.MAX_CONNECTIONS, maxConnections)
128+
.build();
129+
130+
return builder.buildWithDefaults(config);
131+
}
132+
133+
private ExecutorService createVirtualThreadExecutor() {
134+
try {
135+
// Use reflection to call Executors.newVirtualThreadPerTaskExecutor()
136+
Method method = Executors.class.getMethod("newVirtualThreadPerTaskExecutor");
137+
return (ExecutorService) method.invoke(null);
138+
} catch (NoSuchMethodException e) {
139+
throw new UnsupportedOperationException(
140+
"Virtual threads are not available in this Java version. " +
141+
"This benchmark requires Java 21 or higher.", e);
142+
} catch (IllegalAccessException | InvocationTargetException e) {
143+
throw new RuntimeException("Failed to create virtual thread executor", e);
144+
}
145+
}
146+
147+
@Benchmark
148+
public void getObject(Blackhole blackhole) {
149+
safeExecute(() -> {
150+
ResponseInputStream<GetObjectResponse> object = s3AsyncClient.getObject(
151+
r -> r.bucket(benchmark.bucketName()).key(benchmark.objKey(objectSize)),
152+
AsyncResponseTransformer.toBlockingInputStream()).join();
153+
blackhole.consume(object.response());
154+
IoUtils.drainInputStream(object);
155+
});
156+
}
157+
158+
@Benchmark
159+
public void putObject(Blackhole blackhole) {
160+
String jmhThreadName = Thread.currentThread().getName();
161+
safeExecute(() -> {
162+
PutObjectResponse response = s3AsyncClient.putObject(
163+
r -> r.bucket(benchmark.bucketName()).key(putKeyPrefix + jmhThreadName),
164+
benchmark.asyncRequestBody(objectSize)).join();
165+
blackhole.consume(response);
166+
});
167+
}
168+
169+
@TearDown(Level.Trial)
170+
public void tearDown() {
171+
if (virtualThreadExecutor != null) {
172+
virtualThreadExecutor.shutdown();
173+
try {
174+
if (!virtualThreadExecutor.awaitTermination(30, TimeUnit.SECONDS)) {
175+
virtualThreadExecutor.shutdownNow();
176+
}
177+
} catch (InterruptedException e) {
178+
virtualThreadExecutor.shutdownNow();
179+
}
180+
}
181+
182+
if (benchmark != null) {
183+
benchmark.cleanup();
184+
}
185+
186+
if (s3AsyncClient != null) {
187+
s3AsyncClient.close();
188+
}
189+
}
190+
191+
private void safeExecute(Runnable runnable) {
192+
try {
193+
virtualThreadExecutor.submit(runnable).get();
194+
} catch (InterruptedException | ExecutionException e) {
195+
throw new RuntimeException("Error during execution", e);
196+
}
197+
}
198+
199+
private SdkAsyncHttpClient.Builder<?> httpClientBuilder() {
200+
switch (client) {
201+
case Netty:
202+
return NettyNioAsyncHttpClient.builder();
203+
case Crt:
204+
return AwsCrtAsyncHttpClient.builder();
205+
default:
206+
throw new IllegalArgumentException("Unknown HTTP client: " + client);
207+
}
208+
}
209+
}

‎test/http-client-benchmarks/src/main/java/software/amazon/awssdk/benchmark/UnifiedBenchmarkRunner.java‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,6 @@
3131
import org.openjdk.jmh.runner.options.OptionsBuilder;
3232
import software.amazon.awssdk.benchmark.apache4.Apache4Benchmark;
3333
import software.amazon.awssdk.benchmark.apache5.Apache5Benchmark;
34-
import software.amazon.awssdk.benchmark.apache5.Apache5VirtualBenchmark;
3534
import software.amazon.awssdk.benchmark.core.BenchmarkResult;
3635
import software.amazon.awssdk.benchmark.metrics.CloudWatchMetricsPublisher;
3736
import software.amazon.awssdk.regions.Region;
@@ -76,7 +75,7 @@ public static void main(String[] args) throws Exception {
7675
// Only run virtual threads benchmark if Java 21+
7776
if (isJava21OrHigher()) {
7877
logger.info(() -> "Running Apache5 with virtual threads...");
79-
allResults.addAll(runBenchmark("Apache5-Virtual", Apache5VirtualBenchmark.class));
78+
allResults.addAll(runBenchmark("Apache5-Virtual", VirtualThreadBenchmark.class));
8079
} else {
8180
logger.info(() -> "Skipping virtual threads benchmark - requires Java 21 or higher (current: " +
8281
JavaSystemSetting.JAVA_VERSION.getStringValueOrThrow() + ")");

0 commit comments

Comments
 (0)