diff --git a/oap-application/oap-application-test/src/test/java/oap/application/remote/RemoteTest.java b/oap-application/oap-application-test/src/test/java/oap/application/remote/RemoteTest.java index 1682510bd8..c41daf3864 100644 --- a/oap-application/oap-application-test/src/test/java/oap/application/remote/RemoteTest.java +++ b/oap-application/oap-application-test/src/test/java/oap/application/remote/RemoteTest.java @@ -30,15 +30,14 @@ import oap.application.module.Module; import oap.testng.Fixtures; import oap.testng.Ports; -import oap.util.Dates; +import org.assertj.core.api.Assertions; import org.testng.annotations.Test; +import java.net.ConnectException; import java.net.URL; -import java.time.Duration; import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.concurrent.CompletableFuture; import static oap.testng.Asserts.urlOfTestResource; import static org.assertj.core.api.Assertions.assertThat; @@ -49,6 +48,7 @@ public class RemoteTest extends Fixtures { @Test public void invoke() { + Assertions.setMaxStackTraceElementsDisplayed( 1024 ); int port = Ports.getFreePort( getClass() ); List modules = Module.CONFIGURATION.urlsFromClassPath(); @@ -78,33 +78,10 @@ public void invoke() { assertThat( kernel.service( "*.remote-client-unreachable" ) ) .isPresent() .get() - .satisfies( remote -> assertThatThrownBy( remote::accessible ).isInstanceOf( IllegalArgumentException.class ) ); - } - } - - @Test - public void testAsync() { - int port = Ports.getFreePort( getClass() ); - - List modules = Module.CONFIGURATION.urlsFromClassPath(); - modules.add( urlOfTestResource( getClass(), "module.oap" ) ); - try( Kernel kernel = new Kernel( modules ) ) { - kernel.start( ApplicationConfiguration.load( urlOfTestResource( RemoteTest.class, "application-remote.conf" ), - List.of(), - Map.of( "HTTP_PORT", port ) ) ); - - Optional service = kernel.service( "*.remote-client" ); - assertThat( service ).isPresent(); - assertThat( service ) - .get() - .satisfies( remote -> { - CompletableFuture actual = remote.accessibleAsync(); - long timeStart = System.currentTimeMillis(); - assertThat( actual ).succeedsWithin( Duration.ofSeconds( 10 ) ).isEqualTo( true ); - long timeEnd = System.currentTimeMillis(); - - assertThat( timeEnd - timeStart ).isGreaterThanOrEqualTo( Dates.s( 2 ) ); - } ); + .satisfies( remote -> assertThatThrownBy( remote::accessible ) + .isInstanceOf( RuntimeException.class ) + .hasCauseInstanceOf( ConnectException.class ) + ); } } diff --git a/oap-application/oap-application/src/main/java/oap/application/remote/RemoteInvocationHandler.java b/oap-application/oap-application/src/main/java/oap/application/remote/RemoteInvocationHandler.java index 4b357d3f9d..9abd163a25 100644 --- a/oap-application/oap-application/src/main/java/oap/application/remote/RemoteInvocationHandler.java +++ b/oap-application/oap-application/src/main/java/oap/application/remote/RemoteInvocationHandler.java @@ -24,7 +24,6 @@ package oap.application.remote; import com.google.common.base.Preconditions; -import com.google.common.util.concurrent.SimpleTimeLimiter; import io.micrometer.core.instrument.Counter; import io.micrometer.core.instrument.Metrics; import io.micrometer.core.instrument.Tags; @@ -32,23 +31,20 @@ import lombok.extern.slf4j.Slf4j; import oap.application.ServiceKernelCommand; import oap.application.module.Reference; +import oap.http.client.OapHttpClient; import oap.util.Result; import oap.util.Stream; import oap.util.function.Try; -import okhttp3.Call; -import okhttp3.Callback; -import okhttp3.Dispatcher; -import okhttp3.OkHttpClient; -import okhttp3.Request; -import okhttp3.RequestBody; -import okhttp3.Response; -import org.jetbrains.annotations.NotNull; +import org.eclipse.jetty.client.BytesRequestContent; +import org.eclipse.jetty.client.InputStreamResponseListener; +import org.eclipse.jetty.client.Request; +import org.eclipse.jetty.client.Response; +import org.eclipse.jetty.http.HttpMethod; import java.io.BufferedInputStream; import java.io.ByteArrayOutputStream; import java.io.DataInputStream; import java.io.DataOutputStream; -import java.io.IOException; import java.io.InputStream; import java.lang.invoke.MethodHandles; import java.lang.reflect.InvocationHandler; @@ -56,34 +52,18 @@ import java.lang.reflect.Parameter; import java.lang.reflect.Proxy; import java.net.URI; -import java.net.http.HttpTimeoutException; -import java.time.Duration; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Iterator; import java.util.List; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.CompletionStage; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.TimeUnit; import java.util.concurrent.TimeoutException; import static java.net.HttpURLConnection.HTTP_OK; @Slf4j public final class RemoteInvocationHandler implements InvocationHandler { - public static final ExecutorService NEW_SINGLE_THREAD_EXECUTOR = Executors.newSingleThreadExecutor(); - private static final OkHttpClient globalClient; - private static final SimpleTimeLimiter SIMPLE_TIME_LIMITER = SimpleTimeLimiter.create( NEW_SINGLE_THREAD_EXECUTOR ); - - static { - OkHttpClient.Builder builder = new OkHttpClient.Builder(); - Dispatcher dispatcher = new Dispatcher(); - dispatcher.setMaxRequests( 1024 ); - dispatcher.setMaxRequestsPerHost( 1024 ); - builder.dispatcher( dispatcher ); - globalClient = builder.build(); - } - private final Counter timeoutMetrics; private final Counter errorMetrics; private final Counter successMetrics; @@ -116,17 +96,7 @@ public static Object proxy( String source, RemoteLocation remote, Class clazz } private static Object proxy( String source, URI uri, String service, Class clazz, long timeout ) { - return Proxy.newProxyInstance( clazz.getClassLoader(), new Class[] { clazz }, - new RemoteInvocationHandler( source, uri, service, timeout ) ); - } - - @NotNull - private static CompletionStage> retException( Throwable e, boolean async ) { - if( async ) { - return CompletableFuture.failedStage( e ); - } else { - return CompletableFuture.completedStage( Result.failure( e ) ); - } + return Proxy.newProxyInstance( clazz.getClassLoader(), new Class[] { clazz }, new RemoteInvocationHandler( source, uri, service, timeout ) ); } @Override @@ -149,7 +119,11 @@ public Object invoke( Object proxy, Method method, Object[] args ) throws Throwa if( result.isSuccess() ) { return result.successValue; } else { - throw result.failureValue; + if( result.failureValue instanceof RuntimeException ) { + throw result.failureValue; + } else { + throw new RuntimeException( result.failureValue ); + } } } @@ -164,128 +138,88 @@ private Result invoke( Method method, Object[] args ) { byte[] invocationB = getInvocation( method, arguments ); - boolean async = CompletableFuture.class.isAssignableFrom( method.getReturnType() ); - try { - OkHttpClient client = globalClient.newBuilder().callTimeout( Duration.ofMillis( timeout ) ).build(); - Request request = new Request.Builder() - .url( uri.toURL() ) - .post( RequestBody.create( invocationB ) ) - .build(); - Call call = client.newCall( request ); - - CompletableFuture responseAsync = new CompletableFuture<>(); - - if( async ) { - call.enqueue( new Callback() { - @Override - public void onFailure( @NotNull Call call, @NotNull IOException e ) { - responseAsync.completeExceptionally( e ); - } + InputStreamResponseListener inputStreamResponseListener = new InputStreamResponseListener(); - @Override - public void onResponse( @NotNull Call call, @NotNull Response response ) { - responseAsync.complete( response ); - } - } ); - } else { - try { - responseAsync.complete( call.execute() ); - } catch( IOException e ) { - responseAsync.completeExceptionally( e ); - } - } + Request request = OapHttpClient.DEFAULT_HTTP_CLIENT + .newRequest( uri ) + .method( HttpMethod.POST ) + .body( new BytesRequestContent( invocationB ) ) + .timeout( timeout, TimeUnit.MILLISECONDS ); - CompletableFuture> ret = responseAsync.thenCompose( response -> { - try { - if( response.code() == HTTP_OK ) { - InputStream inputStream = response.body().byteStream(); - BufferedInputStream bis = new BufferedInputStream( inputStream ); - DataInputStream dis = new DataInputStream( bis ); - boolean success = dis.readBoolean(); - try { - if( !success ) { - try { - Throwable throwable = FstConsts.readObjectWithSize( dis ); + request.send( inputStreamResponseListener ); - if( throwable instanceof RemoteInvocationException riex ) { - errorMetrics.increment(); - return retException( riex, async ); - } + try { + Response response = inputStreamResponseListener.get( timeout, TimeUnit.MILLISECONDS ); + if( response.getStatus() == HTTP_OK ) { + InputStream inputStream = inputStreamResponseListener.getInputStream(); + BufferedInputStream bis = new BufferedInputStream( inputStream ); + DataInputStream dis = new DataInputStream( bis ); + boolean success = dis.readBoolean(); + + try { + if( !success ) { + try { + Throwable throwable = FstConsts.readObjectWithSize( dis ); + + if( throwable instanceof RemoteInvocationException riex ) { errorMetrics.increment(); - return async ? CompletableFuture.>failedStage( throwable ) : CompletableFuture.completedStage( Result.failure( throwable ) ); - } finally { - dis.close(); + return Result.failure( riex ); } + + errorMetrics.increment(); + return Result.failure( throwable ); + } finally { + dis.close(); + } + } else { + boolean stream = dis.readBoolean(); + if( stream ) { + ChainIterator it = new ChainIterator( dis ); + + return Result.success( Stream.of( it ).onClose( Try.run( () -> { + dis.close(); + successMetrics.increment(); + } ) ) ); } else { - boolean stream = dis.readBoolean(); - if( stream ) { - ChainIterator it = new ChainIterator( dis ); - - return CompletableFuture.completedStage( Result.success( Stream.of( it ).onClose( Try.run( () -> { - dis.close(); - successMetrics.increment(); - } ) ) ) ); - } else { - try { - Result r = Result.success( FstConsts.readObjectWithSize( dis ) ); - successMetrics.increment(); - return CompletableFuture.completedStage( r ); - } finally { - dis.close(); - } + try { + Result r = Result.success( FstConsts.readObjectWithSize( dis ) ); + successMetrics.increment(); + return r; + } finally { + dis.close(); } } - } catch( Exception e ) { - dis.close(); - return retException( e, async ); } - } else { - RemoteInvocationException ex = new RemoteInvocationException( "invocation failed " + this + "#" + service + "@" + method.getName() - + " code " + response.code() - + " body '" + response.body().string() + "'" - + " message '" + response.message() + "'" ); - - return retException( ex, async ); + } catch( Exception e ) { + dis.close(); + return Result.failure( e ); } - } catch( Throwable e ) { - return retException( e, async ); - } - } ); + } else { + RemoteInvocationException ex = new RemoteInvocationException( "invocation failed " + this + "#" + service + "@" + method.getName() + + " code " + response.getStatus() + + " body '" + new String( inputStreamResponseListener.getInputStream().readAllBytes(), StandardCharsets.UTF_8 ) + "'" + + " message '" + response.getReason() + "'" ); - ret.whenComplete( ( _, ex ) -> { - if( ex != null ) { - checkException( ex ); + return Result.failure( ex ); } - } ); - - if( async ) { - return Result.success( ret.thenApply( r -> r.successValue ) ); - } else { - return ret.get(); - } - } catch( Exception e ) { - if( async ) { - return Result.success( CompletableFuture.failedFuture( e ) ); + } catch( InterruptedException | TimeoutException e ) { + timeoutMetrics.increment(); + + return Result.failure( e ); + } catch( ExecutionException e ) { + return Result.failure( e.getCause() ); + } catch( Throwable e ) { + return Result.failure( e ); } + } catch( Exception e ) { return Result.failure( e ); } } - private void checkException( Throwable ex ) { - if( ex instanceof RemoteInvocationException riex ) { - checkException( riex.getCause() ); - return; - } - - if( ex instanceof HttpTimeoutException || ex instanceof TimeoutException ) { - timeoutMetrics.increment(); - } - } - @SneakyThrows private byte[] getInvocation( Method method, List arguments ) { diff --git a/oap-http/oap-http-prometheus/pom.xml b/oap-http/oap-http-prometheus/pom.xml index ad63401a99..debf9dbfba 100644 --- a/oap-http/oap-http-prometheus/pom.xml +++ b/oap-http/oap-http-prometheus/pom.xml @@ -40,6 +40,12 @@ ${project.version} test + + oap + oap-http-test + ${project.version} + test + org.projectlombok diff --git a/oap-http/oap-http-prometheus/src/test/java/oap/http/prometheus/PrometheusExporterTest.java b/oap-http/oap-http-prometheus/src/test/java/oap/http/prometheus/PrometheusExporterTest.java index f79dad4c10..da4038b85f 100644 --- a/oap-http/oap-http-prometheus/src/test/java/oap/http/prometheus/PrometheusExporterTest.java +++ b/oap-http/oap-http-prometheus/src/test/java/oap/http/prometheus/PrometheusExporterTest.java @@ -24,20 +24,20 @@ package oap.http.prometheus; +import io.micrometer.core.instrument.Counter; import io.micrometer.core.instrument.Meter; import io.micrometer.core.instrument.Metrics; +import io.micrometer.core.instrument.Timer; import io.micrometer.core.instrument.search.Search; -import oap.http.Client; import oap.http.server.nio.NioHttpServer; import oap.testng.Fixtures; import oap.testng.Ports; import org.testng.annotations.AfterMethod; import org.testng.annotations.Test; -import java.io.IOException; import java.util.concurrent.TimeUnit; -import static org.assertj.core.api.Assertions.assertThat; +import static oap.http.test.HttpAsserts.assertGet; public class PrometheusExporterTest extends Fixtures { private static void clear() { @@ -47,28 +47,29 @@ private static void clear() { } @Test - public void server() throws IOException { - var port = Ports.getFreePort( getClass() ); + public void server() throws Exception { + int port = Ports.getFreePort( getClass() ); try( var server = new NioHttpServer( new NioHttpServer.DefaultPort( port ) ) ) { var exporter = new PrometheusExporter( server ); - var metric1 = Metrics.counter( "test1" ); - var metric2 = Metrics.timer( "test2" ); + Counter metric1 = Metrics.counter( "test1" ); + Timer metric2 = Metrics.timer( "test2" ); server.start(); metric1.increment( 2 ); metric2.record( 2, TimeUnit.SECONDS ); - var response = Client.DEFAULT.get( "http://localhost:" + port + "/metrics" ).contentString(); - assertThat( response ).contains( """ - # HELP test1_total \s - # TYPE test1_total counter - test1_total 2.0 - """ ); - assertThat( response ).contains( "test2_seconds_count 1" ); - assertThat( response ).contains( "test2_seconds_max 2.0" ); - assertThat( response ).contains( "system_metrics 5" ); + assertGet( "http://localhost:" + port + "/metrics" ) + .body() + .contains( """ + # HELP test1_total \s + # TYPE test1_total counter + test1_total 2.0 + """ ) + .contains( "test2_seconds_count 1" ) + .contains( "test2_seconds_max 2.0" ) + .contains( "system_metrics 5" ); } } diff --git a/oap-http/oap-http-test/src/main/java/oap/http/test/HttpAsserts.java b/oap-http/oap-http-test/src/main/java/oap/http/test/HttpAsserts.java index 3ba39e0ffd..8d08359dc7 100644 --- a/oap-http/oap-http-test/src/main/java/oap/http/test/HttpAsserts.java +++ b/oap-http/oap-http-test/src/main/java/oap/http/test/HttpAsserts.java @@ -23,15 +23,15 @@ */ package oap.http.test; -import com.google.common.base.Preconditions; import lombok.EqualsAndHashCode; import lombok.SneakyThrows; import lombok.ToString; +import lombok.experimental.ExtensionMethod; import lombok.extern.slf4j.Slf4j; -import oap.http.Client; import oap.http.Cookie; -import oap.http.InputStreamRequestBody; -import oap.http.Uri; +import oap.http.Response; +import oap.http.client.JettyRequestExtensions; +import oap.http.client.OapHttpClient; import oap.json.JsonException; import oap.json.testng.JsonAsserts; import oap.testng.Asserts; @@ -40,39 +40,32 @@ import oap.util.Maps; import oap.util.Pair; import oap.util.Stream; -import okhttp3.Dispatcher; -import okhttp3.Headers; -import okhttp3.HttpUrl; -import okhttp3.MediaType; -import okhttp3.OkHttpClient; -import okhttp3.Request; -import okhttp3.RequestBody; -import okhttp3.Response; -import okhttp3.ResponseBody; -import okhttp3.java.net.cookiejar.JavaNetCookieJar; -import org.apache.commons.lang3.StringUtils; import org.assertj.core.api.AbstractIntegerAssert; import org.assertj.core.api.Assertions; +import org.eclipse.jetty.client.BytesRequestContent; +import org.eclipse.jetty.client.ContentResponse; +import org.eclipse.jetty.client.HttpClient; +import org.eclipse.jetty.client.InputStreamRequestContent; +import org.eclipse.jetty.client.StringRequestContent; +import org.eclipse.jetty.http.HttpFields; +import org.eclipse.jetty.http.HttpHeader; +import org.eclipse.jetty.http.HttpMethod; import org.joda.time.DateTime; import org.testng.internal.collections.Ints; import javax.annotation.Nonnull; import javax.annotation.Nullable; import java.io.ByteArrayInputStream; -import java.io.IOException; import java.io.InputStream; import java.io.UncheckedIOException; -import java.lang.reflect.Field; -import java.net.CookieManager; -import java.net.CookiePolicy; import java.net.HttpCookie; import java.util.ArrayList; -import java.util.Date; import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; import java.util.function.Consumer; +import java.util.function.Function; import java.util.regex.Pattern; import static java.net.HttpURLConnection.HTTP_OK; @@ -83,37 +76,14 @@ import static oap.io.content.ContentReader.ofString; import static oap.testng.Asserts.assertString; import static oap.testng.Asserts.contentOfTestResource; +import static oap.util.Pair.__; import static org.assertj.core.api.Assertions.assertThat; +@ExtensionMethod( JettyRequestExtensions.class ) @Slf4j @SuppressWarnings( "unused" ) public class HttpAsserts { - public static final OkHttpClient OK_HTTP_CLIENT; - - private static final JavaNetCookieJar cookieJar; - private static final CookieManager cookieManager; - - private static Field whenCreatedField; - - static { - cookieManager = new CookieManager(); - cookieManager.setCookiePolicy( CookiePolicy.ACCEPT_ALL ); - - cookieJar = new JavaNetCookieJar( cookieManager ); - OK_HTTP_CLIENT = new OkHttpClient.Builder() - .cookieJar( cookieJar ) - .dispatcher( new Dispatcher( Executors.newVirtualThreadPerTaskExecutor() ) ) - .followRedirects( false ) - .followSslRedirects( false ) - .build(); - - try { - whenCreatedField = HttpCookie.class.getDeclaredField( "whenCreated" ); - whenCreatedField.setAccessible( true ); - } catch( NoSuchFieldException e ) { - throw new RuntimeException( e ); - } - } + public static final HttpClient TEST_HTTP_CLIENT = OapHttpClient.customHttpClient().build(); public static String httpPrefix( int port ) { return "http://localhost:" + port; @@ -123,99 +93,61 @@ public static String httpUrl( int port, String suffix ) { return httpPrefix( port ) + ( suffix.startsWith( "/" ) ? suffix : "/" + suffix ); } - public static void reset() { - OK_HTTP_CLIENT.connectionPool().evictAll(); - cookieManager.getCookieStore().removeAll(); - } - @SafeVarargs public static HttpAssertion assertGet( String uri, Pair... params ) throws UncheckedIOException { return assertGet( uri, Maps.of( params ), Map.of() ); } public static HttpAssertion assertGet( String uri, Map params, Map headers ) throws UncheckedIOException { - try { - Request.Builder builder = new Request.Builder(); - - setHeaders( uri, headers, builder ); - - Request request = builder - .url( Uri.uri( uri, params ).toURL() ) - .get() - .build(); - - return getResponseAsHttpAssertion( request ); - } catch( IOException e ) { - throw new UncheckedIOException( e ); - } + return assertGet( TEST_HTTP_CLIENT, uri, params, headers ); } - private static void setHeaders( String uri, Map headers, Request.Builder builder ) { - headers.forEach( ( k, v ) -> { - if( "Cookie".equalsIgnoreCase( k ) ) { - HttpUrl httpUrl = HttpUrl.parse( uri ); - Preconditions.checkNotNull( httpUrl ); - Preconditions.checkNotNull( v ); - - ArrayList cookies = new ArrayList<>(); - - List list = cookieJar.loadForRequest( httpUrl ); - cookies.addAll( list ); - - String[] setCookies = StringUtils.split( v.toString(), ";" ); - for( String setCookie : setCookies ) { - okhttp3.Cookie cookie = okhttp3.Cookie.parse( httpUrl, setCookie ); - Preconditions.checkNotNull( cookie ); - cookies.add( cookie ); - } - - cookieJar.saveFromResponse( httpUrl, cookies ); - } else { - builder.header( k, v == null ? "" : v.toString() ); - } - } ); + public static HttpAssertion assertGet( org.eclipse.jetty.client.HttpClient client, String uri, Map params, Map headers ) throws UncheckedIOException { + return getResponseAsHttpAssertion( client + .newRequest( uri ) + .method( HttpMethod.GET ) + .addParams( params ) + .addHeaders( headers ) ); } public static HttpAssertion assertPost( String uri, InputStream content, @Nullable String contentType, Map headers ) { - try { - Request.Builder builder = new Request.Builder(); - - setHeaders( uri, headers, builder ); - - RequestBody requestBody = new InputStreamRequestBody( contentType != null ? MediaType.get( contentType ) : null, content ); - - Request request = builder - .url( uri ) - .post( requestBody ) - .build(); + return assertPost( TEST_HTTP_CLIENT, uri, content, contentType, headers ); + } - return getResponseAsHttpAssertion( request ); - } catch( IOException e ) { - throw new UncheckedIOException( e ); - } + public static HttpAssertion assertPost( org.eclipse.jetty.client.HttpClient httpClient, String uri, InputStream content, @Nullable String contentType, Map headers ) { + return getResponseAsHttpAssertion( httpClient + .newRequest( uri ) + .method( HttpMethod.POST ) + .addHeaders( headers ) + .body( new InputStreamRequestContent( contentType, content, null ) ) ); } public static HttpAssertion assertPost( String uri, InputStream content, @Nullable String contentType ) { return assertPost( uri, content, contentType, Maps.of() ); } - public static HttpAssertion assertPost( String uri, String content, @Nullable String contentType, Map headers ) { - try { - Request.Builder builder = new Request.Builder(); - - setHeaders( uri, headers, builder ); + public static HttpAssertion assertPost( String uri, byte[] content, @Nullable String contentType, Map headers ) { + return assertPost( TEST_HTTP_CLIENT, uri, content, contentType, headers ); + } - RequestBody requestBody = RequestBody.create( content, contentType != null ? MediaType.parse( contentType ) : null ); + public static HttpAssertion assertPost( org.eclipse.jetty.client.HttpClient httpClient, String uri, byte[] content, @Nullable String contentType, Map headers ) { + return getResponseAsHttpAssertion( httpClient + .newRequest( uri ) + .method( HttpMethod.POST ) + .addHeaders( headers ) + .body( new BytesRequestContent( contentType, content ) ) ); + } - Request request = builder - .url( uri ) - .post( requestBody ) - .build(); + public static HttpAssertion assertPost( String uri, String content, @Nullable String contentType, Map headers ) { + return assertPost( TEST_HTTP_CLIENT, uri, content, contentType, headers ); + } - return getResponseAsHttpAssertion( request ); - } catch( IOException e ) { - throw new UncheckedIOException( e ); - } + public static HttpAssertion assertPost( org.eclipse.jetty.client.HttpClient httpClient, String uri, String content, @Nullable String contentType, Map headers ) { + return getResponseAsHttpAssertion( httpClient + .newRequest( uri ) + .method( HttpMethod.POST ) + .addHeaders( headers ) + .body( new StringRequestContent( contentType, content ) ) ); } public static HttpAssertion assertPost( String uri, String content ) { @@ -230,19 +162,24 @@ public static HttpAssertion assertPost( String uri, String content, String conte return assertPost( uri, content, contentType, Maps.of() ); } - private static @Nonnull HttpAssertion getResponseAsHttpAssertion( Request request ) throws IOException { - try( Response response = OK_HTTP_CLIENT.newCall( request ).execute(); - ResponseBody body = response.body() ) { + @SneakyThrows + private static @Nonnull HttpAssertion getResponseAsHttpAssertion( org.eclipse.jetty.client.Request request ) { + ContentResponse contentResponse = request + .timeout( 10, TimeUnit.SECONDS ) + .send(); + + String mediaType = contentResponse.getMediaType(); + + ArrayList> headers = new ArrayList<>(); + HttpFields responseHeaders = contentResponse.getHeaders(); + responseHeaders.forEach( field -> { + HttpHeader header = field.getHeader(); + headers.add( __( field.getName(), field.getValue() ) ); + } ); - Headers responseHeaders = response.headers(); - ArrayList> headers = new ArrayList<>(); - responseHeaders.toMultimap().forEach( ( k, vs ) -> vs.forEach( v -> headers.add( Pair.__( k, v ) ) ) ); - byte[] bytes = body.bytes(); - MediaType mediaType = body.contentType(); - return new HttpAssertion( new Client.Response( - response.request().url().toString(), - response.code(), response.message(), headers, mediaType != null ? mediaType.toString() : APPLICATION_OCTET_STREAM, new ByteArrayInputStream( bytes ) ) ); - } + return new HttpAssertion( new Response( + request.getURI().toString(), + contentResponse.getStatus(), contentResponse.getReason(), headers, mediaType != null ? mediaType : APPLICATION_OCTET_STREAM, new ByteArrayInputStream( contentResponse.getContent() ) ) ); } public static HttpAssertion assertPut( String uri, String content, String contentType ) { @@ -250,22 +187,16 @@ public static HttpAssertion assertPut( String uri, String content, String conten } public static HttpAssertion assertPut( String uri, String content, String contentType, Map headers ) { - try { - Request.Builder builder = new Request.Builder(); - - setHeaders( uri, headers, builder ); - - RequestBody requestBody = RequestBody.create( content, contentType != null ? MediaType.parse( contentType ) : null ); - - Request request = builder - .url( uri ) - .put( requestBody ) - .build(); + return assertPut( TEST_HTTP_CLIENT, uri, content, contentType, headers ); + } - return getResponseAsHttpAssertion( request ); - } catch( IOException e ) { - throw new UncheckedIOException( e ); - } + public static HttpAssertion assertPut( org.eclipse.jetty.client.HttpClient httpClient, String uri, String content, String contentType, Map headers ) { + return getResponseAsHttpAssertion( httpClient + .newRequest( uri ) + .method( HttpMethod.PUT ) + .addHeaders( headers ) + .body( new StringRequestContent( contentType, content ) ) + ); } public static HttpAssertion assertPut( String uri, byte[] content, String contentType ) { @@ -273,22 +204,16 @@ public static HttpAssertion assertPut( String uri, byte[] content, String conten } public static HttpAssertion assertPut( String uri, byte[] content, String contentType, Map headers ) { - try { - Request.Builder builder = new Request.Builder(); - - setHeaders( uri, headers, builder ); - - RequestBody requestBody = RequestBody.create( content, contentType != null ? MediaType.parse( contentType ) : null ); - - Request request = builder - .url( uri ) - .put( requestBody ) - .build(); + return assertPut( TEST_HTTP_CLIENT, uri, content, contentType, headers ); + } - return getResponseAsHttpAssertion( request ); - } catch( IOException e ) { - throw new UncheckedIOException( e ); - } + public static HttpAssertion assertPut( org.eclipse.jetty.client.HttpClient httpClient, String uri, byte[] content, String contentType, Map headers ) { + return getResponseAsHttpAssertion( httpClient + .newRequest( uri ) + .method( HttpMethod.PUT ) + .addHeaders( headers ) + .body( new BytesRequestContent( contentType, content ) ) + ); } public static HttpAssertion assertPut( String uri, InputStream is, String contentType ) { @@ -296,22 +221,16 @@ public static HttpAssertion assertPut( String uri, InputStream is, String conten } public static HttpAssertion assertPut( String uri, InputStream is, String contentType, Map headers ) { - try { - Request.Builder builder = new Request.Builder(); - - setHeaders( uri, headers, builder ); - - InputStreamRequestBody requestBody = new InputStreamRequestBody( contentType != null ? MediaType.parse( contentType ) : null, is ); - - Request request = builder - .url( uri ) - .put( requestBody ) - .build(); + return assertPut( TEST_HTTP_CLIENT, uri, is, contentType, headers ); + } - return getResponseAsHttpAssertion( request ); - } catch( IOException e ) { - throw new UncheckedIOException( e ); - } + public static HttpAssertion assertPut( org.eclipse.jetty.client.HttpClient httpClient, String uri, InputStream is, String contentType, Map headers ) { + return getResponseAsHttpAssertion( httpClient + .newRequest( uri ) + .method( HttpMethod.PUT ) + .addHeaders( headers ) + .body( new InputStreamRequestContent( contentType, is, null ) ) + ); } public static HttpAssertion assertPatch( String uri, byte[] content, String contentType ) { @@ -319,22 +238,16 @@ public static HttpAssertion assertPatch( String uri, byte[] content, String cont } public static HttpAssertion assertPatch( String uri, byte[] content, String contentType, Map headers ) { - try { - Request.Builder builder = new Request.Builder(); - - setHeaders( uri, headers, builder ); - - RequestBody requestBody = RequestBody.create( content, contentType != null ? MediaType.parse( contentType ) : null ); - - Request request = builder - .url( uri ) - .patch( requestBody ) - .build(); + return assertPatch( TEST_HTTP_CLIENT, uri, content, contentType, headers ); + } - return getResponseAsHttpAssertion( request ); - } catch( IOException e ) { - throw new UncheckedIOException( e ); - } + public static HttpAssertion assertPatch( org.eclipse.jetty.client.HttpClient httpClient, String uri, byte[] content, String contentType, Map headers ) { + return getResponseAsHttpAssertion( httpClient + .newRequest( uri ) + .method( HttpMethod.PATCH ) + .addHeaders( headers ) + .body( new BytesRequestContent( contentType, content ) ) + ); } public static HttpAssertion assertPatch( String uri, String content, String contentType ) { @@ -342,22 +255,16 @@ public static HttpAssertion assertPatch( String uri, String content, String cont } public static HttpAssertion assertPatch( String uri, String content, String contentType, Map headers ) { - try { - Request.Builder builder = new Request.Builder(); - - setHeaders( uri, headers, builder ); - - RequestBody requestBody = RequestBody.create( content, contentType != null ? MediaType.parse( contentType ) : null ); - - Request request = builder - .url( uri ) - .patch( requestBody ) - .build(); + return assertPatch( TEST_HTTP_CLIENT, uri, content, contentType, headers ); + } - return getResponseAsHttpAssertion( request ); - } catch( IOException e ) { - throw new UncheckedIOException( e ); - } + public static HttpAssertion assertPatch( org.eclipse.jetty.client.HttpClient httpClient, String uri, String content, String contentType, Map headers ) { + return getResponseAsHttpAssertion( httpClient + .newRequest( uri ) + .method( HttpMethod.PATCH ) + .addHeaders( headers ) + .body( new StringRequestContent( contentType, content ) ) + ); } public static HttpAssertion assertPatch( String uri, InputStream is, String contentType ) { @@ -366,60 +273,44 @@ public static HttpAssertion assertPatch( String uri, InputStream is, String cont public static HttpAssertion assertPatch( String uri, InputStream is, String contentType, Map headers ) { - try { - Request.Builder builder = new Request.Builder(); - - setHeaders( uri, headers, builder ); - - InputStreamRequestBody requestBody = new InputStreamRequestBody( contentType != null ? MediaType.parse( contentType ) : null, is ); - - Request request = builder - .url( uri ) - .patch( requestBody ) - .build(); + return assertPatch( TEST_HTTP_CLIENT, uri, is, contentType, headers ); + } - return getResponseAsHttpAssertion( request ); - } catch( IOException e ) { - throw new UncheckedIOException( e ); - } + public static HttpAssertion assertPatch( org.eclipse.jetty.client.HttpClient httpClient, String uri, InputStream is, String contentType, Map headers ) { + return getResponseAsHttpAssertion( httpClient + .newRequest( uri ) + .method( HttpMethod.PATCH ) + .addHeaders( headers ) + .body( new InputStreamRequestContent( contentType, is, null ) ) + ); } public static HttpAssertion assertDelete( String uri, Map headers ) { - try { - Request.Builder builder = new Request.Builder(); - - setHeaders( uri, headers, builder ); - - Request request = builder - .url( uri ) - .delete() - .build(); + return assertDelete( TEST_HTTP_CLIENT, uri, headers ); + } - return getResponseAsHttpAssertion( request ); - } catch( IOException e ) { - throw new UncheckedIOException( e ); - } + public static HttpAssertion assertDelete( org.eclipse.jetty.client.HttpClient httpClient, String uri, Map headers ) { + return getResponseAsHttpAssertion( httpClient + .newRequest( uri ) + .method( HttpMethod.DELETE ) + .addHeaders( headers ) + ); } public static HttpAssertion assertDelete( String uri ) { return assertDelete( uri, Map.of() ); } - @SneakyThrows - private static long whenCreatedFieldGet( HttpCookie cookie ) { - return ( long ) whenCreatedField.get( cookie ); - } - @EqualsAndHashCode @ToString public static final class HttpAssertion { - private final Client.Response response; + private final Response response; - private HttpAssertion( Client.Response response ) { + private HttpAssertion( Response response ) { this.response = response; } - public static HttpAssertion assertHttpResponse( Client.Response response ) { + public static HttpAssertion assertHttpResponse( Response response ) { return new HttpAssertion( response ); } @@ -501,11 +392,28 @@ public HttpAssertion bodyContainsPattern( String pattern ) { return this; } + public HttpAssertion hasHeadersSize( int size ) { + assertThat( response.headers ).hasSize( size ); + + return this; + } + public HttpAssertion containsHeader( String name, String value ) { + containsHeader( name ); assertString( response.header( name ).orElse( null ) ).isEqualTo( value ); return this; } + public HttpAssertion containsHeader( String name ) { + assertThat( response.getHeaders() ).containsKey( name ); + return this; + } + + public HttpAssertion doesNotContainHeader( String name ) { + assertThat( response.getHeaders() ).doesNotContainKey( name ); + return this; + } + public HttpAssertion containsCookie( String name, Consumer assertion ) { Optional cookie = Stream.of( getCookies() ).filter( c -> c.getName().equalsIgnoreCase( name ) ).findAny(); Assertions.assertThat( cookie ) @@ -526,11 +434,11 @@ public HttpAssertion containsCookie( String cookie ) { } public HttpAssertion cookies( Consumer cons ) { - HttpUrl httpUrl = HttpUrl.parse( response.url ); - assertThat( httpUrl ).isNotNull(); - List cookies = cookieManager.getCookieStore().get( httpUrl.uri() ); +// HttpUrl httpUrl = HttpUrl.parse( response.url ); +// assertThat( httpUrl ).isNotNull(); +// List cookies = cookieManager.getCookieStore().get( httpUrl.uri() ); - cons.accept( new CookiesHttpAssertion( cookies ) ); +// cons.accept( new CookiesHttpAssertion( cookies ) ); return this; } @@ -541,7 +449,7 @@ private List getCookies() { .toList(); } - public HttpAssertion is( Consumer condition ) { + public HttpAssertion is( Consumer condition ) { condition.accept( response ); return this; } @@ -580,7 +488,7 @@ public HttpAssertion respondedJson( Class contextClass, String resource, Map< return respondedJson( contentOfTestResource( contextClass, resource, ofString() ), substitutions ); } - public HttpAssertion satisfies( Consumer assertion ) { + public HttpAssertion satisfies( Consumer assertion ) { assertion.accept( response ); return this; } @@ -598,6 +506,10 @@ public T unmarshal( Class clazz ) { public Asserts.StringAssertion body() { return assertString( response.contentString() ); } + + public Asserts.StringAssertion body( Function conv ) { + return assertString( conv.apply( response.content() ) ); + } } public static final class CookiesHttpAssertion { @@ -609,7 +521,7 @@ public CookiesHttpAssertion( List cookies ) { .withPath( c.getPath() ) .withDomain( c.getDomain() ) .withMaxAge( c.getMaxAge() < 0 ? null : ( int ) c.getMaxAge() ) - .withExpires( c.getMaxAge() <= 0 ? null : new Date( ( whenCreatedFieldGet( c ) + c.getMaxAge() ) * 1000L ) ) +// .withExpires( c.getMaxAge() <= 0 ? null : new Date( ( whenCreatedFieldGet( c ) + c.getMaxAge() ) * 1000L ) ) .withDiscard( c.getDiscard() ) .withSecure( c.getSecure() ) .withHttpOnly( c.isHttpOnly() ) @@ -702,13 +614,13 @@ public CookieHttpAssertion isNotHttpOnly() { } public static final class JsonHttpAssertion { - private final Client.Response response; + private final Response response; - private JsonHttpAssertion( Client.Response response ) { + private JsonHttpAssertion( Response response ) { this.response = response; } - public static JsonHttpAssertion assertJsonResponse( Client.Response response ) { + public static JsonHttpAssertion assertJsonResponse( Response response ) { return new JsonHttpAssertion( response ); } diff --git a/oap-http/oap-http-test/src/main/java/oap/http/test/MockCookieStore.java b/oap-http/oap-http-test/src/main/java/oap/http/test/MockCookieStore.java deleted file mode 100644 index 3f67467818..0000000000 --- a/oap-http/oap-http-test/src/main/java/oap/http/test/MockCookieStore.java +++ /dev/null @@ -1,164 +0,0 @@ -package oap.http.test; - -import oap.http.test.cookies.MockCookieFactory; -import org.apache.http.annotation.Contract; -import org.apache.http.annotation.ThreadingBehavior; -import org.apache.http.client.CookieStore; -import org.apache.http.cookie.Cookie; -import org.apache.http.cookie.CookieIdentityComparator; -import org.joda.time.DateTime; - -import javax.annotation.Nullable; -import java.io.IOException; -import java.io.ObjectInputStream; -import java.util.ArrayList; -import java.util.Date; -import java.util.Iterator; -import java.util.List; -import java.util.TreeSet; -import java.util.concurrent.locks.ReadWriteLock; -import java.util.concurrent.locks.ReentrantReadWriteLock; - -import static org.joda.time.DateTimeZone.UTC; - -@Contract( threading = ThreadingBehavior.SAFE ) -public class MockCookieStore implements CookieStore { - - - private final TreeSet cookies; - private transient ReadWriteLock lock; - - public MockCookieStore() { - super(); - this.cookies = new TreeSet<>( new CookieIdentityComparator() ); - this.lock = new ReentrantReadWriteLock(); - } - - private void readObject( final ObjectInputStream stream ) throws IOException, ClassNotFoundException { - stream.defaultReadObject(); - - /* Reinstantiate transient fields. */ - this.lock = new ReentrantReadWriteLock(); - } - - /** - * Adds an {@link Cookie HTTP cookie}, replacing any existing equivalent cookies. - * If the given cookie has already expired it will not be added, but existing - * values will still be removed. - * - * @param cookie the {@link Cookie cookie} to be added - * @see #addCookies(Cookie[]) - */ - @Override - public void addCookie( final Cookie cookie ) { - if( cookie != null ) { - lock.writeLock().lock(); - try { - // first remove any old cookie that is equivalent - cookies.remove( cookie ); - if( !cookie.isExpired( new DateTime( UTC ).toDate() ) ) { - cookies.add( MockCookieFactory.wrap( cookie ) ); - } - } finally { - lock.writeLock().unlock(); - } - } - } - - /** - * Adds an array of {@link Cookie HTTP cookies}. Cookies are added individually and - * in the given array order. If any of the given cookies has already expired it will - * not be added, but existing values will still be removed. - * - * @param cookies the {@link Cookie cookies} to be added - * @see #addCookie(Cookie) - */ - public void addCookies( final Cookie[] cookies ) { - if( cookies != null ) { - for( final Cookie cookie : cookies ) { - this.addCookie( cookie ); - } - } - } - - /** - * Returns an immutable array of {@link Cookie cookies} that this HTTP - * state currently contains. - * - * @return an array of {@link Cookie cookies}. - */ - @Override - public List getCookies() { - lock.readLock().lock(); - try { - //create defensive copy so it won't be concurrently modified - return new ArrayList( cookies ); - } finally { - lock.readLock().unlock(); - } - } - - /** - * Removes all of {@link Cookie cookies} in this HTTP state - * that have expired by the specified {@link java.util.Date date}. - * - * @return true if any cookies were purged. - * @see Cookie#isExpired(Date) - */ - @Override - public boolean clearExpired( final Date ignored ) { - if( ignored == null ) { - return false; - } - - Date date = DateTime.now( UTC ).toDate(); - - lock.writeLock().lock(); - try { - boolean removed = false; - for( final Iterator it = cookies.iterator(); it.hasNext(); ) { - if( it.next().isExpired( date ) ) { - it.remove(); - removed = true; - } - } - return removed; - } finally { - lock.writeLock().unlock(); - } - } - - /** - * Clears all cookies. - */ - @Override - public void clear() { - lock.writeLock().lock(); - try { - cookies.clear(); - } finally { - lock.writeLock().unlock(); - } - } - - @Override - public String toString() { - lock.readLock().lock(); - try { - return cookies.toString(); - } finally { - lock.readLock().unlock(); - } - } - - @Nullable - public Cookie getCookie( String name ) { - for( Cookie cookie : getCookies() ) { - if( name.equals( cookie.getName() ) ) { - return cookie; - } - } - - return null; - } -} diff --git a/oap-http/oap-http-test/src/main/java/oap/http/test/cookies/MockClientCookie.java b/oap-http/oap-http-test/src/main/java/oap/http/test/cookies/MockClientCookie.java deleted file mode 100644 index b90e480e7b..0000000000 --- a/oap-http/oap-http-test/src/main/java/oap/http/test/cookies/MockClientCookie.java +++ /dev/null @@ -1,19 +0,0 @@ -package oap.http.test.cookies; - -import org.apache.http.cookie.ClientCookie; - -public class MockClientCookie extends MockCookie implements ClientCookie { - MockClientCookie( T cookie ) { - super( cookie ); - } - - @Override - public String getAttribute( String name ) { - return cookie.getAttribute( name ); - } - - @Override - public boolean containsAttribute( String name ) { - return cookie.containsAttribute( name ); - } -} diff --git a/oap-http/oap-http-test/src/main/java/oap/http/test/cookies/MockCookie.java b/oap-http/oap-http-test/src/main/java/oap/http/test/cookies/MockCookie.java deleted file mode 100644 index 05f6ccf45f..0000000000 --- a/oap-http/oap-http-test/src/main/java/oap/http/test/cookies/MockCookie.java +++ /dev/null @@ -1,78 +0,0 @@ -package oap.http.test.cookies; - -import org.apache.http.cookie.Cookie; -import org.joda.time.DateTime; - -import java.util.Date; - -import static org.joda.time.DateTimeZone.UTC; - -public class MockCookie implements Cookie { - protected final T cookie; - - MockCookie( T cookie ) { - this.cookie = cookie; - } - - @Override - public String getName() { - return cookie.getName(); - } - - @Override - public String getValue() { - return cookie.getValue(); - } - - @Override - public String getComment() { - return cookie.getComment(); - } - - @Override - public String getCommentURL() { - return cookie.getCommentURL(); - } - - @Override - public Date getExpiryDate() { - return cookie.getExpiryDate(); - } - - @Override - public boolean isPersistent() { - return cookie.isPersistent(); - } - - @Override - public String getDomain() { - return cookie.getDomain(); - } - - @Override - public String getPath() { - return cookie.getPath(); - } - - @Override - public int[] getPorts() { - return cookie.getPorts(); - } - - @Override - public boolean isSecure() { - return cookie.isSecure(); - } - - @Override - public int getVersion() { - return cookie.getVersion(); - } - - @Override - public boolean isExpired( Date ignored ) { - Date date = DateTime.now( UTC ).toDate(); - - return cookie.isExpired( date ); - } -} diff --git a/oap-http/oap-http-test/src/main/java/oap/http/test/cookies/MockCookieFactory.java b/oap-http/oap-http-test/src/main/java/oap/http/test/cookies/MockCookieFactory.java deleted file mode 100644 index 11f6fcbf82..0000000000 --- a/oap-http/oap-http-test/src/main/java/oap/http/test/cookies/MockCookieFactory.java +++ /dev/null @@ -1,10 +0,0 @@ -package oap.http.test.cookies; - -import org.apache.http.cookie.ClientCookie; -import org.apache.http.cookie.Cookie; - -public class MockCookieFactory { - public static Cookie wrap( Cookie cookie ) { - return cookie instanceof ClientCookie ? new MockClientCookie<>( ( ClientCookie ) cookie ) : new MockCookie<>( cookie ); - } -} diff --git a/oap-http/oap-http-test/src/test/java/oap/http/ClientTest.java b/oap-http/oap-http-test/src/test/java/oap/http/ClientTest.java deleted file mode 100644 index c655f999c7..0000000000 --- a/oap-http/oap-http-test/src/test/java/oap/http/ClientTest.java +++ /dev/null @@ -1,140 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) Open Application Platform Authors - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package oap.http; - -import lombok.extern.slf4j.Slf4j; -import oap.testng.Fixtures; -import oap.testng.TestDirectoryFixture; -import org.apache.http.entity.ContentType; -import org.mockserver.integration.ClientAndServer; -import org.mockserver.matchers.Times; -import org.mockserver.model.HttpRequest; -import org.mockserver.model.HttpResponse; -import org.testng.annotations.AfterMethod; -import org.testng.annotations.BeforeMethod; -import org.testng.annotations.Test; - -import java.io.IOException; -import java.util.Optional; -import java.util.concurrent.atomic.AtomicInteger; - -import static java.net.HttpURLConnection.HTTP_OK; -import static oap.testng.Asserts.assertFile; -import static org.assertj.core.api.Assertions.assertThat; - -@Slf4j -public class ClientTest extends Fixtures { - private final TestDirectoryFixture testDirectoryFixture; - private int port; - private ClientAndServer mockServer; - private Client.Response response; - - public ClientTest() { - testDirectoryFixture = fixture( new TestDirectoryFixture() ); - } - - @BeforeMethod - public void start() { - mockServer = ClientAndServer.startClientAndServer( 0 ); - port = mockServer.getLocalPort(); - response = null; - } - - @AfterMethod - public void stop() { - mockServer.stop( true ); - } - - @Test - public void download() { - mockServer - .when( - HttpRequest.request() - .withMethod( "GET" ) - .withPath( "/file" ), - Times.once() - ) - .respond( - org.mockserver.model.HttpResponse.response() - .withStatusCode( HTTP_OK ) - .withBody( "test1" ) - ); - - var path = testDirectoryFixture.testPath( "new.file" ); - var progress = new AtomicInteger(); - var download = Client.DEFAULT.download( "http://localhost:" + port + "/file", - Optional.empty(), Optional.of( path ), progress::set ); - - assertThat( download ).contains( path ); - assertThat( download ).isPresent(); - assertFile( path ).exists().hasSize( 5 ); - assertThat( progress.get() ).isEqualTo( 100 ); - } - - @Test - public void downloadTempFile() { - mockServer - .when( - HttpRequest.request() - .withMethod( "GET" ) - .withPath( "/file.gz" ), - Times.once() - ) - .respond( - org.mockserver.model.HttpResponse.response() - .withStatusCode( HTTP_OK ) - .withBody( "test1" ) - ); - - var progress = new AtomicInteger(); - var download = Client.DEFAULT.download( "http://localhost:" + port + "/file.gz", - Optional.empty(), Optional.empty(), progress::set ); - assertThat( download ).isPresent(); - assertFile( download.get() ).exists().hasSize( 5 ); - assertFile( download.get() ).hasExtension( "gz" ); - assertThat( progress.get() ).isEqualTo( 100 ); - } - - @Test - public void postOutputStream() throws IOException { - mockServer.when( HttpRequest.request() - .withMethod( "POST" ) - .withPath( "/test" ) - .withBody( "test\ntest1" ), - Times.once() - ).respond( HttpResponse.response().withStatusCode( HTTP_OK ).withBody( "ok" ) ); - - try( var os = Client.DEFAULT.post( "http://localhost:" + port + "/test", ContentType.TEXT_PLAIN ) ) { - os.write( "test".getBytes() ); - os.write( '\n' ); - os.write( "test1".getBytes() ); - - response = os.waitAndGetResponse(); - - assertThat( response ).isNotNull(); - assertThat( response.contentString() ).isEqualTo( "ok" ); - } - } -} diff --git a/oap-http/oap-http-test/src/test/java/oap/http/GzipHttpTest.java b/oap-http/oap-http-test/src/test/java/oap/http/GzipHttpTest.java index a795d75e50..4b779dd358 100644 --- a/oap-http/oap-http-test/src/test/java/oap/http/GzipHttpTest.java +++ b/oap-http/oap-http-test/src/test/java/oap/http/GzipHttpTest.java @@ -24,25 +24,26 @@ package oap.http; +import oap.http.client.OapHttpClient; import oap.http.server.nio.NioHttpServer; import oap.io.IoStreams; import oap.io.content.ContentWriter; import oap.testng.Fixtures; import oap.testng.Ports; +import org.eclipse.jetty.client.HttpClient; import org.testng.annotations.AfterMethod; import org.testng.annotations.BeforeMethod; import org.testng.annotations.Test; +import java.io.ByteArrayInputStream; import java.util.Map; -import static java.net.HttpURLConnection.HTTP_OK; import static oap.compression.Compression.ContentWriter.ofGzip; import static oap.http.Http.ContentType.TEXT_PLAIN; import static oap.http.Http.Headers.ACCEPT_ENCODING; import static oap.http.Http.Headers.CONTENT_ENCODING; -import static oap.io.IoStreams.Encoding.GZIP; -import static oap.io.IoStreams.Encoding.PLAIN; -import static org.assertj.core.api.Assertions.assertThat; +import static oap.http.test.HttpAsserts.assertGet; +import static oap.http.test.HttpAsserts.assertPost; public class GzipHttpTest extends Fixtures { private NioHttpServer server; @@ -58,24 +59,29 @@ public void afterMethod() { } @Test - public void gzipOutput() { + public void gzipOutput() throws Exception { server.bind( "test", exchange -> exchange.responseOk( "test", true, TEXT_PLAIN ) ); server.start(); - var responseHtml = Client.DEFAULT.get( "http://localhost:" + server.defaultPort.httpPort + "/test" ); + assertGet( "http://localhost:" + server.defaultPort.httpPort + "/test" ) + .isOk() + .hasContentType( TEXT_PLAIN ) + .body() + .isEqualTo( "test" ); - assertThat( responseHtml.code ).isEqualTo( HTTP_OK ); - assertThat( responseHtml.contentType ).isEqualTo( TEXT_PLAIN ); - assertThat( responseHtml.contentString() ).isEqualTo( "test" ); + try( HttpClient httpClient = OapHttpClient.customHttpClient().build() ) { + // auto-decompression + httpClient.getContentDecoderFactories().clear(); - var responseGzip = Client.DEFAULT.get( "http://localhost:" + server.defaultPort.httpPort + "/test", - Map.of(), Map.of( ACCEPT_ENCODING, "gzip,deflate" ) ); - - assertThat( responseGzip.code ).isEqualTo( HTTP_OK ); - assertThat( responseGzip.contentType ).isEqualTo( TEXT_PLAIN ); - assertThat( IoStreams.asString( responseGzip.getInputStream(), GZIP ) ).isEqualTo( "test" ); + assertGet( httpClient, "http://localhost:" + server.defaultPort.httpPort + "/test", Map.of(), Map.of( ACCEPT_ENCODING, "gzip,deflate" ) ) + .isOk() + .hasContentType( TEXT_PLAIN ) + .containsHeader( "Content-Encoding", "gzip" ) + .body( bytes -> IoStreams.asString( new ByteArrayInputStream( bytes ), IoStreams.Encoding.GZIP ) ) + .isEqualTo( "test" ); + } } @Test @@ -85,11 +91,10 @@ public void gzipInput() { ); server.start(); - var response = Client.DEFAULT.post( "http://localhost:" + server.defaultPort.httpPort + "/test", - ContentWriter.write( "test2", ofGzip() ), TEXT_PLAIN, Map.of( CONTENT_ENCODING, "gzip" ) ); - - assertThat( response.code ).isEqualTo( HTTP_OK ); - assertThat( response.contentType ).isEqualTo( TEXT_PLAIN ); - assertThat( IoStreams.asString( response.getInputStream(), PLAIN ) ).isEqualTo( "test2" ); + assertPost( "http://localhost:" + server.defaultPort.httpPort + "/test", + ContentWriter.write( "test2", ofGzip() ), TEXT_PLAIN, Map.of( CONTENT_ENCODING, "gzip" ) ) + .hasContentType( TEXT_PLAIN ) + .body() + .isEqualTo( "test2" ); } } diff --git a/oap-http/oap-http-test/src/test/java/oap/http/file/HttpFileSyncTest.java b/oap-http/oap-http-test/src/test/java/oap/http/file/HttpFileSyncTest.java deleted file mode 100644 index 0591829c77..0000000000 --- a/oap-http/oap-http-test/src/test/java/oap/http/file/HttpFileSyncTest.java +++ /dev/null @@ -1,127 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) Open Application Platform Authors - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package oap.http.file; - -import oap.io.AbstractFileSync; -import oap.testng.Fixtures; -import oap.testng.Ports; -import oap.testng.SystemTimerFixture; -import oap.testng.TestDirectoryFixture; -import org.apache.http.client.utils.DateUtils; -import org.joda.time.DateTimeUtils; -import org.mockserver.integration.ClientAndServer; -import org.mockserver.matchers.Times; -import org.mockserver.model.Header; -import org.mockserver.model.HttpRequest; -import org.mockserver.model.HttpResponse; -import org.testng.annotations.AfterMethod; -import org.testng.annotations.BeforeMethod; -import org.testng.annotations.Test; - -import java.util.Date; - -import static java.net.HttpURLConnection.HTTP_NOT_MODIFIED; -import static java.net.HttpURLConnection.HTTP_OK; -import static org.assertj.core.api.Assertions.assertThat; - -public class HttpFileSyncTest extends Fixtures { - private final TestDirectoryFixture testDirectoryFixture; - private ClientAndServer mockServer; - - public HttpFileSyncTest() { - fixture( new SystemTimerFixture( true ) ); - testDirectoryFixture = fixture( new TestDirectoryFixture() ); - } - - @BeforeMethod - public void start() { - mockServer = ClientAndServer.startClientAndServer( Ports.getFreePort( getClass() ) ); - } - - @AfterMethod - public void stop() { - mockServer.stop( true ); - } - - @Test - public void sync() throws Exception { - var b = new StringBuilder(); - DateTimeUtils.setCurrentMillisFixed( 10 * 1000 ); - - var date10 = new Date( 10 * 1000 ); - var date20 = new Date( 20 * 1000 ); - - var localFile = testDirectoryFixture.testPath( "ltest.file" ); - - var fileSync = AbstractFileSync.create( "http://localhost:" + mockServer.getLocalPort() + "/file", localFile ); - fileSync.addListener( path -> b.append( "f" ) ); - - mockServer - .reset() - .when( HttpRequest.request().withMethod( "GET" ).withPath( "/file" ), Times.once() ) - .respond( - HttpResponse.response() - .withStatusCode( HTTP_OK ) - .withBody( "test" ) - .withHeader( Header.header( "Last-Modified", DateUtils.formatDate( date10 ) ) ) - .withHeader( Header.header( "Content-Disposition", "inline; filename=\"test.file\"" ) ) - ); - - fileSync.run(); - assertThat( localFile ).hasContent( "test" ); - assertThat( java.nio.file.Files.getLastModifiedTime( localFile ).toMillis() ).isEqualTo( 10L * 1000 ); - assertThat( b ).contains( "f" ); - - mockServer - .reset() - .when( HttpRequest.request().withMethod( "GET" ).withPath( "/file" ) - .withHeader( Header.header( "If-Modified-Since", DateUtils.formatDate( date10 ) ) ), - Times.once() ) - .respond( - HttpResponse.response() - .withStatusCode( HTTP_NOT_MODIFIED ) - .withHeader( Header.header( "Last-Modified", DateUtils.formatDate( date10 ) ) ) - ); - fileSync.run(); - assertThat( localFile ).hasContent( "test" ); - assertThat( b ).contains( "f" ); - - mockServer - .reset() - .when( HttpRequest.request().withMethod( "GET" ).withPath( "/file" ) - .withHeader( Header.header( "If-Modified-Since", DateUtils.formatDate( date10 ) ) ), - Times.once() ) - .respond( - HttpResponse.response() - .withStatusCode( HTTP_OK ) - .withBody( "test2" ) - .withHeader( Header.header( "Last-Modified", DateUtils.formatDate( date20 ) ) ) - .withHeader( Header.header( "Content-Disposition", "inline; filename=\"test.file\"" ) ) - ); - fileSync.run(); - assertThat( localFile ).hasContent( "test2" ); - assertThat( b ).contains( "ff" ); - } -} diff --git a/oap-http/oap-http-test/src/test/java/oap/http/server/nio/NioHttpServerTest.java b/oap-http/oap-http-test/src/test/java/oap/http/server/nio/NioHttpServerTest.java index 6d3c8de6a0..904abcf1d9 100644 --- a/oap-http/oap-http-test/src/test/java/oap/http/server/nio/NioHttpServerTest.java +++ b/oap-http/oap-http-test/src/test/java/oap/http/server/nio/NioHttpServerTest.java @@ -24,8 +24,8 @@ package oap.http.server.nio; -import oap.http.Client; import oap.http.Http; +import oap.http.client.OapHttpClient; import oap.http.server.nio.handlers.BlockingReadTimeoutHandler; import oap.http.server.nio.handlers.CompressionNioHandler; import oap.http.server.nio.handlers.KeepaliveRequestsHandler; @@ -34,9 +34,13 @@ import oap.testng.Fixtures; import oap.testng.Ports; import oap.util.Dates; +import org.eclipse.jetty.client.transport.HttpClientTransportDynamic; +import org.eclipse.jetty.io.ClientConnector; +import org.eclipse.jetty.util.ssl.SslContextFactory; import org.testng.annotations.Test; import java.io.IOException; +import java.util.Map; import static oap.http.Http.Headers.CONNECTION; import static oap.http.Http.Headers.DATE; @@ -51,12 +55,10 @@ public void testResponseHeaders() throws IOException { try( NioHttpServer httpServer = new NioHttpServer( new NioHttpServer.DefaultPort( port ) ) ) { httpServer.start(); - Client.Response response = Client.DEFAULT.get( "http://localhost:" + port + "/" ); - - assertThat( response.getHeaders() ) - .hasSize( 3 ) - .containsKey( DATE ) - .containsKey( CONNECTION ); + assertGet( "http://localhost:" + port + "/" ) + .hasHeadersSize( 3 ) + .containsHeader( DATE ) + .containsHeader( CONNECTION ); } try( NioHttpServer httpServer = new NioHttpServer( new NioHttpServer.DefaultPort( port ) ) ) { @@ -64,12 +66,10 @@ public void testResponseHeaders() throws IOException { httpServer.alwaysSetKeepAlive = false; httpServer.start(); - Client.Response response = Client.DEFAULT.get( "http://localhost:" + port + "/" ); - - assertThat( response.getHeaders() ) - .hasSize( 1 ) - .doesNotContainKey( DATE ) - .doesNotContainKey( CONNECTION ); + assertGet( "http://localhost:" + port + "/" ) + .hasHeadersSize( 1 ) + .doesNotContainHeader( DATE ) + .doesNotContainHeader( CONNECTION ); } } @@ -93,9 +93,9 @@ public void testBindToSpecificPort() throws IOException { httpServer.start(); httpServer.bind( "/test3", exchange -> exchange.responseOk( "test3", Http.ContentType.TEXT_PLAIN ), "test2" ); - assertThat( Client.DEFAULT.get( "http://localhost:" + testPort + "/test" ).contentString() ).isEqualTo( "test" ); - assertThat( Client.DEFAULT.get( "http://localhost:" + testPort2 + "/test2" ).contentString() ).isEqualTo( "test2" ); - assertThat( Client.DEFAULT.get( "http://localhost:" + testPort2 + "/test3" ).contentString() ).isEqualTo( "test3" ); + assertGet( "http://localhost:" + testPort + "/test" ).body().isEqualTo( "test" ); + assertGet( "http://localhost:" + testPort2 + "/test2" ).body().isEqualTo( "test2" ); + assertGet( "http://localhost:" + testPort2 + "/test3" ).body().isEqualTo( "test3" ); } } @@ -103,25 +103,28 @@ public void testBindToSpecificPort() throws IOException { * keytool -genkey -alias ssl -keyalg RSA -keysize 2048 -dname "CN=localhost,OU=IT" -keystore master.jks -storepass 1234567 -keypass 1234567 */ @Test - public void testHttps() throws IOException { + public void testHttps() throws Exception { int httpPort = Ports.getFreePort( getClass() ); int httpsPort = Ports.getFreePort( getClass() ); + ClientConnector connector = new ClientConnector(); + + SslContextFactory.Client sslContextFactory = new SslContextFactory.Client(); + sslContextFactory.setKeyStorePath( Resources.filePath( getClass(), "/oap/http/test_https.jks" ).get() ); + sslContextFactory.setKeyStorePassword( "1234567" ); + connector.setSslContextFactory( sslContextFactory ); + try( NioHttpServer httpServer = new NioHttpServer( new NioHttpServer.DefaultPort( httpPort, httpsPort, Resources.urlOrThrow( getClass(), "/oap/http/test_https.jks" ), "1234567" ) ); - Client client = Client - .custom( Resources.filePath( getClass(), "/oap/http/test_https.jks" ).get(), "1234567", 10000, 10000 ) - .build() ) { + org.eclipse.jetty.client.HttpClient httpClient = OapHttpClient.customHttpClient().transport( new HttpClientTransportDynamic( connector ) ).build() ) { + httpClient.start(); new TestHttpHandler( httpServer, "/test", "default-https" ); new HealthHttpHandler( httpServer, "/healtz", "default-http" ).start(); httpServer.start(); - assertThat( client.get( "https://localhost:" + httpsPort + "/test" ) - .contentString() ).isEqualTo( "ok" ); - + assertGet( httpClient, "https://localhost:" + httpsPort + "/test", Map.of(), Map.of() ).body().isEqualTo( "ok" ); assertGet( "http://localhost:" + httpPort + "/healtz" ).hasCode( Http.StatusCode.NO_CONTENT ); - } } diff --git a/oap-http/oap-http-test/src/test/java/oap/http/server/nio/handlers/KeepaliveRequestsHandlerTest.java b/oap-http/oap-http-test/src/test/java/oap/http/server/nio/handlers/KeepaliveRequestsHandlerTest.java index fecf877f35..b0662df39b 100644 --- a/oap-http/oap-http-test/src/test/java/oap/http/server/nio/handlers/KeepaliveRequestsHandlerTest.java +++ b/oap-http/oap-http-test/src/test/java/oap/http/server/nio/handlers/KeepaliveRequestsHandlerTest.java @@ -1,15 +1,17 @@ package oap.http.server.nio.handlers; -import oap.http.Client; import oap.http.Http; +import oap.http.client.OapHttpClient; import oap.http.server.nio.NioHttpServer; import oap.testng.Fixtures; import oap.testng.Ports; +import org.eclipse.jetty.client.HttpClient; import org.testng.annotations.Test; -import java.io.IOException; import java.util.LinkedHashSet; +import java.util.Map; +import static oap.http.test.HttpAsserts.assertGet; import static oap.testng.Asserts.assertEventually; import static org.assertj.core.api.Assertions.assertThat; @@ -22,9 +24,11 @@ public KeepaliveRequestsHandlerTest() { } @Test - public void testCloseConnectionBlocking() throws IOException { + public void testCloseConnectionBlocking() throws Exception { LinkedHashSet ids = new LinkedHashSet<>(); - try( NioHttpServer httpServer = new NioHttpServer( new NioHttpServer.DefaultPort( testHttpPort ) ) ) { + try( NioHttpServer httpServer = new NioHttpServer( new NioHttpServer.DefaultPort( testHttpPort ) ); + HttpClient client = OapHttpClient.customHttpClient().maxConnectionsPerDestination( 10 ).build() ) { + client.start(); KeepaliveRequestsHandler keepaliveRequestsHandler = new KeepaliveRequestsHandler( 2 ); httpServer.handlers.add( keepaliveRequestsHandler ); @@ -37,10 +41,9 @@ public void testCloseConnectionBlocking() throws IOException { exchange.responseOk( "ok", Http.ContentType.TEXT_PLAIN ); } ); - Client client = Client.custom().setMaxConnTotal( 10 ).setMaxConnPerRoute( 10 ).build(); for( int i = 0; i < 101; i++ ) { - assertThat( client.get( "http://localhost:" + testHttpPort + "/test" ).contentString() ).isEqualTo( "ok" ); + assertGet( client, "http://localhost:" + testHttpPort + "/test", Map.of(), Map.of() ).body().isEqualTo( "ok" ); } assertThat( ids ).hasSize( 51 ); @@ -51,9 +54,11 @@ public void testCloseConnectionBlocking() throws IOException { } @Test - public void testCloseConnectionAsync() throws IOException { + public void testCloseConnectionAsync() throws Exception { LinkedHashSet ids = new LinkedHashSet<>(); - try( NioHttpServer httpServer = new NioHttpServer( new NioHttpServer.DefaultPort( testHttpPort ) ) ) { + try( NioHttpServer httpServer = new NioHttpServer( new NioHttpServer.DefaultPort( testHttpPort ) ); + HttpClient client = OapHttpClient.customHttpClient().maxConnectionsPerDestination( 10 ).build() ) { + client.start(); KeepaliveRequestsHandler keepaliveRequestsHandler = new KeepaliveRequestsHandler( 2 ); httpServer.handlers.add( keepaliveRequestsHandler ); @@ -66,10 +71,8 @@ public void testCloseConnectionAsync() throws IOException { exchange.responseOk( "ok", Http.ContentType.TEXT_PLAIN ); }, true ); - Client client = Client.custom().setMaxConnTotal( 10 ).setMaxConnPerRoute( 10 ).build(); - for( int i = 0; i < 101; i++ ) { - assertThat( client.get( "http://localhost:" + testHttpPort + "/test" ).contentString() ).isEqualTo( "ok" ); + assertGet( client, "http://localhost:" + testHttpPort + "/test", Map.of(), Map.of() ).body().isEqualTo( "ok" ); } assertThat( ids ).hasSize( 51 ); diff --git a/oap-http/oap-http/pom.xml b/oap-http/oap-http/pom.xml index d5eb23fe59..b2abe8d564 100644 --- a/oap-http/oap-http/pom.xml +++ b/oap-http/oap-http/pom.xml @@ -78,14 +78,33 @@ + + + org.eclipse.jetty + jetty-client + ${oap.deps.jetty-client.version} + + + org.eclipse.jetty.compression + jetty-compression-gzip + ${oap.deps.jetty-client.version} + + + org.eclipse.jetty.compression + jetty-compression-brotli + ${oap.deps.jetty-client.version} + - com.squareup.okhttp3 - okhttp-jvm + org.eclipse.jetty.compression + jetty-compression-zstandard + ${oap.deps.jetty-client.version} - com.squareup.okhttp3 - okhttp-java-net-cookiejar + dnsjava + dnsjava + ${oap.deps.dnsjava.version} + org.projectlombok lombok diff --git a/oap-http/oap-http/src/main/java/oap/http/Client.java b/oap-http/oap-http/src/main/java/oap/http/Client.java deleted file mode 100644 index d6b85e0d16..0000000000 --- a/oap-http/oap-http/src/main/java/oap/http/Client.java +++ /dev/null @@ -1,872 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) Open Application Platform Authors - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ -package oap.http; - -import com.fasterxml.jackson.databind.MappingIterator; -import com.google.common.base.Preconditions; -import com.google.common.io.ByteStreams; -import lombok.SneakyThrows; -import lombok.ToString; -import lombok.extern.slf4j.Slf4j; -import oap.concurrent.AsyncCallbacks; -import oap.http.client.HttpClient; -import oap.io.Closeables; -import oap.io.Files; -import oap.io.IoStreams; -import oap.json.Binder; -import oap.reflect.TypeRef; -import oap.util.BiStream; -import oap.util.Dates; -import oap.util.Maps; -import oap.util.Pair; -import oap.util.Result; -import oap.util.Stream; -import oap.util.Throwables; -import oap.util.function.Try; -import oap.util.function.Try.ThrowingRunnable; -import org.apache.commons.lang3.StringUtils; -import org.apache.http.Header; -import org.apache.http.HttpResponse; -import org.apache.http.NameValuePair; -import org.apache.http.client.CookieStore; -import org.apache.http.client.config.CookieSpecs; -import org.apache.http.client.config.RequestConfig; -import org.apache.http.client.entity.UrlEncodedFormEntity; -import org.apache.http.client.methods.HttpDelete; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.client.methods.HttpPatch; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.client.methods.HttpPut; -import org.apache.http.client.methods.HttpRequestBase; -import org.apache.http.client.methods.HttpUriRequest; -import org.apache.http.client.utils.DateUtils; -import org.apache.http.concurrent.FutureCallback; -import org.apache.http.config.RegistryBuilder; -import org.apache.http.conn.ssl.DefaultHostnameVerifier; -import org.apache.http.conn.util.PublicSuffixMatcherLoader; -import org.apache.http.cookie.Cookie; -import org.apache.http.entity.ByteArrayEntity; -import org.apache.http.entity.ContentType; -import org.apache.http.entity.InputStreamEntity; -import org.apache.http.entity.StringEntity; -import org.apache.http.impl.client.BasicCookieStore; -import org.apache.http.impl.client.DefaultConnectionKeepAliveStrategy; -import org.apache.http.impl.nio.client.CloseableHttpAsyncClient; -import org.apache.http.impl.nio.client.HttpAsyncClientBuilder; -import org.apache.http.impl.nio.client.HttpAsyncClients; -import org.apache.http.impl.nio.conn.PoolingNHttpClientConnectionManager; -import org.apache.http.impl.nio.reactor.DefaultConnectingIOReactor; -import org.apache.http.impl.nio.reactor.IOReactorConfig; -import org.apache.http.message.BasicNameValuePair; -import org.apache.http.nio.conn.NoopIOSessionStrategy; -import org.apache.http.nio.conn.SchemeIOSessionStrategy; -import org.apache.http.nio.conn.ssl.SSLIOSessionStrategy; -import org.apache.http.nio.reactor.IOReactorException; -import org.apache.http.ssl.SSLContexts; - -import javax.annotation.Nonnull; -import javax.annotation.Nullable; -import java.io.BufferedInputStream; -import java.io.Closeable; -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.io.PipedInputStream; -import java.io.PipedOutputStream; -import java.io.UncheckedIOException; -import java.io.UnsupportedEncodingException; -import java.net.URI; -import java.nio.file.Path; -import java.util.Date; -import java.util.Iterator; -import java.util.List; -import java.util.Map; -import java.util.Objects; -import java.util.Optional; -import java.util.concurrent.CompletableFuture; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.Future; -import java.util.concurrent.TimeoutException; -import java.util.function.Consumer; -import java.util.stream.Collectors; - -import static java.net.HttpURLConnection.HTTP_MOVED_TEMP; -import static java.net.HttpURLConnection.HTTP_NOT_MODIFIED; -import static java.net.HttpURLConnection.HTTP_OK; -import static java.nio.charset.StandardCharsets.UTF_8; -import static java.util.concurrent.TimeUnit.MILLISECONDS; -import static oap.http.Http.ContentType.APPLICATION_OCTET_STREAM; -import static oap.io.IoStreams.Encoding.PLAIN; -import static oap.io.ProgressInputStream.progress; - -@Deprecated( since = "24.2.4" ) -/** - * replaced by okhttp - */ -@Slf4j -public final class Client implements Closeable, AutoCloseable { - public static final Client DEFAULT = custom() - .onError( ( c, e ) -> log.error( e.getMessage(), e ) ) - .onTimeout( c -> log.error( "timeout" ) ) - .build(); - public static final String NO_RESPONSE = "no response"; - private static final FutureCallback FUTURE_CALLBACK = new FutureCallback<>() { - @Override - public void completed( org.apache.http.HttpResponse result ) { - } - - @Override - public void failed( Exception e ) { - log.error( "Error appears", e ); - } - - @Override - public void cancelled() { - } - }; - private final CookieStore cookieStore; - private final ClientBuilder builder; - private CloseableHttpAsyncClient client; - - private Client( CookieStore cookieStore, ClientBuilder builder ) { - this.client = builder.client(); - - this.cookieStore = cookieStore; - this.builder = builder; - } - - public static ClientBuilder custom( Path certificateLocation, String certificatePassword, int connectTimeout, int readTimeout ) { - return new ClientBuilder( certificateLocation, certificatePassword, connectTimeout, readTimeout ); - } - - public static ClientBuilder custom() { - return new ClientBuilder( null, null, Dates.m( 1 ), Dates.m( 5 ) ); - } - - private static List> headers( org.apache.http.HttpResponse response ) { - return Stream.of( response.getAllHeaders() ) - .map( h -> Pair.__( h.getName(), h.getValue() ) ) - .toList(); - } - - private static String[] split( final String s ) { - if( StringUtils.isBlank( s ) ) { - return null; - } - return s.split( " *, *" ); - } - - public Response get( String uri ) { - return get( uri, Map.of(), Map.of() ); - } - - public Response get( URI uri ) { - return get( uri, Map.of() ); - } - - @SafeVarargs - public final Response get( String uri, Pair... params ) { - return get( uri, Maps.of( params ) ); - } - - public Response get( String uri, Map params ) { - return get( uri, params, Map.of() ); - } - - public Response get( String uri, Map params, Map headers ) { - return get( uri, params, headers, builder.timeout ) - .orElseThrow( Throwables::propagate ); - } - - public Response get( URI uri, Map headers ) { - return get( uri, headers, builder.timeout ) - .orElseThrow( Throwables::propagate ); - } - - public Result get( String uri, Map params, long timeout ) { - return get( uri, params, Map.of(), timeout ); - } - - public Result get( String uri, Map params, Map headers, long timeout ) { - return get( Uri.uri( uri, params ), headers, timeout ); - } - - public Result get( URI uri, Map headers, long timeout ) { - var request = new HttpGet( uri ); - return getResponse( request, timeout, execute( request, headers ) ); - } - - public Response post( String uri, Map params ) { - return post( uri, params, Map.of() ); - } - - public Response post( String uri, Map params, Map headers ) { - return post( uri, params, headers, builder.timeout ) - .orElseThrow( Throwables::propagate ); - } - - public Result post( String uri, Map params, long timeout ) { - return post( uri, params, Map.of(), timeout ); - } - - public Result post( String uri, Map params, Map headers, long timeout ) { - try { - var request = new HttpPost( uri ); - request.setEntity( new UrlEncodedFormEntity( Stream.of( params.entrySet() ) - .map( e -> new BasicNameValuePair( e.getKey(), - e.getValue() == null ? "" : e.getValue().toString() ) ) - .toList() - ) ); - return getResponse( request, Math.max( builder.timeout, timeout ), execute( request, headers ) ); - } catch( UnsupportedEncodingException e ) { - throw new UncheckedIOException( e ); - } - } - - public Response post( String uri, String content, String contentType ) { - return post( uri, content, contentType, Maps.of() ); - } - - public Response post( String uri, String content, String contentType, Map headers ) { - return post( uri, content, contentType, headers, builder.timeout ) - .orElseThrow( Throwables::propagate ); - } - - public Result post( String uri, String content, String contentType, long timeout ) { - return post( uri, content, contentType, Map.of(), timeout ); - } - - public Result post( String uri, String content, String contentType, Map headers, long timeout ) { - var request = new HttpPost( uri ); - request.setEntity( new StringEntity( content, ContentType.create( contentType ) ) ); - return getResponse( request, timeout, execute( request, headers ) ); - } - - public Result post( String uri, byte[] content, long timeout ) { - var request = new HttpPost( uri ); - request.setEntity( new ByteArrayEntity( content, ContentType.APPLICATION_OCTET_STREAM ) ); - return getResponse( request, timeout, execute( request, Map.of() ) ); - } - - public Result post( String uri, byte[] content, int off, int length, long timeout ) { - var request = new HttpPost( uri ); - request.setEntity( new ByteArrayEntity( content, off, length, ContentType.APPLICATION_OCTET_STREAM ) ); - return getResponse( request, timeout, execute( request, Map.of() ) ); - } - - @SneakyThrows - public OutputStreamWithResponse post( String uri, ContentType contentType ) throws UncheckedIOException { - var request = new HttpPost( uri ); - - return post( contentType, request ); - } - - @SneakyThrows - public OutputStreamWithResponse post( URI uri, ContentType contentType ) throws UncheckedIOException { - var request = new HttpPost( uri ); - - return post( contentType, request ); - } - - private OutputStreamWithResponse post( ContentType contentType, HttpPost request ) throws UncheckedIOException { - try { - var pos = new PipedOutputStream(); - var pis = new PipedInputStream( pos ); - request.setEntity( new InputStreamEntity( pis, contentType ) ); - - return new OutputStreamWithResponse( pos, execute( request, Map.of() ), request, builder.timeout ); - } catch( IOException e ) { - throw new UncheckedIOException( e ); - } - } - - public Response post( String uri, InputStream content, String contentType ) { - var request = new HttpPost( uri ); - request.setEntity( new InputStreamEntity( content, ContentType.create( contentType ) ) ); - return getResponse( request, builder.timeout, execute( request, Map.of() ) ) - .orElseThrow( Throwables::propagate ); - } - - public Response post( String uri, InputStream content, String contentType, Map headers ) { - var request = new HttpPost( uri ); - request.setEntity( new InputStreamEntity( content, ContentType.create( contentType ) ) ); - return getResponse( request, builder.timeout, execute( request, headers ) ) - .orElseThrow( Throwables::propagate ); - } - - public Response post( String uri, byte[] content, String contentType, Map headers ) { - var request = new HttpPost( uri ); - request.setEntity( new ByteArrayEntity( content, ContentType.create( contentType ) ) ); - return getResponse( request, builder.timeout, execute( request, headers ) ) - .orElseThrow( Throwables::propagate ); - } - - private Result getResponse( HttpRequestBase request, long timeout, CompletableFuture future ) { - try { - return Result.success( timeout == 0 ? future.get() : future.get( timeout, MILLISECONDS ) ); - } catch( ExecutionException e ) { - var newEx = new UncheckedIOException( request.getURI().toString(), new IOException( e.getCause().getMessage(), e.getCause() ) ); - builder.onError.accept( this, newEx ); - return Result.failure( e.getCause() ); - } catch( TimeoutException e ) { - this.builder.onTimeout.accept( this ); - return Result.failure( e ); - } catch( InterruptedException e ) { - Thread.currentThread().interrupt(); - this.builder.onError.accept( this, e ); - return Result.failure( e ); - } - } - - public Response put( String uri, String content, String contentType, Map headers ) { - var request = new HttpPut( uri ); - request.setEntity( new StringEntity( content, ContentType.create( contentType ) ) ); - return getResponse( request, builder.timeout, execute( request, headers ) ) - .orElseThrow( Throwables::propagate ); - } - - public Response put( String uri, String content, String contentType ) { - return put( uri, content, contentType, Map.of() ); - } - - public Response put( String uri, byte[] content, String contentType, Map headers ) { - var request = new HttpPut( uri ); - request.setEntity( new ByteArrayEntity( content, ContentType.parse( contentType ) ) ); - return getResponse( request, builder.timeout, execute( request, headers ) ) - .orElseThrow( Throwables::propagate ); - } - - public Response put( String uri, byte[] content, String contentType ) { - return put( uri, content, contentType, Map.of() ); - } - - public Response put( String uri, InputStream is, String contentType, Map headers ) { - var request = new HttpPut( uri ); - request.setEntity( new InputStreamEntity( is, ContentType.parse( contentType ) ) ); - return getResponse( request, builder.timeout, execute( request, headers ) ) - .orElseThrow( Throwables::propagate ); - } - - public Response put( String uri, InputStream is, String contentType ) { - return put( uri, is, contentType, Map.of() ); - } - - public Response patch( String uri, String content, String contentType, Map headers ) { - var request = new HttpPatch( uri ); - request.setEntity( new StringEntity( content, ContentType.create( contentType ) ) ); - return getResponse( request, builder.timeout, execute( request, headers ) ) - .orElseThrow( Throwables::propagate ); - } - - public Response patch( String uri, String content, String contentType ) { - return patch( uri, content, contentType, Map.of() ); - } - - public Response patch( String uri, byte[] content, String contentType, Map headers ) { - var request = new HttpPatch( uri ); - request.setEntity( new ByteArrayEntity( content, ContentType.parse( contentType ) ) ); - return getResponse( request, builder.timeout, execute( request, headers ) ) - .orElseThrow( Throwables::propagate ); - } - - public Response patch( String uri, byte[] content, String contentType ) { - return patch( uri, content, contentType, Map.of() ); - } - - public Response patch( String uri, InputStream is, String contentType, Map headers ) { - var request = new HttpPatch( uri ); - request.setEntity( new InputStreamEntity( is, ContentType.parse( contentType ) ) ); - return getResponse( request, builder.timeout, execute( request, headers ) ) - .orElseThrow( Throwables::propagate ); - } - - public Response patch( String uri, InputStream is, String contentType ) { - return patch( uri, is, contentType, Map.of() ); - } - - public Response delete( String uri ) { - return delete( uri, builder.timeout ); - } - - public Response delete( String uri, long timeout ) { - return delete( uri, Map.of(), timeout ); - } - - public Response delete( String uri, Map headers ) { - return delete( uri, headers, builder.timeout ); - } - - public Response delete( String uri, Map headers, long timeout ) { - var request = new HttpDelete( uri ); - return getResponse( request, Math.max( builder.timeout, timeout ), execute( request, headers ) ) - .orElseThrow( Throwables::propagate ); - } - - public List getCookies() { - return cookieStore.getCookies(); - } - - public void clearCookies() { - cookieStore.clear(); - } - - private CompletableFuture execute( HttpUriRequest request, Map headers ) { - return execute( request, headers, () -> {} ); - } - - @SneakyThrows - private CompletableFuture execute( HttpUriRequest request, Map headers, - ThrowingRunnable asyncRunnable ) { - headers.forEach( ( name, value ) -> request.setHeader( name, value == null ? "" : value.toString() ) ); - - CompletableFuture completableFuture = new CompletableFuture(); - - client.execute( request, new FutureCallback<>() { - @Override - public void completed( HttpResponse response ) { - try { - List> responseHeaders = headers( response ); - Response result; - if( response.getEntity() != null ) { - var entity = response.getEntity(); - result = new Response( - request.getURI().toString(), - response.getStatusLine().getStatusCode(), - response.getStatusLine().getReasonPhrase(), - responseHeaders, - entity.getContentType() != null - ? entity.getContentType().getValue() - : APPLICATION_OCTET_STREAM, - entity.getContent() - ); - } else result = new Response( - request.getURI().toString(), - response.getStatusLine().getStatusCode(), - response.getStatusLine().getReasonPhrase(), - responseHeaders - ); - builder.onSuccess.accept( Client.this ); - - completableFuture.complete( result ); - } catch( IOException e ) { - completableFuture.completeExceptionally( e ); - } - } - - @Override - public void failed( Exception ex ) { - completableFuture.completeExceptionally( ex ); - } - - @Override - public void cancelled() { - completableFuture.cancel( false ); - } - } ); - - asyncRunnable.run(); - - return completableFuture; - } - - @SneakyThrows - public Optional download( String url, Optional modificationTime, Optional file, Consumer progress ) { - try { - var response = resolve( url, modificationTime ).orElse( null ); - if( response == null ) return Optional.empty(); - - var entity = response.getEntity(); - - final Path path = file.orElseGet( Try.supply( () -> { - final IoStreams.Encoding encoding = IoStreams.Encoding.from( url ); - - final File tempFile = File.createTempFile( "file", "down" + encoding.extension ); - tempFile.deleteOnExit(); - return tempFile.toPath(); - } ) ); - - try( InputStream in = new BufferedInputStream( entity.getContent() ) ) { - IoStreams.write( path, PLAIN, in, false, file.isPresent(), progress( entity.getContentLength(), progress ) ); - } - - final Header lastModified = response.getLastHeader( "Last-Modified" ); - if( lastModified != null ) { - final Date date = DateUtils.parseDate( lastModified.getValue() ); - - Files.setLastModifiedTime( path, date.getTime() ); - } - - builder.onSuccess.accept( this ); - - return Optional.of( path ); - } catch( ExecutionException | IOException e ) { - builder.onError.accept( this, e ); - throw e; - } catch( InterruptedException e ) { - Thread.currentThread().interrupt(); - builder.onTimeout.accept( this ); - return Optional.empty(); - } - } - - private Optional resolve( String url, Optional ifModifiedSince ) throws InterruptedException, ExecutionException, IOException { - HttpGet request = new HttpGet( url ); - ifModifiedSince.ifPresent( ims -> request.addHeader( "If-Modified-Since", DateUtils.formatDate( new Date( ims ) ) ) ); - Future future = client.execute( request, FUTURE_CALLBACK ); - HttpResponse response = future.get(); - if( response.getStatusLine().getStatusCode() == HTTP_OK && response.getEntity() != null ) - return Optional.of( response ); - else if( response.getStatusLine().getStatusCode() == HTTP_MOVED_TEMP ) { - Header location = response.getFirstHeader( "Location" ); - if( location == null ) throw new IOException( "redirect w/o location!" ); - log.debug( "following {}", location.getValue() ); - return resolve( location.getValue(), Optional.empty() ); - } else if( response.getStatusLine().getStatusCode() == HTTP_NOT_MODIFIED ) { - return Optional.empty(); - } else - throw new IOException( response.getStatusLine().toString() ); - } - - public void reset() { - Closeables.close( client ); - client = builder.client(); - clearCookies(); - } - - @Override - public void close() { - Closeables.close( client ); - } - - @ToString( exclude = { "inputStream", "content" }, doNotUseGetters = true ) - public static class Response implements Closeable, AutoCloseable { - public final String url; - public final int code; - public final String reasonPhrase; - public final String contentType; - public final List> headers; - private InputStream inputStream; - private volatile byte[] content = null; - - public Response( String url, int code, String reasonPhrase, List> headers, @Nonnull String contentType, InputStream inputStream ) { - this.url = url; - this.code = code; - this.reasonPhrase = reasonPhrase; - this.headers = headers; - this.contentType = Objects.requireNonNull( contentType ); - this.inputStream = inputStream; - } - - public Response( String url, int code, String reasonPhrase, List> headers ) { - this( url, code, reasonPhrase, headers, BiStream.of( headers ) - .filter( ( name, value ) -> "Content-type".equalsIgnoreCase( name ) ) - .mapToObj( ( name, value ) -> value ) - .findAny() - .orElse( APPLICATION_OCTET_STREAM ), null ); - } - - public Optional header( @Nonnull String headerName ) { - return BiStream.of( headers ) - .filter( ( name, value ) -> headerName.equalsIgnoreCase( name ) ) - .mapToObj( ( name, value ) -> value ) - .findAny(); - } - - @Nullable - @SneakyThrows - public byte[] content() { - if( content == null && inputStream == null ) return null; - if( content == null ) synchronized( this ) { - if( content == null ) { - content = ByteStreams.toByteArray( inputStream ); - close(); - } - } - return content; - } - - public InputStream getInputStream() { - return inputStream; - } - - public String contentString() { - var text = content(); - - if( text == null ) return null; - - return new String( text, UTF_8 ); - } - - public Optional unmarshal( Class clazz ) { - if( inputStream != null ) synchronized( this ) { - if( inputStream != null ) - return Optional.of( Binder.json.unmarshal( clazz, inputStream ) ); - } - - var contentString = contentString(); - if( contentString == null ) return Optional.empty(); - - return Optional.of( Binder.json.unmarshal( clazz, contentString ) ); - } - - public Optional unmarshal( TypeRef ref ) { - if( inputStream != null ) synchronized( this ) { - if( inputStream != null ) - return Optional.of( Binder.json.unmarshal( ref, inputStream ) ); - } - - var contentString = contentString(); - if( contentString == null ) return Optional.empty(); - - return Optional.of( Binder.json.unmarshal( ref, contentString ) ); - } - - @SneakyThrows - public Stream unmarshalStream( TypeRef ref ) { - MappingIterator objectMappingIterator = null; - - if( inputStream != null ) { - synchronized( this ) { - if( inputStream != null ) { - objectMappingIterator = Binder.json.readerFor( ref ).readValues( inputStream ); - } - } - } - - if( objectMappingIterator == null ) { - var contentString = contentString(); - if( contentString == null ) - return Stream.empty(); - - objectMappingIterator = Binder.json.readerFor( ref ).readValues( contentString ); - } - - var finalObjectMappingIterator = objectMappingIterator; - - var it = new Iterator() { - @Override - public boolean hasNext() { - return finalObjectMappingIterator.hasNext(); - } - - @Override - @SuppressWarnings( "unchecked" ) - public T next() { - return ( T ) finalObjectMappingIterator.next(); - } - }; - - var stream = Stream.of( it ); - if( inputStream != null ) stream = stream.onClose( Try.run( () -> inputStream.close() ) ); - return stream; - } - - @Override - public void close() { - Closeables.close( inputStream ); - inputStream = null; - } - - public Map getHeaders() { - return headers.stream().collect( Collectors.toMap( p -> p._1, p -> p._2 ) ); - } - } - - public static class ClientBuilder extends AsyncCallbacks { - - private final Path certificateLocation; - private final String certificatePassword; - private final long timeout; - private CookieStore cookieStore; - private long connectTimeout; - private int maxConnTotal = 10000; - private int maxConnPerRoute = 1000; - private boolean redirectsEnabled = false; - private String cookieSpec = CookieSpecs.STANDARD; - - public ClientBuilder( Path certificateLocation, String certificatePassword, long connectTimeout, long timeout ) { - cookieStore = new BasicCookieStore(); - - this.certificateLocation = certificateLocation; - this.certificatePassword = certificatePassword; - this.connectTimeout = connectTimeout; - this.timeout = timeout; - } - - public ClientBuilder withCookieStore( CookieStore cookieStore ) { - this.cookieStore = cookieStore; - - return this; - } - - private HttpAsyncClientBuilder initialize() { - try { - final PoolingNHttpClientConnectionManager connManager = new PoolingNHttpClientConnectionManager( - new DefaultConnectingIOReactor( IOReactorConfig.custom() - .setConnectTimeout( ( int ) connectTimeout ) - .setSoTimeout( ( int ) timeout ) - .build() ), - RegistryBuilder.create() - .register( "http", NoopIOSessionStrategy.INSTANCE ) - .register( "https", - new SSLIOSessionStrategy( certificateLocation != null - ? HttpClient.createSSLContext( certificateLocation, certificatePassword ) - : SSLContexts.createDefault(), - split( System.getProperty( "https.protocols" ) ), - split( System.getProperty( "https.cipherSuites" ) ), - new DefaultHostnameVerifier( PublicSuffixMatcherLoader.getDefault() ) ) ) - .build() ); - - connManager.setMaxTotal( maxConnTotal ); - connManager.setDefaultMaxPerRoute( maxConnPerRoute ); - - return ( certificateLocation != null - ? HttpAsyncClients.custom() - .setSSLContext( HttpClient.createSSLContext( certificateLocation, certificatePassword ) ) - : HttpAsyncClients.custom() ) - .setMaxConnPerRoute( maxConnPerRoute ) - .setConnectionManager( connManager ) - .setMaxConnTotal( maxConnTotal ) - .setKeepAliveStrategy( DefaultConnectionKeepAliveStrategy.INSTANCE ) - .setDefaultRequestConfig( RequestConfig - .custom() - .setRedirectsEnabled( redirectsEnabled ) - .setCookieSpec( cookieSpec ) - .build() ) - .setDefaultCookieStore( cookieStore ); - } catch( IOReactorException e ) { - throw new UncheckedIOException( e ); - } - } - - public ClientBuilder setConnectTimeout( long connectTimeout ) { - this.connectTimeout = connectTimeout; - - return this; - } - - public ClientBuilder setMaxConnTotal( int maxConnTotal ) { - this.maxConnTotal = maxConnTotal; - - return this; - } - - public ClientBuilder setMaxConnPerRoute( int maxConnPerRoute ) { - this.maxConnPerRoute = maxConnPerRoute; - - return this; - } - - public ClientBuilder setRedirectsEnabled( boolean redirectsEnabled ) { - this.redirectsEnabled = redirectsEnabled; - - return this; - } - - public ClientBuilder setCookieSpec( String cookieSpec ) { - this.cookieSpec = cookieSpec; - - return this; - } - - private CloseableHttpAsyncClient client() { - final CloseableHttpAsyncClient build = initialize().build(); - build.start(); - return build; - } - - public Client build() { - return new Client( cookieStore, this ); - } - } - - public class OutputStreamWithResponse extends OutputStream implements Closeable, AutoCloseable { - private final CompletableFuture completableFuture; - private final HttpRequestBase request; - private final long timeout; - private PipedOutputStream pos; - private Response response; - - public OutputStreamWithResponse( PipedOutputStream pos, CompletableFuture completableFuture, HttpRequestBase request, long timeout ) { - this.pos = pos; - this.completableFuture = completableFuture; - this.request = request; - this.timeout = timeout; - } - - @Override - public void write( int b ) throws IOException { - pos.write( b ); - } - - @Override - public void write( @Nonnull byte[] b ) throws IOException { - pos.write( b ); - } - - @Override - public void write( @Nonnull byte[] b, int off, int len ) throws IOException { - pos.write( b, off, len ); - } - - @Override - public void flush() throws IOException { - pos.flush(); - } - - public Response waitAndGetResponse() { - Preconditions.checkState( response == null ); - try { - pos.flush(); - pos.close(); - Response result = getResponse( request, timeout, completableFuture ) - .orElseThrow( Throwables::propagate ); - response = result; - return result; - } catch( IOException e ) { - throw Throwables.propagate( e ); - } finally { - try { - pos.close(); - } catch( IOException e ) { - log.error( "Cannot close output", e ); - } finally { - pos = null; - } - } - } - - @Override - public void close() { - try { - if( response == null ) { - waitAndGetResponse(); - } - } finally { - response.close(); - } - } - } -} diff --git a/oap-http/oap-http/src/main/java/oap/http/InputStreamRequestBody.java b/oap-http/oap-http/src/main/java/oap/http/InputStreamRequestBody.java deleted file mode 100644 index 110f2530b4..0000000000 --- a/oap-http/oap-http/src/main/java/oap/http/InputStreamRequestBody.java +++ /dev/null @@ -1,35 +0,0 @@ -package oap.http; - -import okhttp3.MediaType; -import okhttp3.RequestBody; -import okio.BufferedSink; -import okio.Okio; -import okio.Source; - -import java.io.IOException; -import java.io.InputStream; - -public class InputStreamRequestBody extends RequestBody { - private final MediaType mediaType; - private final InputStream inputStream; - - public InputStreamRequestBody( MediaType mediaType, InputStream inputStream ) { - this.mediaType = mediaType; - this.inputStream = inputStream; - } - - @Override - public MediaType contentType() { - return mediaType; - } - - // You can override contentLength() if you know the length in advance for better performance/features (e.g., S3 uploads). - // If you don't override it, OkHttp will use chunked transfer encoding for large bodies. - - @Override - public void writeTo( BufferedSink sink ) throws IOException { - try( Source source = Okio.source( inputStream ) ) { - sink.writeAll( source ); - } - } -} diff --git a/oap-http/oap-http/src/main/java/oap/http/Response.java b/oap-http/oap-http/src/main/java/oap/http/Response.java new file mode 100644 index 0000000000..b90c3c4c44 --- /dev/null +++ b/oap-http/oap-http/src/main/java/oap/http/Response.java @@ -0,0 +1,168 @@ +package oap.http; + +import com.fasterxml.jackson.databind.MappingIterator; +import com.google.common.io.ByteStreams; +import lombok.SneakyThrows; +import lombok.ToString; +import oap.io.Closeables; +import oap.json.Binder; +import oap.reflect.TypeRef; +import oap.util.BiStream; +import oap.util.Pair; +import oap.util.Stream; +import oap.util.function.Try; + +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import java.io.Closeable; +import java.io.InputStream; +import java.util.ArrayDeque; +import java.util.Deque; +import java.util.Iterator; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; + +import static java.nio.charset.StandardCharsets.UTF_8; +import static oap.http.Http.ContentType.APPLICATION_OCTET_STREAM; + +@ToString( exclude = { "inputStream", "content" }, doNotUseGetters = true ) +public class Response implements Closeable, AutoCloseable { + public final String url; + public final int code; + public final String reasonPhrase; + public final String contentType; + public final List> headers; + private InputStream inputStream; + private volatile byte[] content = null; + + public Response( String url, int code, String reasonPhrase, List> headers, @Nonnull String contentType, InputStream inputStream ) { + this.url = url; + this.code = code; + this.reasonPhrase = reasonPhrase; + this.headers = headers; + this.contentType = Objects.requireNonNull( contentType ); + this.inputStream = inputStream; + } + + public Response( String url, int code, String reasonPhrase, List> headers ) { + this( url, code, reasonPhrase, headers, BiStream.of( headers ) + .filter( ( name, value ) -> "Content-type".equalsIgnoreCase( name ) ) + .mapToObj( ( name, value ) -> value ) + .findAny() + .orElse( APPLICATION_OCTET_STREAM ), null ); + } + + public Optional header( @Nonnull String headerName ) { + return BiStream.of( headers ) + .filter( ( name, value ) -> headerName.equalsIgnoreCase( name ) ) + .mapToObj( ( name, value ) -> value ) + .findAny(); + } + + @Nullable + @SneakyThrows + public byte[] content() { + if( content == null && inputStream == null ) return null; + if( content == null ) synchronized( this ) { + if( content == null ) { + content = ByteStreams.toByteArray( inputStream ); + close(); + } + } + return content; + } + + public InputStream getInputStream() { + return inputStream; + } + + public String contentString() { + var text = content(); + + if( text == null ) return null; + + return new String( text, UTF_8 ); + } + + public Optional unmarshal( Class clazz ) { + if( inputStream != null ) synchronized( this ) { + if( inputStream != null ) + return Optional.of( Binder.json.unmarshal( clazz, inputStream ) ); + } + + var contentString = contentString(); + if( contentString == null ) return Optional.empty(); + + return Optional.of( Binder.json.unmarshal( clazz, contentString ) ); + } + + public Optional unmarshal( TypeRef ref ) { + if( inputStream != null ) synchronized( this ) { + if( inputStream != null ) + return Optional.of( Binder.json.unmarshal( ref, inputStream ) ); + } + + var contentString = contentString(); + if( contentString == null ) return Optional.empty(); + + return Optional.of( Binder.json.unmarshal( ref, contentString ) ); + } + + @SneakyThrows + public Stream unmarshalStream( TypeRef ref ) { + MappingIterator objectMappingIterator = null; + + if( inputStream != null ) { + synchronized( this ) { + if( inputStream != null ) { + objectMappingIterator = Binder.json.readerFor( ref ).readValues( inputStream ); + } + } + } + + if( objectMappingIterator == null ) { + var contentString = contentString(); + if( contentString == null ) + return Stream.empty(); + + objectMappingIterator = Binder.json.readerFor( ref ).readValues( contentString ); + } + + var finalObjectMappingIterator = objectMappingIterator; + + var it = new Iterator() { + @Override + public boolean hasNext() { + return finalObjectMappingIterator.hasNext(); + } + + @Override + @SuppressWarnings( "unchecked" ) + public T next() { + return ( T ) finalObjectMappingIterator.next(); + } + }; + + var stream = Stream.of( it ); + if( inputStream != null ) stream = stream.onClose( Try.run( () -> inputStream.close() ) ); + return stream; + } + + @Override + public void close() { + Closeables.close( inputStream ); + inputStream = null; + } + + public Map> getHeaders() { + LinkedHashMap> ret = new LinkedHashMap<>(); + + for( Pair header : headers ) { + ret.computeIfAbsent( header._1, _ -> new ArrayDeque<>() ).add( header._2 ); + } + return ret; + } +} diff --git a/oap-http/oap-http/src/main/java/oap/http/client/HttpClient.java b/oap-http/oap-http/src/main/java/oap/http/client/HttpClient.java deleted file mode 100644 index b4d2e222db..0000000000 --- a/oap-http/oap-http/src/main/java/oap/http/client/HttpClient.java +++ /dev/null @@ -1,93 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) Open Application Platform Authors - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package oap.http.client; - -import lombok.SneakyThrows; -import lombok.extern.slf4j.Slf4j; -import oap.io.IoStreams; - -import javax.net.ssl.SSLContext; -import javax.net.ssl.TrustManager; -import javax.net.ssl.TrustManagerFactory; -import javax.net.ssl.X509TrustManager; -import java.nio.file.Path; -import java.security.KeyManagementException; -import java.security.KeyStore; -import java.security.NoSuchAlgorithmException; -import java.security.SecureRandom; -import java.security.cert.X509Certificate; - -import static oap.io.IoStreams.Encoding.PLAIN; - -@Slf4j -public final class HttpClient { - private java.net.http.HttpClient impl; - - private HttpClient() { - try { - SSLContext sslContext = SSLContext.getInstance( "TLS" ); - sslContext.init( null, new TrustManager[] { ACCEPTING_TRUST_MANAGER }, new SecureRandom() ); - impl = java.net.http.HttpClient.newBuilder() - .version( java.net.http.HttpClient.Version.HTTP_2 ) - .followRedirects( java.net.http.HttpClient.Redirect.ALWAYS ) - .sslContext( sslContext ) - .build(); - } catch( NoSuchAlgorithmException | KeyManagementException e ) { - log.error( e.getMessage(), e ); - } - } - - public static final HttpClient DEFAULT = new HttpClient(); - public static final X509TrustManager ACCEPTING_TRUST_MANAGER = new X509TrustManager() { - @Override - public void checkClientTrusted( X509Certificate[] x509Certificates, String s ) { - } - - @Override - public void checkServerTrusted( X509Certificate[] x509Certificates, String s ) { - } - - @Override - public X509Certificate[] getAcceptedIssuers() { - return new X509Certificate[0]; - } - }; - - @SneakyThrows - public static SSLContext createSSLContext( Path certificateLocation, String certificatePassword ) { - try( var inputStream = IoStreams.in( certificateLocation, PLAIN ) ) { - KeyStore keyStore = KeyStore.getInstance( "JKS" ); - keyStore.load( inputStream, certificatePassword.toCharArray() ); - - TrustManagerFactory trustManagerFactory = TrustManagerFactory.getInstance( TrustManagerFactory.getDefaultAlgorithm() ); - trustManagerFactory.init( keyStore ); - - SSLContext sslContext = SSLContext.getInstance( "TLS" ); - sslContext.init( null, trustManagerFactory.getTrustManagers(), null ); - - return sslContext; - } - } -} diff --git a/oap-http/oap-http/src/main/java/oap/http/client/JettyRequestExtensions.java b/oap-http/oap-http/src/main/java/oap/http/client/JettyRequestExtensions.java new file mode 100644 index 0000000000..2d25560f24 --- /dev/null +++ b/oap-http/oap-http/src/main/java/oap/http/client/JettyRequestExtensions.java @@ -0,0 +1,19 @@ +package oap.http.client; + +import org.eclipse.jetty.client.Request; + +import java.util.Map; + +public class JettyRequestExtensions { + public static Request addParams( Request request, Map params ) { + params.forEach( ( k, v ) -> request.param( k, v.toString() ) ); + + return request; + } + + public static Request addHeaders( Request request, Map headers ) { + return request.headers( h -> { + headers.forEach( ( k, v ) -> h.add( k, v == null ? "" : v.toString() ) ); + } ); + } +} diff --git a/oap-http/oap-http/src/main/java/oap/http/client/OapHttpClient.java b/oap-http/oap-http/src/main/java/oap/http/client/OapHttpClient.java new file mode 100644 index 0000000000..41e8b995dd --- /dev/null +++ b/oap-http/oap-http/src/main/java/oap/http/client/OapHttpClient.java @@ -0,0 +1,174 @@ +package oap.http.client; + +import io.micrometer.core.instrument.Gauge; +import io.micrometer.core.instrument.Metrics; +import io.micrometer.core.instrument.binder.BaseUnits; +import lombok.SneakyThrows; +import oap.util.Dates; +import oap.util.Lists; +import org.eclipse.jetty.client.AbstractConnectionPool; +import org.eclipse.jetty.client.AbstractConnectorHttpClientTransport; +import org.eclipse.jetty.client.HttpClient; +import org.eclipse.jetty.client.WWWAuthenticationProtocolHandler; +import org.eclipse.jetty.http.HttpCookieStore; +import org.eclipse.jetty.io.ClientConnector; +import org.eclipse.jetty.util.thread.QueuedThreadPool; +import org.eclipse.jetty.util.thread.VirtualThreadPool; +import org.xbill.DNS.Address; + +import java.net.InetAddress; +import java.net.InetSocketAddress; +import java.net.SocketAddress; +import java.net.UnknownHostException; +import java.nio.channels.SocketChannel; +import java.util.List; +import java.util.concurrent.atomic.LongAdder; + +public class OapHttpClient { + public static final HttpClient DEFAULT_HTTP_CLIENT = customHttpClient().build(); + + public static OapHttpClientBuilder customHttpClient() { + return new OapHttpClientBuilder(); + } + + public static void resetCookies() { + resetCookies( DEFAULT_HTTP_CLIENT ); + } + + public static void resetCookies( HttpClient httpClient ) { + httpClient.getHttpCookieStore().clear(); + } + + public static class OapHttpClientBuilder { + public AbstractConnectorHttpClientTransport httpClientTransport; + public long connectionTimeout = Dates.s( 10 ); + public boolean followRedirects = false; + public int maxConnectionsPerDestination = 64; + public boolean dnsjava = false; + private String metrics; + private HttpCookieStore cookieStore; + + public OapHttpClientBuilder transport( AbstractConnectorHttpClientTransport httpClientTransport ) { + this.httpClientTransport = httpClientTransport; + + return this; + } + + public OapHttpClientBuilder connectionTimeout( long connectionTimeout ) { + this.connectionTimeout = connectionTimeout; + + return this; + } + + public OapHttpClientBuilder followRedirects( boolean followRedirects ) { + this.followRedirects = followRedirects; + + return this; + } + + public OapHttpClientBuilder maxConnectionsPerDestination( int maxConnectionsPerDestination ) { + this.maxConnectionsPerDestination = maxConnectionsPerDestination; + + return this; + } + + public OapHttpClientBuilder dnsjava( boolean enabled ) { + this.dnsjava = enabled; + + return this; + } + + public OapHttpClientBuilder metrics( String name ) { + this.metrics = name; + + return this; + } + + public OapHttpClientBuilder cookieStore( HttpCookieStore cookieStore ) { + this.cookieStore = cookieStore; + + return this; + } + + @SneakyThrows + public HttpClient build() { + HttpClient httpClient = httpClientTransport != null ? new HttpClient( httpClientTransport ) : new HttpClient(); + httpClient.setConnectTimeout( connectionTimeout ); + QueuedThreadPool qtp = new QueuedThreadPool(); + qtp.setVirtualThreadsExecutor( new VirtualThreadPool() ); + httpClient.setExecutor( qtp ); + httpClient.setFollowRedirects( followRedirects ); + httpClient.setMaxConnectionsPerDestination( maxConnectionsPerDestination ); + + if( cookieStore != null ) { + httpClient.setHttpCookieStore( cookieStore ); + } + + if( dnsjava ) { + httpClient.setSocketAddressResolver( ( host, port, context, promise ) -> { + try { + if( "localhost".equals( host ) ) { + promise.succeeded( List.of( new InetSocketAddress( InetAddress.getLocalHost(), port ) ) ); + return; + } + + InetAddress[] inetAddresses = Address.getAllByName( host ); + + promise.succeeded( Lists.map( inetAddresses, ia -> new InetSocketAddress( ia, port ) ) ); + } catch( UnknownHostException e ) { + promise.failed( e ); + } + } ); + } + + if( metrics != null ) { + ( ( AbstractConnectorHttpClientTransport ) httpClient.getHttpClientTransport() ).getClientConnector().addEventListener( new ClientConnectorConnectListener( metrics ) ); + + Gauge.builder( "dsp_tunneling_connections", httpClient, cs -> cs.getDestinations().stream().mapToInt( d -> ( ( AbstractConnectionPool ) d.getConnectionPool() ).getMaxConnectionCount() ).sum() ) + .baseUnit( BaseUnits.CONNECTIONS ) + .tags( "event", "max", "client", metrics ) + .register( Metrics.globalRegistry ); + Gauge.builder( "dsp_tunneling_connections", httpClient, cs -> cs.getDestinations().stream().mapToInt( d -> ( ( AbstractConnectionPool ) d.getConnectionPool() ).getConnectionCount() ).sum() ) + .baseUnit( BaseUnits.CONNECTIONS ) + .tags( "event", "total", "client", metrics ) + .register( Metrics.globalRegistry ); + Gauge.builder( "dsp_tunneling_connections", httpClient, cs -> cs.getDestinations().stream().mapToInt( d -> ( ( AbstractConnectionPool ) d.getConnectionPool() ).getActiveConnectionCount() ).sum() ) + .baseUnit( BaseUnits.CONNECTIONS ) + .tags( "event", "active", "client", metrics ) + .register( Metrics.globalRegistry ); + } + + httpClient.start(); + + httpClient.getProtocolHandlers().remove( WWWAuthenticationProtocolHandler.NAME ); + + return httpClient; + } + + public static class ClientConnectorConnectListener implements ClientConnector.ConnectListener { + public final LongAdder connectSuccessCounter = new LongAdder(); + public final LongAdder connectFailedCounter = new LongAdder(); + + public ClientConnectorConnectListener( String name ) { + Gauge.builder( "dsp_tunneling_connections", this, cs -> cs.connectSuccessCounter.doubleValue() ) + .baseUnit( BaseUnits.CONNECTIONS ) + .tags( "event", "success", "client", name ) + .register( Metrics.globalRegistry ); + Gauge.builder( "dsp_tunneling_connections", this, cs -> cs.connectFailedCounter.doubleValue() ) + .baseUnit( BaseUnits.CONNECTIONS ) + .tags( "event", "failed", "client", name ) + .register( Metrics.globalRegistry ); + } + + @Override + public void onConnectSuccess( SocketChannel socketChannel ) { + connectSuccessCounter.increment(); + } + + @Override + public void onConnectFailure( SocketChannel socketChannel, SocketAddress socketAddress, Throwable failure ) { + connectFailedCounter.increment(); + } + } + } +} diff --git a/oap-http/oap-http/src/main/java/oap/http/file/HttpFileSync.java b/oap-http/oap-http/src/main/java/oap/http/file/HttpFileSync.java deleted file mode 100644 index e4f3bc9f41..0000000000 --- a/oap-http/oap-http/src/main/java/oap/http/file/HttpFileSync.java +++ /dev/null @@ -1,48 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) Open Application Platform Authors - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package oap.http.file; - -import lombok.SneakyThrows; -import lombok.extern.slf4j.Slf4j; -import oap.http.Client; -import oap.io.AbstractFileSync; - -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.Optional; - -@Slf4j -public class HttpFileSync extends AbstractFileSync { - public HttpFileSync() { - super( "http", "https" ); - } - - @SneakyThrows - protected Optional download() { - Optional modificationTime = Files.exists( localFile ) - ? Optional.of( Files.getLastModifiedTime( localFile ).toMillis() ) : Optional.empty(); - return Client.DEFAULT.download( uri.toString(), modificationTime, Optional.of( localFile ), i -> {} ); - } -} diff --git a/oap-http/oap-http/src/main/java/oap/http/server/nio/HttpServerExchange.java b/oap-http/oap-http/src/main/java/oap/http/server/nio/HttpServerExchange.java index 5c7f7251ca..90f44f921a 100644 --- a/oap-http/oap-http/src/main/java/oap/http/server/nio/HttpServerExchange.java +++ b/oap-http/oap-http/src/main/java/oap/http/server/nio/HttpServerExchange.java @@ -273,7 +273,7 @@ public void responseOk( Object content, boolean raw, String contentType ) { } public byte[] readBody() throws IOException { - var ret = new ByteArrayOutputStream(); + ByteArrayOutputStream ret = new ByteArrayOutputStream(); IOUtils.copy( getInputStream(), ret ); return ret.toByteArray(); @@ -283,7 +283,7 @@ public void responseStream( Stream content, boolean raw, String contentType ) setStatusCode( HTTP_OK ); setResponseHeader( Headers.CONTENT_TYPE, contentType ); - var out = exchange.getOutputStream(); + OutputStream out = exchange.getOutputStream(); content .map( v -> contentToString( raw, v, contentType ) ) @@ -319,16 +319,20 @@ public HttpServerExchange setResponseCookie( oap.http.Cookie cookie ) { } private Cookie convert( oap.http.Cookie cookie ) { - return new CookieImpl( cookie.getName(), cookie.getValue() ) + CookieImpl newCookie = new CookieImpl( cookie.getName(), cookie.getValue() ); + newCookie .setPath( cookie.getPath() ) .setDomain( cookie.getDomain() ) - .setMaxAge( cookie.getMaxAge() ) .setExpires( cookie.getExpires() ) .setDiscard( cookie.isDiscard() ) .setSecure( cookie.isSecure() ) .setHttpOnly( cookie.isHttpOnly() ) .setVersion( cookie.getVersion() ) .setComment( cookie.getComment() ); + if( cookie.getMaxAge() != null ) { + newCookie.setMaxAge( cookie.getMaxAge() ); + } + return newCookie; } public Iterable responseCookies() { diff --git a/oap-http/oap-pnio-v3/src/test/java/oap/http/pniov3/PnioServerTest.java b/oap-http/oap-pnio-v3/src/test/java/oap/http/pniov3/PnioServerTest.java index 727e3ba762..3578ff9c2e 100644 --- a/oap-http/oap-pnio-v3/src/test/java/oap/http/pniov3/PnioServerTest.java +++ b/oap-http/oap-pnio-v3/src/test/java/oap/http/pniov3/PnioServerTest.java @@ -1,17 +1,21 @@ package oap.http.pniov3; -import oap.concurrent.Executors; -import oap.concurrent.ThreadPoolExecutor; -import oap.http.Client; +import oap.http.Http; +import oap.http.client.OapHttpClient; import oap.http.server.nio.NioHttpServer; import oap.testng.Fixtures; import oap.util.Dates; +import org.eclipse.jetty.client.HttpClient; import org.testng.annotations.Test; -import java.io.IOException; +import java.util.Map; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicInteger; +import static oap.http.test.HttpAsserts.assertGet; + public class PnioServerTest extends Fixtures { private final PortFixture fixture; @@ -20,41 +24,55 @@ public PnioServerTest() { } @Test - public void testRequestUndertow() throws InterruptedException, IOException { + public void testRequestUndertow() throws Exception { int port = fixture.definePort( "test" ); - ThreadPoolExecutor threadPoolExecutor = Executors.newFixedBlockingThreadPool( 1024 ); + try( ExecutorService threadPoolExecutor = Executors.newVirtualThreadPerTaskExecutor() ) { + try( HttpClient httpClient = OapHttpClient.customHttpClient().maxConnectionsPerDestination( 2000 ).build() ) { + httpClient.start(); + + AtomicInteger errorCount = new AtomicInteger(); + AtomicInteger okCount = new AtomicInteger(); - Client client = Client.custom().setMaxConnPerRoute( 20000 ).setMaxConnTotal( 20000 ).build(); + try( NioHttpServer pnioServer = new NioHttpServer( new NioHttpServer.DefaultPort( port ) ) ) { + pnioServer.ioThreads = Runtime.getRuntime().availableProcessors(); + pnioServer.bind( "/pnio", exchange -> { + exchange.setStatusCode( 204 ); + exchange.endExchange(); + } ); + pnioServer.start(); - AtomicInteger errorCount = new AtomicInteger(); - AtomicInteger okCount = new AtomicInteger(); + long start = System.currentTimeMillis(); - try( NioHttpServer pnioServer = new NioHttpServer( new NioHttpServer.DefaultPort( port ) ) ) { - pnioServer.ioThreads = Runtime.getRuntime().availableProcessors(); - pnioServer.bind( "/pnio", exchange -> { - exchange.setStatusCode( 204 ); - exchange.endExchange(); - } ); - pnioServer.start(); + for( int i = 0; i < 20; i++ ) { + threadPoolExecutor.execute( () -> { + try { + assertGet( httpClient, "http://localhost:" + port + "/pnio?trace=true", Map.of(), Map.of() ) + .hasCode( Http.StatusCode.NO_CONTENT ); + okCount.incrementAndGet(); + } catch( Exception e ) { + errorCount.incrementAndGet(); + } + } ); + } - long start = System.currentTimeMillis(); + System.out.println( "ok " + okCount.get() + " error " + errorCount.get() + " duration " + Dates.durationToString( System.currentTimeMillis() - start ) ); + } - for( int i = 0; i < 20; i++ ) { - threadPoolExecutor.submit( () -> { + for( int i = 0; i < 20; i++ ) { +// threadPoolExecutor.execute( () -> { try { - Client.Response response = client.get( "http://localhost:" + port + "/pnio?trace=true" ); + assertGet( httpClient, "http://localhost:" + port + "/pnio?trace=true", Map.of(), Map.of() ) + .hasCode( Http.StatusCode.NO_CONTENT ); okCount.incrementAndGet(); } catch( Exception e ) { errorCount.incrementAndGet(); } - } ); +// } ); + } } - - System.out.println( "ok " + okCount.get() + " error " + errorCount.get() + " duration " + Dates.durationToString( System.currentTimeMillis() - start ) ); - threadPoolExecutor.shutdown(); - threadPoolExecutor.awaitTermination( 10, TimeUnit.SECONDS ); + threadPoolExecutor.awaitTermination( 20, TimeUnit.SECONDS ); } } } diff --git a/oap-message/oap-message-client/src/main/java/oap/message/client/MessageSender.java b/oap-message/oap-message-client/src/main/java/oap/message/client/MessageSender.java index 4c6e647deb..42a9199da4 100644 --- a/oap-message/oap-message-client/src/main/java/oap/message/client/MessageSender.java +++ b/oap-message/oap-message-client/src/main/java/oap/message/client/MessageSender.java @@ -33,6 +33,8 @@ import oap.LogConsolidated; import oap.concurrent.scheduler.Scheduled; import oap.concurrent.scheduler.Scheduler; +import oap.http.Http; +import oap.http.client.OapHttpClient; import oap.io.Closeables; import oap.io.Files; import oap.io.content.ContentReader; @@ -46,16 +48,14 @@ import oap.util.FastByteArrayOutputStream; import oap.util.Pair; import oap.util.Throwables; -import okhttp3.Dispatcher; -import okhttp3.MediaType; -import okhttp3.OkHttpClient; -import okhttp3.Request; -import okhttp3.RequestBody; -import okhttp3.Response; import org.apache.commons.codec.DecoderException; import org.apache.commons.codec.binary.Hex; import org.apache.commons.codec.digest.DigestUtils; import org.apache.commons.io.FilenameUtils; +import org.eclipse.jetty.client.BytesRequestContent; +import org.eclipse.jetty.client.InputStreamResponseListener; +import org.eclipse.jetty.client.Response; +import org.eclipse.jetty.http.HttpMethod; import org.joda.time.DateTimeUtils; import org.slf4j.event.Level; @@ -69,20 +69,14 @@ import java.nio.file.DirectoryStream; import java.nio.file.Path; import java.nio.file.Paths; -import java.time.Duration; -import java.util.concurrent.CompletableFuture; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; -import java.util.concurrent.TimeoutException; +import java.util.concurrent.locks.ReentrantLock; @Slf4j @ToString public class MessageSender implements Closeable, AutoCloseable { - public static final MediaType MEDIA_TYPE = MediaType.parse( "application/octet-stream" ); private static final Pair STATUS_OK = Pair.__( MessageStatus.OK, MessageProtocol.STATUS_OK ); private final Object syncDiskLock = new Object(); private final String host; @@ -94,9 +88,9 @@ public class MessageSender implements Closeable, AutoCloseable { private final ConcurrentMap> lastStatus = new ConcurrentHashMap<>(); private final String messageUrl; private final long memorySyncPeriod; + private final ReentrantLock lock = new ReentrantLock(); public String uniqueName = Cuid.UNIQUE.next(); public long storageLockExpiration = Dates.h( 1 ); - public int poolSize = 4; public long diskSyncPeriod = Dates.m( 1 ); public long globalIoRetryTimeout = Dates.s( 1 ); public long retryTimeout = Dates.s( 1 ); @@ -106,8 +100,6 @@ public class MessageSender implements Closeable, AutoCloseable { private volatile boolean closed = false; private Scheduled diskSyncScheduler; private boolean networkAvailable = true; - private OkHttpClient httpClient; - private ExecutorService executor; private long ioExceptionStartRetryTimeout = -1; public MessageSender( String host, int port, String httpPrefix, Path persistenceDirectory, long memorySyncPeriod ) { @@ -145,31 +137,17 @@ public final long getClientId() { return clientId; } + @SneakyThrows public void start() { log.info( "[{}] message server messageUrl {} storage {} storageLockExpiration {}", uniqueName, messageUrl, directory, Dates.durationToString( storageLockExpiration ) ); - log.info( "[{}] connection timeout {} rw timeout {} pool size {} keepAliveDuration {}", - uniqueName, Dates.durationToString( connectionTimeout ), Dates.durationToString( timeout ), poolSize, + log.info( "[{}] connection timeout {} rw timeout {} keepAliveDuration {}", + uniqueName, Dates.durationToString( connectionTimeout ), Dates.durationToString( timeout ), Dates.durationToString( keepAliveDuration ) ); log.info( "[{}] retry timeout {} disk sync period '{}' memory sync period '{}'", uniqueName, Dates.durationToString( retryTimeout ), Dates.durationToString( diskSyncPeriod ), Dates.durationToString( memorySyncPeriod ) ); log.info( "custom status = {}", MessageProtocol.printMapping() ); - executor = Executors.newVirtualThreadPerTaskExecutor(); - - OkHttpClient.Builder clientBuilder = new OkHttpClient.Builder() - .connectTimeout( Duration.ofMillis( connectionTimeout ) ) - .callTimeout( Duration.ofMillis( timeout ) ); - - if( poolSize > 0 ) { - Dispatcher dispatcher = new Dispatcher(); - dispatcher.setMaxRequests( poolSize ); - dispatcher.setMaxRequestsPerHost( poolSize ); - clientBuilder.dispatcher( dispatcher ); - } - - httpClient = clientBuilder.build(); - if( diskSyncPeriod > 0 ) diskSyncScheduler = Scheduler.scheduleWithFixedDelay( diskSyncPeriod, TimeUnit.MILLISECONDS, this::syncDisk ); @@ -203,12 +181,16 @@ public MessageSender send( byte messageType, short version, byte[] data, int off } + @SneakyThrows @Override public void close() { - synchronized( syncDiskLock ) { + lock.lock(); + try { closed = true; Closeables.close( diskSyncScheduler ); + } finally { + lock.unlock(); } int count = 0; @@ -263,50 +245,51 @@ public MessageAvailabilityReport availabilityReport( byte messageType ) { } @SuppressWarnings( "checkstyle:OverloadMethodsDeclarationOrder" ) - public CompletableFuture send( Messages.MessageInfo messageInfo, long now ) { + public Messages.MessageInfo send( Messages.MessageInfo messageInfo, long now ) { Message message = messageInfo.message; log.debug( "[{}] sending data [type = {}] to server...", uniqueName, MessageProtocol.messageTypeToString( message.messageType ) ); - return CompletableFuture.supplyAsync( () -> { - Metrics.counter( "oap.messages", "type", MessageProtocol.messageTypeToString( message.messageType ), "status", "trysend" ).increment(); + Metrics.counter( "oap.messages", "type", MessageProtocol.messageTypeToString( message.messageType ), "status", "trysend" ).increment(); - try( FastByteArrayOutputStream buf = new FastByteArrayOutputStream(); - DataOutputStream out = new DataOutputStream( buf ) ) { - out.writeByte( message.messageType ); - out.writeShort( message.version ); - out.writeLong( message.clientId ); + try( FastByteArrayOutputStream buf = new FastByteArrayOutputStream(); + DataOutputStream out = new DataOutputStream( buf ) ) { + out.writeByte( message.messageType ); + out.writeShort( message.version ); + out.writeLong( message.clientId ); - out.write( message.md5.bytes ); + out.write( message.md5.bytes ); - out.write( MessageProtocol.RESERVED, 0, MessageProtocol.RESERVED_LENGTH ); - out.writeInt( message.data.length ); - out.write( message.data ); + out.write( MessageProtocol.RESERVED, 0, MessageProtocol.RESERVED_LENGTH ); + out.writeInt( message.data.length ); + out.write( message.data ); - Request request = new Request.Builder() - .url( messageUrl ) - .post( RequestBody.create( buf.array, MEDIA_TYPE, 0, buf.length ) ) - .build(); + InputStreamResponseListener inputStreamResponseListener = new InputStreamResponseListener(); + OapHttpClient.DEFAULT_HTTP_CLIENT + .newRequest( messageUrl ) + .method( HttpMethod.POST ) + .timeout( timeout, TimeUnit.MILLISECONDS ) + .body( new BytesRequestContent( Http.ContentType.APPLICATION_OCTET_STREAM, buf.array ) ) + .send( inputStreamResponseListener ); - Response response = httpClient.newCall( request ).execute(); + Response response = inputStreamResponseListener.get( timeout, TimeUnit.MILLISECONDS ); - if( response.code() >= 300 || response.code() < 200 ) { - throw new IOException( "Not OK (" + response.code() + ") response code returned for url: " + messageUrl ); - } - return onOkRespone( messageInfo, response, now ); + if( response.getStatus() >= 300 || response.getStatus() < 200 ) { + throw new IOException( "Not OK (" + response.getStatus() + ") response code returned for url: " + messageUrl ); + } + return onOkRespone( messageInfo, inputStreamResponseListener, now ); - } catch( UnknownHostException e ) { - processException( messageInfo, now, message, e, true ); + } catch( UnknownHostException e ) { + processException( messageInfo, now, message, e, true ); - ioExceptionStartRetryTimeout = now; + ioExceptionStartRetryTimeout = now; - throw Throwables.propagate( e ); - } catch( Throwable e ) { - processException( messageInfo, now, message, e, false ); + throw Throwables.propagate( e ); + } catch( Throwable e ) { + processException( messageInfo, now, message, e, false ); - throw Throwables.propagate( e ); - } - }, executor ); + throw Throwables.propagate( e ); + } } private void processException( Messages.MessageInfo messageInfo, long now, Message message, Throwable e, boolean globalRetryTimeout ) { @@ -317,10 +300,10 @@ private void processException( Messages.MessageInfo messageInfo, long now, Messa messages.retry( messageInfo, now + retryTimeout ); } - private Messages.MessageInfo onOkRespone( Messages.MessageInfo messageInfo, Response response, long now ) { + private Messages.MessageInfo onOkRespone( Messages.MessageInfo messageInfo, InputStreamResponseListener inputStreamResponseListener, long now ) { Message message = messageInfo.message; - InputStream body = response.body().byteStream(); + InputStream body = inputStreamResponseListener.getInputStream(); try( DataInputStream in = new DataInputStream( body ) ) { byte version = in.readByte(); @@ -383,10 +366,6 @@ private Messages.MessageInfo onOkRespone( Messages.MessageInfo messageInfo, Resp } public void syncMemory() { - syncMemory( -1 ); - } - - public void syncMemory( long timeoutMs ) { if( getReadyMessages() + getRetryMessages() + getInProgressMessages() > 0 ) log.trace( "[{}] sync ready {} retry {} inprogress {} ...", uniqueName, getReadyMessages(), getRetryMessages(), getInProgressMessages() ); @@ -406,20 +385,10 @@ public void syncMemory( long timeoutMs ) { if( messageInfo != null ) { log.trace( "[{}] message {}...", uniqueName, messageInfo.message.md5 ); - CompletableFuture future = send( messageInfo, now ); - future.handle( ( mi, e ) -> { - messages.removeInProgress( mi ); - log.trace( "[{}] message {}... done", uniqueName, mi.message.md5 ); - return null; - } ); - - if( timeoutMs >= 0 ) { - try { - future.get( timeoutMs, TimeUnit.MILLISECONDS ); - } catch( InterruptedException | TimeoutException | ExecutionException e ) { - log.error( e.getMessage(), e ); - } - } + Messages.MessageInfo mi = send( messageInfo, now ); + + messages.removeInProgress( mi ); + log.trace( "[{}] message {}... done", uniqueName, mi.message.md5 ); } if( isGlobalIoRetryTimeout( now ) ) { @@ -500,7 +469,8 @@ public MessageSender syncDisk() { Path lockFile; - synchronized( syncDiskLock ) { + lock.lock(); + try { if( closed ) return this; if( ( lockFile = lock( uniqueName, messagePath, storageLockExpiration ) ) != null ) { @@ -521,6 +491,8 @@ public MessageSender syncDisk() { Files.delete( lockFile ); } } + } finally { + lock.unlock(); } } } diff --git a/oap-message/oap-message-client/src/main/resources/META-INF/oap-module.oap b/oap-message/oap-message-client/src/main/resources/META-INF/oap-module.oap index bc0707b928..420a949c4f 100644 --- a/oap-message/oap-message-client/src/main/resources/META-INF/oap-module.oap +++ b/oap-message/oap-message-client/src/main/resources/META-INF/oap-module.oap @@ -4,14 +4,12 @@ services { oap-http-message-sender { implementation = oap.message.client.MessageSender parameters { - connectionTimeout = 30s timeout = 5s retryTimeout = 1s globalIoRetryTimeout = 1s diskSyncPeriod = 1m memorySyncPeriod = 100 keepAliveDuration = 30d - poolSize = -1 storageLockExpiration = 1h port = 8081 diff --git a/oap-message/oap-message-test/src/test/java/oap/message/MessageServerTest.java b/oap-message/oap-message-test/src/test/java/oap/message/MessageServerTest.java index 97c54d57e5..1752fdf68d 100644 --- a/oap-message/oap-message-test/src/test/java/oap/message/MessageServerTest.java +++ b/oap-message/oap-message-test/src/test/java/oap/message/MessageServerTest.java @@ -94,16 +94,12 @@ public void rejectedException() throws IOException { try( MessageSender client1 = new MessageSender( "localhost", port, "/messages", testDirectoryFixture.testPath( "tmp" ), -1 ); MessageSender client2 = new MessageSender( "localhost", port, "/messages", testDirectoryFixture.testPath( "tmp" ), -1 ) ) { - client1.poolSize = 1; - client2.poolSize = 1; - client1.start(); client2.start(); messageHttpHandler.preStart(); server.start(); - client1.send( MessageListenerMock.MESSAGE_TYPE, ( short ) 1, "rejectedException", ofString() ) - .syncMemory( Dates.s( 10 ) ); + client1.send( MessageListenerMock.MESSAGE_TYPE, ( short ) 1, "rejectedException", ofString() ).syncMemory(); assertThat( listener1.getMessages() ).containsOnly( new TestMessage( 1, "rejectedException" ) ); @@ -111,12 +107,9 @@ public void rejectedException() throws IOException { } try( MessageSender client = new MessageSender( "localhost", port, "/messages", testDirectoryFixture.testPath( "tmp" ), -1 ) ) { - client.poolSize = 1; - client.start(); - client.send( MessageListenerMock.MESSAGE_TYPE, ( short ) 1, "rejectedException 2", ofString() ).syncDisk() - .syncMemory( Dates.s( 10 ) ); + client.send( MessageListenerMock.MESSAGE_TYPE, ( short ) 1, "rejectedException 2", ofString() ).syncDisk().syncMemory(); assertThat( listener1.getMessages() ).containsOnly( new TestMessage( 1, "rejectedException" ), new TestMessage( 1, "rejectedException 2" ) ); assertThat( client.getReadyMessages() ).isEqualTo( 0L ); @@ -148,7 +141,7 @@ public void sendAndReceive() throws IOException { .send( MessageListenerMock.MESSAGE_TYPE, ( short ) 1, "sendAndReceive 2", ofString() ) .send( MessageListenerMock.MESSAGE_TYPE, ( short ) 1, "sendAndReceive 1", ofString() ) .send( MessageListenerMock.MESSAGE_TYPE2, ( short ) 1, "sendAndReceive 3", ofString() ) - .syncMemory( Dates.s( 10 ) ); + .syncMemory(); assertThat( listener1.getMessages() ).containsOnly( new TestMessage( 1, "sendAndReceive 1" ), new TestMessage( 1, "sendAndReceive 2" ) ); @@ -182,7 +175,7 @@ public void sendAndReceiveJson() throws IOException { .send( MessageListenerMock.MESSAGE_TYPE, ( short ) 1, "sendAndReceiveJson 2", ofJson() ) .send( MessageListenerMock.MESSAGE_TYPE, ( short ) 1, "sendAndReceiveJson 2", ofJson() ) .send( MessageListenerMock.MESSAGE_TYPE, ( short ) 1, "sendAndReceiveJson 1", ofJson() ) - .syncMemory( Dates.s( 10 ) ); + .syncMemory(); assertThat( listener1.messages ).containsOnly( new TestMessage( 1, Hex.encodeHexString( DigestUtils.getMd5Digest().digest( "\"sendAndReceiveJson 1\"".getBytes( UTF_8 ) ) ), "sendAndReceiveJson 1" ), @@ -201,7 +194,6 @@ public void sendAndReceiveJsonOneThread() throws IOException { try( NioHttpServer server = new NioHttpServer( new NioHttpServer.DefaultPort( port ) ); MessageHttpHandler messageHttpHandler = new MessageHttpHandler( server, "/messages", controlStatePath, List.of( listener1 ), -1 ); MessageSender client = new MessageSender( "localhost", port, "/messages", testDirectoryFixture.testPath( "tmp" ), -1 ) ) { - client.poolSize = 1; server.bind( "/messages", messageHttpHandler ); client.start(); @@ -213,7 +205,7 @@ public void sendAndReceiveJsonOneThread() throws IOException { .send( MessageListenerMock.MESSAGE_TYPE, ( short ) 1, "sendAndReceiveJsonOneThread 2", ofJson() ) .send( MessageListenerMock.MESSAGE_TYPE, ( short ) 1, "sendAndReceiveJsonOneThread 2", ofJson() ) .send( MessageListenerMock.MESSAGE_TYPE, ( short ) 1, "sendAndReceiveJsonOneThread 1", ofJson() ) - .syncMemory( Dates.s( 10 ) ); + .syncMemory(); assertThat( listener1.messages ).containsOnly( new TestMessage( 1, Hex.encodeHexString( DigestUtils.getMd5Digest().digest( "\"sendAndReceiveJsonOneThread 1\"".getBytes( UTF_8 ) ) ), "sendAndReceiveJsonOneThread 1" ), @@ -240,7 +232,7 @@ public void unknownErrorNoRetry() throws IOException { listener1.throwUnknownError( Integer.MAX_VALUE, true ); client.send( MessageListenerMock.MESSAGE_TYPE, ( short ) 1, "unknownErrorNoRetry", ofString() ) - .syncMemory( Dates.s( 10 ) ); + .syncMemory(); assertThat( client.getReadyMessages() ).isEqualTo( 0L ); assertThat( client.getRetryMessages() ).isEqualTo( 0L ); @@ -273,7 +265,7 @@ public void unknownError() throws IOException { client.send( MessageListenerMock.MESSAGE_TYPE, ( short ) 1, "unknownError", ofString() ); for( int i = 0; i < 5; i++ ) { - client.syncMemory( Dates.s( 10 ) ); + client.syncMemory(); Dates.incFixed( 100 + 1 ); } assertThat( listener1.throwUnknownError ).isLessThanOrEqualTo( 0 ); @@ -306,7 +298,7 @@ public void statusError() throws IOException { listener1.setStatus( 567 ); client.send( MessageListenerMock.MESSAGE_TYPE, ( short ) 1, "statusError", ofString() ) - .syncMemory( Dates.s( 10 ) ); + .syncMemory(); assertThat( client.getRetryMessages() ).isEqualTo( 1 ); assertThat( listener1.getMessages() ).isEmpty(); @@ -315,7 +307,7 @@ public void statusError() throws IOException { Dates.incFixed( 100 + 1 ); - client.syncMemory( Dates.s( 10 ) ); + client.syncMemory(); assertThat( listener1.getMessages() ).containsOnly( new TestMessage( 1, "statusError" ) ); } @@ -347,7 +339,7 @@ public void ttl() throws IOException { .send( MessageListenerMock.MESSAGE_TYPE, ( short ) 1, "ttl", ofString() ) .send( MessageListenerMock.MESSAGE_TYPE, ( short ) 1, "ttl", ofString() ) .send( MessageListenerMock.MESSAGE_TYPE, ( short ) 1, "ttl", ofString() ) - .syncMemory( Dates.s( 10 ) ); + .syncMemory(); assertThat( listener1.getMessages() ).containsOnly( new TestMessage( 1, "ttl" ) ); @@ -358,7 +350,7 @@ public void ttl() throws IOException { .send( MessageListenerMock.MESSAGE_TYPE, ( short ) 1, "ttl", ofString() ) .send( MessageListenerMock.MESSAGE_TYPE, ( short ) 1, "ttl", ofString() ) .send( MessageListenerMock.MESSAGE_TYPE, ( short ) 1, "ttl", ofString() ) - .syncMemory( Dates.s( 10 ) ); + .syncMemory(); assertThat( listener1.getMessages() ).containsExactly( new TestMessage( 1, "ttl" ), @@ -392,7 +384,7 @@ public void persistence() throws IOException { .send( MessageListenerMock.MESSAGE_TYPE, ( short ) 1, "persistence", ofString() ) .send( MessageListenerMock.MESSAGE_TYPE, ( short ) 1, "persistence", ofString() ) .send( MessageListenerMock.MESSAGE_TYPE, ( short ) 1, "persistence", ofString() ) - .syncMemory( Dates.s( 10 ) ); + .syncMemory(); assertThat( client.getReadyMessages() ).isEqualTo( 0L ); assertThat( client.getRetryMessages() ).isEqualTo( 0L ); @@ -414,7 +406,7 @@ public void persistence() throws IOException { .send( MessageListenerMock.MESSAGE_TYPE, ( short ) 1, "persistence", ofString() ) .send( MessageListenerMock.MESSAGE_TYPE, ( short ) 1, "persistence", ofString() ) .send( MessageListenerMock.MESSAGE_TYPE, ( short ) 1, "persistence", ofString() ) - .syncMemory( Dates.s( 10 ) ); + .syncMemory(); assertThat( client.getReadyMessages() ).isEqualTo( 0L ); assertThat( client.getRetryMessages() ).isEqualTo( 0L ); @@ -448,7 +440,7 @@ public void clientPersistence() throws IOException { client.start(); client.send( MessageListenerMock.MESSAGE_TYPE, ( short ) 2, "clientPersistence 1", ofString() ) - .syncMemory( Dates.s( 10 ) ); + .syncMemory(); client.send( MessageListenerMock.MESSAGE_TYPE2, ( short ) 2, "clientPersistence 2", ofString() ); assertThat( listener1.getMessages() ).isEmpty(); @@ -478,7 +470,7 @@ public void clientPersistence() throws IOException { assertThat( listener1.getMessages() ).isEmpty(); client.syncDisk(); - client.syncMemory( Dates.s( 10 ) ); + client.syncMemory(); assertThat( persistenceDirectory ).isEmptyDirectory(); @@ -509,14 +501,13 @@ public void clientPersistenceLockExpiration() throws IOException { Path msgDirectory = testDirectoryFixture.testPath( "tmp" ); try( MessageSender client = new MessageSender( "localhost", port, "/messages", msgDirectory, -1 ) ) { client.retryTimeout = 100; - client.poolSize = 2; client.start(); listener1.throwUnknownError = 2; client .send( MessageListenerMock.MESSAGE_TYPE, ( short ) 1, "clientPersistenceLockExpiration 1", ofString() ) .send( MessageListenerMock.MESSAGE_TYPE, ( short ) 1, "clientPersistenceLockExpiration 2", ofString() ) - .syncMemory( Dates.s( 10 ) ); + .syncMemory(); } assertThat( Files.wildcard( msgDirectory, "**/*.bin" ) ).hasSize( 2 ); @@ -541,7 +532,7 @@ public void clientPersistenceLockExpiration() throws IOException { client .syncDisk() - .syncMemory( Dates.s( 10 ) ); + .syncMemory(); assertThat( listener1.getMessages() ).containsExactly( new TestMessage( 1, "clientPersistenceLockExpiration 2" ) ); assertThat( client.getReadyMessages() ).isEqualTo( 0L ); @@ -572,14 +563,14 @@ public void availabilityReport() throws IOException { listener1.setStatus( 300 ); client.send( MessageListenerMock.MESSAGE_TYPE, ( short ) 1, "availabilityReport", ofString() ) - .syncMemory( Dates.s( 20 ) ); + .syncMemory(); assertThat( client.availabilityReport( MessageListenerMock.MESSAGE_TYPE ).state ).isEqualTo( State.FAILED ); assertThat( client.availabilityReport( MessageListenerMock.MESSAGE_TYPE2 ).state ).isEqualTo( State.OPERATIONAL ); listener1.setStatus( MessageProtocol.STATUS_OK ); - client.syncMemory( Dates.s( 10 ) ); + client.syncMemory(); assertThat( client.availabilityReport( MessageListenerMock.MESSAGE_TYPE ).state ).isEqualTo( State.OPERATIONAL ); assertThat( client.availabilityReport( MessageListenerMock.MESSAGE_TYPE2 ).state ).isEqualTo( State.OPERATIONAL ); @@ -598,7 +589,7 @@ public void testKernel() { kernelFixture.service( "oap-message-client", MessageSender.class ) .send( ( byte ) 12, ( short ) 1, "testKernel", ofString() ) - .syncMemory( Dates.s( 10 ) ); + .syncMemory(); assertThat( kernelFixture.service( "oap-message-test", MessageListenerMock.class ).getMessages() ) .containsExactly( new TestMessage( 1, "testKernel" ) ); diff --git a/oap-stdlib-test/src/test/java/oap/io/FileSystemFileSyncTest.java b/oap-stdlib-test/src/test/java/oap/io/FileSystemFileSyncTest.java deleted file mode 100644 index d51063e0be..0000000000 --- a/oap-stdlib-test/src/test/java/oap/io/FileSystemFileSyncTest.java +++ /dev/null @@ -1,77 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) Open Application Platform Authors - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package oap.io; - -import lombok.SneakyThrows; -import oap.testng.Fixtures; -import oap.testng.TestDirectoryFixture; -import org.testng.annotations.Test; - -import java.nio.file.Paths; - -import static oap.io.content.ContentWriter.ofString; -import static org.assertj.core.api.Assertions.assertThat; - -public class FileSystemFileSyncTest extends Fixtures { - private final TestDirectoryFixture testDirectoryFixture; - - public FileSystemFileSyncTest() { - testDirectoryFixture = fixture( new TestDirectoryFixture() ); - } - - @Test - @SneakyThrows - public void sync() { - StringBuilder b = new StringBuilder(); - - var remoteFile = testDirectoryFixture.testPath( "rtest.file" ).toUri(); - var localFile = testDirectoryFixture.testPath( "ltest.file" ); - - Files.write( Paths.get( remoteFile ), "test", ofString() ); - - Files.setLastModifiedTime( Paths.get( remoteFile ), 10 ); - - final AbstractFileSync fileSync = AbstractFileSync.create( remoteFile, localFile ); - fileSync.addListener( path -> b.append( "f" ) ); - fileSync.run(); - - assertThat( localFile ).hasContent( "test" ); - assertThat( java.nio.file.Files.getLastModifiedTime( localFile ).toMillis() ).isEqualTo( 10L ); - assertThat( b ).contains( "f" ); - - Files.write( Paths.get( remoteFile ), "test2", ofString() ); - Files.setLastModifiedTime( Paths.get( remoteFile ), 10 ); - - fileSync.run(); - assertThat( localFile ).hasContent( "test" ); - assertThat( b ).contains( "f" ); - - Files.setLastModifiedTime( Paths.get( remoteFile ), 20L ); - fileSync.run(); - assertThat( localFile ).hasContent( "test2" ); - assertThat( b ).contains( "ff" ); - } - -} diff --git a/oap-stdlib/src/main/java/oap/compression/Compression.java b/oap-stdlib/src/main/java/oap/compression/Compression.java index fe6de26699..877270d03e 100644 --- a/oap-stdlib/src/main/java/oap/compression/Compression.java +++ b/oap-stdlib/src/main/java/oap/compression/Compression.java @@ -24,46 +24,46 @@ package oap.compression; import lombok.SneakyThrows; -import oap.util.function.Try; -import org.apache.commons.compress.compressors.CompressorStreamFactory; import org.apache.commons.io.IOUtils; +import java.io.ByteArrayInputStream; import java.io.ByteArrayOutputStream; +import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; -import java.util.function.BiFunction; -import java.util.function.Function; import java.util.zip.GZIPInputStream; import java.util.zip.GZIPOutputStream; import static java.nio.charset.StandardCharsets.UTF_8; public class Compression { - public static final int DEFAULT_BUFFER_SIZE = 512; - private static final Function gzipInputStreamSupplyer; - private static final BiFunction gzipOutputStreamSupplier; - - static { - if( "apache".equals( System.getProperty( "oap.io.gzip", "apache" ) ) ) { - CompressorStreamFactory factory = new CompressorStreamFactory( true ); - gzipInputStreamSupplyer = Try.map( is -> factory.createCompressorInputStream( CompressorStreamFactory.GZIP, is ) ); - gzipOutputStreamSupplier = Try.biMap( ( os, bufferSize ) -> factory.createCompressorOutputStream( CompressorStreamFactory.GZIP, os ) ); - } else { - gzipInputStreamSupplyer = Try.map( GZIPInputStream::new ); - gzipOutputStreamSupplier = Try.biMap( GZIPOutputStream::new ); + public static void gzip( OutputStream out, byte[] bytes, int offset, int length ) throws IOException { + try( OutputStream gos = new GZIPOutputStream( out ) ) { + gos.write( bytes, offset, length ); } } - public static OutputStream gzip( OutputStream os ) { - return gzip( os, DEFAULT_BUFFER_SIZE ); + public static byte[] gzip( byte[] bytes, int offset, int length ) throws IOException { + ByteArrayOutputStream byteArrayOutputStream = new ByteArrayOutputStream(); + gzip( byteArrayOutputStream, bytes, offset, length ); + return byteArrayOutputStream.toByteArray(); + } + + public static byte[] gzip( byte[] bytes ) throws IOException { + return gzip( bytes, 0, bytes.length ); + } + + public static byte[] ungzip( byte[] bytes ) throws IOException { + return ungzip( bytes, 0, bytes.length ); } - public static OutputStream gzip( OutputStream os, int bufferSize ) { - return gzipOutputStreamSupplier.apply( os, bufferSize ); + public static byte[] ungzip( byte[] bytes, int offset, int length ) throws IOException { + return new GZIPInputStream( new ByteArrayInputStream( bytes, offset, length ) ).readAllBytes(); } - public static InputStream ungzip( InputStream is ) { - return gzipInputStreamSupplyer.apply( is ); + public static void ungzip( byte[] bytes, int offset, int length, OutputStream out ) throws IOException { + GZIPInputStream gzipInputStream = new GZIPInputStream( new ByteArrayInputStream( bytes, offset, length ) ); + IOUtils.copy( gzipInputStream, out ); } public static class ContentWriter { @@ -72,7 +72,7 @@ public static oap.io.content.ContentWriter ofGzip() { @Override @SneakyThrows public void write( OutputStream os, String object ) { - try( var gos = gzip( os ) ) { + try( GZIPOutputStream gos = new GZIPOutputStream( os ) ) { gos.write( object.getBytes( UTF_8 ) ); } } @@ -86,8 +86,8 @@ public static oap.io.content.ContentReader ofBytes() { @Override @SneakyThrows public byte[] read( InputStream is ) { - var baos = new ByteArrayOutputStream(); - try( var gos = ungzip( is ) ) { + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + try( GZIPInputStream gos = new GZIPInputStream( is ) ) { IOUtils.copy( gos, baos ); } return baos.toByteArray(); diff --git a/oap-stdlib/src/main/java/oap/compression/CompressionUtils.java b/oap-stdlib/src/main/java/oap/compression/CompressionUtils.java deleted file mode 100644 index 0c13e68b11..0000000000 --- a/oap-stdlib/src/main/java/oap/compression/CompressionUtils.java +++ /dev/null @@ -1,73 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) Open Application Platform Authors - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package oap.compression; - -import org.apache.commons.io.IOUtils; - -import java.io.ByteArrayInputStream; -import java.io.ByteArrayOutputStream; -import java.io.IOException; -import java.util.zip.GZIPInputStream; -import java.util.zip.GZIPOutputStream; - -import static java.nio.charset.StandardCharsets.UTF_8; - -/** - * @see Compression - */ -@Deprecated -public class CompressionUtils { - private CompressionUtils() { - } - - public static byte[] gzip( String content ) throws IOException { - var baos = new ByteArrayOutputStream(); - try( var gos = new GZIPOutputStream( baos ) ) { - gos.write( content.getBytes() ); - } - - return baos.toByteArray(); - } - - public static String ungzip( byte[] content ) throws IOException { - var bais = new ByteArrayInputStream( content ); - var baos = new ByteArrayOutputStream(); - try( var gos = new GZIPInputStream( bais ) ) { - IOUtils.copy( gos, baos ); - } - - return baos.toString( UTF_8 ); - } - - public static String ungzip( byte[] data, int offset, int dataLength ) throws IOException { - var bais = new ByteArrayInputStream( data, offset, dataLength ); - var baos = new ByteArrayOutputStream(); - try( var gos = new GZIPInputStream( bais ) ) { - IOUtils.copy( gos, baos ); - } - - return baos.toString( UTF_8 ); - } -} diff --git a/oap-stdlib/src/main/java/oap/io/AbstractFileSync.java b/oap-stdlib/src/main/java/oap/io/AbstractFileSync.java deleted file mode 100644 index 64bcd9b9f0..0000000000 --- a/oap-stdlib/src/main/java/oap/io/AbstractFileSync.java +++ /dev/null @@ -1,127 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) Open Application Platform Authors - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package oap.io; - -import lombok.SneakyThrows; - -import java.io.IOException; -import java.net.URI; -import java.nio.file.Path; -import java.util.ArrayList; -import java.util.HashSet; -import java.util.Optional; -import java.util.ServiceLoader; -import java.util.Set; - -import static java.util.Arrays.asList; - -public abstract class AbstractFileSync implements Runnable { - private final HashSet protocols; - private final ArrayList listeners = new ArrayList<>(); - protected URI uri; - protected Path localFile; - - protected AbstractFileSync( String... protocols ) { - this.protocols = new HashSet<>( asList( protocols ) ); - } - - @SneakyThrows - public static AbstractFileSync create( String url, Path localFile ) { - return create( new URI( url ), localFile ); - } - - @SneakyThrows - public static AbstractFileSync create( URI uri, Path localFile ) { - var protocol = uri.getScheme(); - - final ServiceLoader load = ServiceLoader.load( AbstractFileSync.class ); - for( AbstractFileSync fs : load ) { - if( fs.accept( protocol ) ) { - fs.init( uri, localFile ); - return fs; - } - } - - throw new IOException( "unknown protocol: " + protocol ); - } - - public Set getProtocols() { - return protocols; - } - - protected boolean accept( String protocol ) { - return protocols.contains( protocol ); - } - - void init( URI uri, Path localFile ) { - this.uri = uri; - this.localFile = localFile; - } - - public URI getUri() { - return uri; - } - - public Path getLocalFile() { - return localFile; - } - - protected void fireDownloaded( Path path ) { - this.listeners.forEach( l -> l.downloaded( path ) ); - } - - protected void fireNotModified() { - this.listeners.forEach( FileDownloaderListener::notModified ); - } - - public void addListener( FileDownloaderListener listener ) { - this.listeners.add( listener ); - } - - public void removeListener( FileDownloaderListener listener ) { - this.listeners.remove( listener ); - } - - @Override - public synchronized void run() { - final Optional downloaded = download(); - - if( downloaded.isPresent() ) { - final Path path = downloaded.get(); - - fireDownloaded( path ); - } else - fireNotModified(); - } - - protected abstract Optional download(); - - public interface FileDownloaderListener { - void downloaded( Path path ); - - default void notModified() { - } - } -} diff --git a/oap-stdlib/src/main/java/oap/io/FileSystemFileSync.java b/oap-stdlib/src/main/java/oap/io/FileSystemFileSync.java deleted file mode 100644 index 0520f5a747..0000000000 --- a/oap-stdlib/src/main/java/oap/io/FileSystemFileSync.java +++ /dev/null @@ -1,63 +0,0 @@ -/* - * The MIT License (MIT) - * - * Copyright (c) Open Application Platform Authors - * - * Permission is hereby granted, free of charge, to any person obtaining a copy - * of this software and associated documentation files (the "Software"), to deal - * in the Software without restriction, including without limitation the rights - * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell - * copies of the Software, and to permit persons to whom the Software is - * furnished to do so, subject to the following conditions: - * - * The above copyright notice and this permission notice shall be included in all - * copies or substantial portions of the Software. - * - * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR - * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, - * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE - * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER - * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, - * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE - * SOFTWARE. - */ - -package oap.io; - -import lombok.SneakyThrows; -import lombok.extern.slf4j.Slf4j; - -import java.io.InputStream; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.Optional; - -@Slf4j -public class FileSystemFileSync extends AbstractFileSync { - public FileSystemFileSync() { - super( "file" ); - } - - @Override - @SneakyThrows - protected Optional download() { - var remoteFile = Paths.get( uri ); - var localFile = this.localFile.toAbsolutePath(); - - var remoteFileLastModifiedTime = java.nio.file.Files.getLastModifiedTime( remoteFile ).toMillis(); - var localFileLastModifiedTime = Files.exists( localFile ) - ? java.nio.file.Files.getLastModifiedTime( localFile ).toMillis() - : Long.MIN_VALUE; - - if( remoteFileLastModifiedTime > localFileLastModifiedTime ) { - try( InputStream in = IoStreams.in( uri.toURL().openStream(), IoStreams.Encoding.PLAIN ) ) { - IoStreams.write( localFile, IoStreams.Encoding.PLAIN, in, false, true ); - } - - oap.io.Files.setLastModifiedTime( localFile, remoteFileLastModifiedTime ); - - return Optional.of( localFile ); - } else return Optional.empty(); - } -} diff --git a/oap-stdlib/src/main/java/oap/json/Binder.java b/oap-stdlib/src/main/java/oap/json/Binder.java index b56bcac876..4274099177 100644 --- a/oap-stdlib/src/main/java/oap/json/Binder.java +++ b/oap-stdlib/src/main/java/oap/json/Binder.java @@ -328,13 +328,13 @@ public String marshal( Object value, boolean prettyPrinter ) { } public void marshal( Object value, StringBuilder sb ) { - try( var writer = new StringBuilderWriter( sb ) ) { + try( StringBuilderWriter writer = new StringBuilderWriter( sb ) ) { marshal( value, writer, false ); } } public void marshal( Object value, StringBuilder sb, boolean prettyPrinter ) { - try( var writer = new StringBuilderWriter( sb ) ) { + try( StringBuilderWriter writer = new StringBuilderWriter( sb ) ) { marshal( value, writer, prettyPrinter ); } } @@ -345,9 +345,13 @@ public void marshal( Object value, Writer writer ) { public void marshal( Object value, Writer writer, boolean prettyPrinter ) { try { - if( prettyPrinter ) mapper.writerWithDefaultPrettyPrinter().writeValue( writer, value ); - else + if( prettyPrinter ) { + DefaultPrettyPrinter defaultPrettyPrinter = new DefaultPrettyPrinter(); + defaultPrettyPrinter.indentArraysWith( DefaultIndenter.SYSTEM_LINEFEED_INSTANCE ); + mapper.writer( defaultPrettyPrinter ).writeValue( writer, value ); + } else { mapper.writeValue( writer, value ); + } } catch( IOException e ) { throw new JsonException( e ); } diff --git a/oap-stdlib/src/main/resources/META-INF/services/oap.io.AbstractFileSync b/oap-stdlib/src/main/resources/META-INF/services/oap.io.AbstractFileSync deleted file mode 100644 index ab6f2cad58..0000000000 --- a/oap-stdlib/src/main/resources/META-INF/services/oap.io.AbstractFileSync +++ /dev/null @@ -1,2 +0,0 @@ -oap.io.FileSystemFileSync -oap.http.file.HttpFileSync diff --git a/oap-ws/oap-ws-sso-api/src/main/java/oap/ws/sso/Authentication.java b/oap-ws/oap-ws-sso-api/src/main/java/oap/ws/sso/Authentication.java index 8e31fa33e7..6051ffdc07 100644 --- a/oap-ws/oap-ws-sso-api/src/main/java/oap/ws/sso/Authentication.java +++ b/oap-ws/oap-ws-sso-api/src/main/java/oap/ws/sso/Authentication.java @@ -33,6 +33,8 @@ import java.io.Serializable; import java.util.Date; +import static org.joda.time.DateTimeZone.UTC; + @ToString @EqualsAndHashCode public class Authentication implements Serializable { @@ -47,7 +49,7 @@ public Authentication( Token accessToken, Token refreshToken, UserView user ) { this.accessToken = accessToken; this.refreshToken = refreshToken; this.user = user; - this.created = new DateTime(); + this.created = new DateTime( System.currentTimeMillis(), UTC ); } @ToString diff --git a/oap-ws/oap-ws-sso-api/src/main/java/oap/ws/sso/JwtTokenGenerator.java b/oap-ws/oap-ws-sso-api/src/main/java/oap/ws/sso/JwtTokenGenerator.java index d89b9eb7b5..384bd7e261 100644 --- a/oap-ws/oap-ws-sso-api/src/main/java/oap/ws/sso/JwtTokenGenerator.java +++ b/oap-ws/oap-ws-sso-api/src/main/java/oap/ws/sso/JwtTokenGenerator.java @@ -27,7 +27,6 @@ import com.auth0.jwt.JWT; import com.auth0.jwt.algorithms.Algorithm; import com.auth0.jwt.exceptions.JWTCreationException; -import org.joda.time.DateTimeUtils; import java.util.Date; @@ -51,7 +50,7 @@ public JwtTokenGenerator( String accessSecret, String refreshSecret, String issu public Authentication.Token generateAccessToken( User user ) throws JWTCreationException { Algorithm algorithm = Algorithm.HMAC256( accessSecret ); - Date expiresAt = new org.joda.time.DateTime( DateTimeUtils.currentTimeMillis() + accessSecretExpiration, UTC ).toDate(); + Date expiresAt = new org.joda.time.DateTime( System.currentTimeMillis() + accessSecretExpiration, UTC ).toDate(); return new Authentication.Token( expiresAt, JWT.create() .withClaim( "id", user.getId() ) .withClaim( "user", user.getEmail() ) @@ -64,7 +63,7 @@ public Authentication.Token generateAccessToken( User user ) throws JWTCreationE public Authentication.Token generateAccessTokenWithActiveOrgId( User user, String activeOrganization ) throws JWTCreationException { Algorithm algorithm = Algorithm.HMAC256( accessSecret ); - Date expiresAt = new org.joda.time.DateTime( DateTimeUtils.currentTimeMillis() + accessSecretExpiration, UTC ).toDate(); + Date expiresAt = new org.joda.time.DateTime( System.currentTimeMillis() + accessSecretExpiration, UTC ).toDate(); return new Authentication.Token( expiresAt, JWT.create() .withClaim( "id", user.getId() ) .withClaim( "user", user.getEmail() ) @@ -78,7 +77,7 @@ public Authentication.Token generateAccessTokenWithActiveOrgId( User user, Strin public Authentication.Token generateRefreshToken( User user ) throws JWTCreationException { Algorithm algorithm = Algorithm.HMAC256( refreshSecret ); - Date expiresAt = new org.joda.time.DateTime( DateTimeUtils.currentTimeMillis() + refreshSecretExpiration, UTC ).toDate(); + Date expiresAt = new org.joda.time.DateTime( System.currentTimeMillis() + refreshSecretExpiration, UTC ).toDate(); return new Authentication.Token( expiresAt, JWT.create() .withClaim( "id", user.getId() ) .withClaim( "user", user.getEmail() ) diff --git a/oap-ws/oap-ws-sso-api/src/main/java/oap/ws/sso/SSO.java b/oap-ws/oap-ws-sso-api/src/main/java/oap/ws/sso/SSO.java index 55f3b314c1..e9995af2bb 100644 --- a/oap-ws/oap-ws-sso-api/src/main/java/oap/ws/sso/SSO.java +++ b/oap-ws/oap-ws-sso-api/src/main/java/oap/ws/sso/SSO.java @@ -31,7 +31,6 @@ import oap.ws.Response; import oap.ws.SessionManager; import org.joda.time.DateTime; -import org.joda.time.Duration; import javax.annotation.Nonnull; import javax.annotation.Nullable; @@ -66,16 +65,13 @@ public static Optional getRefreshAuthentication( HttpServerExchange exch } public static Response authenticatedResponse( Authentication authentication, String cookieDomain, Boolean cookieSecure ) { - long accessTokenMaxAge = new Duration( new DateTime( UTC ), new DateTime( authentication.accessToken.expires, UTC ) ).getStandardSeconds(); - long refreshTokenMaxAge = new Duration( new DateTime( UTC ), new DateTime( authentication.refreshToken.expires, UTC ) ).getStandardSeconds(); - return Response .jsonOk() .withHeader( AUTHENTICATION_KEY, authentication.accessToken.jwt ) .withCookie( Cookie.builder( AUTHENTICATION_KEY, authentication.accessToken.jwt ) .withDomain( cookieDomain ) .withPath( "/" ) - .withMaxAge( ( int ) accessTokenMaxAge ) + .withExpires( new DateTime( authentication.accessToken.expires, UTC ) ) .withHttpOnly( true ) .withSecure( cookieSecure ) .build() @@ -83,7 +79,7 @@ public static Response authenticatedResponse( Authentication authentication, Str .withCookie( Cookie.builder( REFRESH_TOKEN_KEY, authentication.refreshToken.jwt ) .withDomain( cookieDomain ) .withPath( "/" ) - .withMaxAge( ( int ) refreshTokenMaxAge ) + .withExpires( new DateTime( authentication.refreshToken.expires, UTC ) ) .withHttpOnly( true ) .withSecure( cookieSecure ) .build() @@ -106,19 +102,19 @@ private static DateTime getExpirationTimeCookie( Date expirationInToken, Date co public static Response logoutResponse( String cookieDomain ) { return Response .noContent() - .withCookie( Cookie.builder( AUTHENTICATION_KEY, "" ) + .withCookie( Cookie.builder( AUTHENTICATION_KEY, "" ) .withDomain( cookieDomain ) .withPath( "/" ) .withExpires( new DateTime( 1970, 1, 1, 1, 1, UTC ) ) .build() ) - .withCookie( Cookie.builder( REFRESH_TOKEN_KEY, "" ) + .withCookie( Cookie.builder( REFRESH_TOKEN_KEY, "" ) .withDomain( cookieDomain ) .withPath( "/" ) .withExpires( new DateTime( 1970, 1, 1, 1, 1, UTC ) ) .build() ) - .withCookie( Cookie.builder( SessionManager.COOKIE_ID, "" ) + .withCookie( Cookie.builder( SessionManager.COOKIE_ID, "" ) .withDomain( cookieDomain ) .withPath( "/" ) .withExpires( new DateTime( 1970, 1, 1, 1, 1, UTC ) ) @@ -128,7 +124,7 @@ public static Response logoutResponse( String cookieDomain ) { public static Response notAuthenticatedResponse( int code, String reasonPhrase, String cookieDomain ) { return new Response( code, reasonPhrase ) - .withCookie( Cookie.builder( AUTHENTICATION_KEY, "" ) + .withCookie( Cookie.builder( AUTHENTICATION_KEY, "" ) .withDomain( cookieDomain ) .withPath( "/" ) .withExpires( new DateTime( 1970, 1, 1, 1, 1, UTC ) ) diff --git a/oap-ws/oap-ws-sso-api/src/test/java/oap/ws/sso/JwtTokenGeneratorExtractorTest.java b/oap-ws/oap-ws-sso-api/src/test/java/oap/ws/sso/JwtTokenGeneratorExtractorTest.java index 750cedb3a1..5f8c0f4990 100644 --- a/oap-ws/oap-ws-sso-api/src/test/java/oap/ws/sso/JwtTokenGeneratorExtractorTest.java +++ b/oap-ws/oap-ws-sso-api/src/test/java/oap/ws/sso/JwtTokenGeneratorExtractorTest.java @@ -30,7 +30,6 @@ import oap.ws.sso.AbstractUserTest.TestSecurityRolesProvider; import oap.ws.sso.AbstractUserTest.TestUser; import org.joda.time.DateTime; -import org.joda.time.DateTimeUtils; import org.testng.annotations.Test; import static oap.ws.sso.JWTExtractor.TokenStatus.VALID; @@ -49,12 +48,12 @@ public JwtTokenGeneratorExtractorTest() { @Test public void generateAndExtractToken() { - DateTimeUtils.setCurrentMillisFixed( DateTimeUtils.currentTimeMillis() ); - Authentication.Token token = jwtTokenGenerator.generateAccessToken( new TestUser( "email@email.com", "password", Pair.of( "org1", "ADMIN" ) ) ); assertNotNull( token.expires ); assertThat( token.jwt ).isNotEmpty(); - assertThat( token.expires ).isEqualTo( new DateTime( UTC ).plusMinutes( 15 ).toDate() ); + assertThat( token.expires ) + .isBeforeOrEqualTo( new DateTime( UTC ).plusMinutes( 15 ).toDate() ) + .isAfterOrEqualTo( new DateTime( UTC ).plusMinutes( 14 ).toDate() ); assertThat( jwtExtractor.verifyToken( token.jwt ) ).isEqualTo( VALID ); JwtToken jwtToken = jwtExtractor.decodeJWT( token.jwt ); diff --git a/oap-ws/oap-ws-test/src/main/java/oap/ws/validate/testng/ValidationAssertion.java b/oap-ws/oap-ws-test/src/main/java/oap/ws/validate/testng/ValidationAssertion.java index b684158c45..cfd016fd3e 100644 --- a/oap-ws/oap-ws-test/src/main/java/oap/ws/validate/testng/ValidationAssertion.java +++ b/oap-ws/oap-ws-test/src/main/java/oap/ws/validate/testng/ValidationAssertion.java @@ -24,7 +24,7 @@ package oap.ws.validate.testng; -import oap.http.Client; +import oap.http.Response; import oap.json.Binder; import oap.ws.validate.ValidationErrors; @@ -34,11 +34,11 @@ public final class ValidationAssertion { private final ValidationErrors errors; - private ValidationAssertion( Client.Response response ) { + private ValidationAssertion( Response response ) { errors = Binder.json.unmarshal( ValidationErrors.class, response.contentString() ); } - public static ValidationAssertion assertValidation( Client.Response response ) { + public static ValidationAssertion assertValidation( Response response ) { assertJsonResponse( response ); return new ValidationAssertion( response ); } diff --git a/oap-ws/oap-ws-test/src/test/java/oap/ws/WebServicesSessionTest.java b/oap-ws/oap-ws-test/src/test/java/oap/ws/WebServicesSessionTest.java index 368b2bf58b..b1432f4915 100644 --- a/oap-ws/oap-ws-test/src/test/java/oap/ws/WebServicesSessionTest.java +++ b/oap-ws/oap-ws-test/src/test/java/oap/ws/WebServicesSessionTest.java @@ -26,7 +26,6 @@ import oap.application.testng.KernelFixture; import oap.http.Http; -import oap.http.test.HttpAsserts; import oap.testng.Fixtures; import oap.testng.TestDirectoryFixture; import org.testng.annotations.Test; @@ -50,7 +49,7 @@ public WebServicesSessionTest() { public void sessionViaResponse() { assertGet( kernel.httpUrl( "/session/put" ), Map.of( "value", "vvv" ), Map.of() ) .hasCode( Http.StatusCode.NO_CONTENT ); - HttpAsserts.assertGet( kernel.httpUrl( "/session/get" ) ) + assertGet( kernel.httpUrl( "/session/get" ) ) .isOk() .hasBody( "vvv" ); } @@ -59,7 +58,7 @@ public void sessionViaResponse() { public void sessionDirectly() { assertGet( kernel.httpUrl( "/session/putDirectly" ), Map.of( "value", "vvv" ), Map.of() ) .hasCode( Http.StatusCode.NO_CONTENT ); - HttpAsserts.assertGet( kernel.httpUrl( "/session/get" ) ) + assertGet( kernel.httpUrl( "/session/get" ) ) .isOk() .hasBody( "vvv" ); } @@ -68,7 +67,7 @@ public void sessionDirectly() { public void respondHtmlContentType() { assertGet( kernel.httpUrl( "/session/putDirectly" ), Map.of( "value", "vvv" ), Map.of() ) .hasCode( Http.StatusCode.NO_CONTENT ); - HttpAsserts.assertGet( kernel.httpUrl( "/session/html" ) ) + assertGet( kernel.httpUrl( "/session/html" ) ) .isOk() .hasBody( "vvv" ) .hasContentType( Http.ContentType.TEXT_HTML ); diff --git a/oap-ws/oap-ws-test/src/test/java/oap/ws/WebServicesTest.java b/oap-ws/oap-ws-test/src/test/java/oap/ws/WebServicesTest.java index caef53d5a3..ec095e8eb5 100644 --- a/oap-ws/oap-ws-test/src/test/java/oap/ws/WebServicesTest.java +++ b/oap-ws/oap-ws-test/src/test/java/oap/ws/WebServicesTest.java @@ -25,7 +25,6 @@ import lombok.extern.slf4j.Slf4j; import oap.application.testng.KernelFixture; -import oap.http.Client; import oap.http.Http; import oap.http.server.nio.HttpHandler; import oap.http.server.nio.HttpServerExchange; @@ -51,6 +50,7 @@ import static oap.http.Http.StatusCode.OK; import static oap.http.server.nio.HttpServerExchange.HttpMethod.GET; import static oap.http.test.HttpAsserts.assertGet; +import static oap.http.test.HttpAsserts.assertPost; import static oap.io.Resources.urlOrThrow; import static oap.util.Pair.__; import static oap.ws.WsParam.From.BODY; @@ -143,7 +143,7 @@ public void invocationString() { .responded( OK, "OK", APPLICATION_JSON, "\"1234\"" ); HttpAsserts.assertPost( kernel.httpUrl( "/x/v/math/string" ), "1234", Http.ContentType.APPLICATION_OCTET_STREAM ) .satisfies( response -> assertThat( response.headers ) - .contains( __( "content-type", "application/json" ) ) ); + .contains( __( "Content-Type", "application/json" ) ) ); } @Test @@ -201,15 +201,10 @@ public void shouldVerifyGZIPRequestProcessing() throws Exception { gzip.write( "{\"i\":1,\"s\":\"sss\"}".getBytes( StandardCharsets.UTF_8 ) ); gzip.close(); - Client.Response response = Client - .custom() - .build() - .post( kernel.httpUrl( "/x/v/math/json" ), - new ByteArrayInputStream( byteArrayOutputStream.toByteArray() ), - APPLICATION_JSON, Map.of( "Content-Encoding", "gzip" ) ); - - assertThat( response.code ).isEqualTo( OK ); - assertThat( response.contentString() ).isEqualTo( "{\"i\":1,\"s\":\"sss\"}" ); + assertPost( kernel.httpUrl( "/x/v/math/json" ), new ByteArrayInputStream( byteArrayOutputStream.toByteArray() ), APPLICATION_JSON, Map.of( "Content-Encoding", "gzip" ) ) + .isOk() + .respondedJson( """ + { "i":1, "s":"sss" }""" ); } /** diff --git a/oap-ws/oap-ws/src/main/java/oap/ws/WebService.java b/oap-ws/oap-ws/src/main/java/oap/ws/WebService.java index 2dce5f731d..289b1b4175 100644 --- a/oap-ws/oap-ws/src/main/java/oap/ws/WebService.java +++ b/oap-ws/oap-ws/src/main/java/oap/ws/WebService.java @@ -23,7 +23,6 @@ */ package oap.ws; -import io.undertow.server.handlers.Cookie; import lombok.extern.slf4j.Slf4j; import oap.http.Http; import oap.http.server.nio.HttpHandler; @@ -44,7 +43,8 @@ import java.util.List; import java.util.Map; import java.util.Optional; -import java.util.concurrent.CompletableFuture; + +import static org.joda.time.DateTimeZone.UTC; @Slf4j public class WebService implements HttpHandler { @@ -163,54 +163,51 @@ private void handleInternal( InvocationContext context ) { return; } - if( context.session != null && !containsSessionCookie( context.exchange.responseCookies() ) ) { + Response response = produceResultResponse( context.method, wsMethod, context.method.invoke( instance, paramValues ) ); + + if( context.session != null && !containsSessionCookie( response.cookies ) ) { oap.http.Cookie cookie = oap.http.Cookie.builder( SessionManager.COOKIE_ID, context.session.id ) .withPath( sessionManager.cookiePath ) - .withExpires( DateTime.now().plus( sessionManager.cookieExpiration ) ) + .withExpires( new DateTime( System.currentTimeMillis(), UTC ).plus( sessionManager.cookieExpiration ) ) .withDomain( sessionManager.cookieDomain ) .withSecure( sessionManager.cookieSecure ) .withHttpOnly( true ) .build(); - context.exchange.setResponseCookie( cookie ); + response.cookies.add( cookie ); } - CompletableFuture responseFuture = produceResultResponse( context.method, wsMethod, context.method.invoke( instance, paramValues ) ); - responseFuture.thenAccept( response -> { - Interceptors.after( interceptors, response, context ); - response.send( context.exchange ); - } ); + Interceptors.after( interceptors, response, context ); + response.send( context.exchange ); } ); } - private boolean containsSessionCookie( Iterable cookies ) { - for( Cookie p : cookies ) { + private boolean containsSessionCookie( Iterable cookies ) { + for( oap.http.Cookie p : cookies ) { if( SessionManager.COOKIE_ID.equals( p.getName() ) ) return true; } return false; } - private CompletableFuture produceResultResponse( Reflection.Method method, Optional wsMethod, Object result ) { + private Response produceResultResponse( Reflection.Method method, Optional wsMethod, Object result ) { boolean isRaw = wsMethod.map( WsMethod::raw ).orElse( false ); String produces = wsMethod.map( WsMethod::produces ) .orElse( Http.ContentType.APPLICATION_JSON ); if( method.isVoid() ) { - return CompletableFuture.completedFuture( Response.noContent() ); - } else if( result instanceof Response response ) return CompletableFuture.completedFuture( response ); - else if( result instanceof Optional optResult ) return CompletableFuture.completedFuture( optResult.isEmpty() + return Response.noContent(); + } else if( result instanceof Response response ) return response; + else if( result instanceof Optional optResult ) return optResult.isEmpty() ? Response.notFound() - : Response.ok().withBody( optResult.get(), isRaw ).withContentType( produces ) ); + : Response.ok().withBody( optResult.get(), isRaw ).withContentType( produces ); else if( result instanceof Result resultResult ) if( resultResult.isSuccess() ) - return CompletableFuture.completedFuture( Response.ok().withBody( resultResult.successValue, isRaw ).withContentType( produces ) ); - else return CompletableFuture.completedFuture( new Response( Http.StatusCode.INTERNAL_SERVER_ERROR, "" ) + return Response.ok().withBody( resultResult.successValue, isRaw ).withContentType( produces ); + else return new Response( Http.StatusCode.INTERNAL_SERVER_ERROR, "" ) .withBody( resultResult.failureValue, false ) - .withContentType( Http.ContentType.APPLICATION_JSON ) ); + .withContentType( Http.ContentType.APPLICATION_JSON ); else if( result instanceof java.util.stream.Stream stream ) { - return CompletableFuture.completedFuture( Response.ok().withBody( stream, isRaw ).withContentType( produces ) ); - } else if( result instanceof CompletableFuture future ) { - return future.thenCompose( fResult -> produceResultResponse( method, wsMethod, fResult ) ); - } else return CompletableFuture.completedFuture( Response.ok().withBody( result, isRaw ).withContentType( produces ) ); + return Response.ok().withBody( stream, isRaw ).withContentType( produces ); + } else return Response.ok().withBody( result, isRaw ).withContentType( produces ); } @Override diff --git a/oap-ws/oap-ws/src/main/java/oap/ws/interceptor/Interceptors.java b/oap-ws/oap-ws/src/main/java/oap/ws/interceptor/Interceptors.java index a617427999..bdf754db97 100644 --- a/oap-ws/oap-ws/src/main/java/oap/ws/interceptor/Interceptors.java +++ b/oap-ws/oap-ws/src/main/java/oap/ws/interceptor/Interceptors.java @@ -34,17 +34,17 @@ @Slf4j public class Interceptors { public static Optional before( List interceptors, InvocationContext context ) { - for( var interceptor : interceptors ) { + for( Interceptor interceptor : interceptors ) { log.trace( "running before call {}", interceptor.getClass().getSimpleName() ); - var response = interceptor.before( context ); + Optional response = interceptor.before( context ); if( response.isPresent() ) return response; } return Optional.empty(); } public static void after( List interceptors, Response response, InvocationContext context ) { - for( var i = interceptors.size() - 1; i >= 0; i-- ) { - var interceptor = interceptors.get( i ); + for( int i = interceptors.size() - 1; i >= 0; i-- ) { + Interceptor interceptor = interceptors.get( i ); log.trace( "running after call {}", interceptor.getClass().getSimpleName() ); interceptor.after( response, context ); } diff --git a/pom.xml b/pom.xml index e27d0008e6..dce352c45e 100644 --- a/pom.xml +++ b/pom.xml @@ -23,14 +23,6 @@ pom import - - - com.squareup.okhttp3 - okhttp-bom - ${oap.deps.okhttp.version} - pom - import - @@ -65,7 +57,7 @@ - 25.3.3 + 25.4.0 25.0.1 25.0.0 @@ -74,8 +66,6 @@ 5.18.0 1.17.6 - 5.3.2 - 4.4.16 4.5.14 4.1.5 @@ -109,7 +99,7 @@ 4.13.0 9.8 - 2.3.19.Final + 2.3.23.Final 3.23.3 3.3.4 @@ -122,6 +112,9 @@ 2.3 4.9.3 + 3.6.4 + 12.1.6 + 4.9.8