From 29a503ee917d02c7bc34a060dc49b6114f67fbe3 Mon Sep 17 00:00:00 2001 From: jowerner <4639399+jowerner@users.noreply.github.com> Date: Thu, 29 Jul 2021 20:07:15 +0200 Subject: [PATCH] initial commit --- pom.xml | 15 +- .../com/gargoylesoftware/htmlunit/Cache.java | 2 +- .../htmlunit/CookieManager.java | 2 +- .../htmlunit/DefaultCredentialsProvider.java | 39 +- .../htmlunit/HttpWebConnection.java | 667 ++++++++++-------- .../htmlunit/NetscapeDraftHeaderParser.java | 119 ++++ .../htmlunit/StringWebResponse.java | 2 +- .../gargoylesoftware/htmlunit/WebClient.java | 45 +- .../gargoylesoftware/htmlunit/WebRequest.java | 8 +- .../javascript/msxml/XMLHTTPRequest.java | 4 +- .../htmlunit/html/HtmlForm.java | 2 +- .../htmlunit/html/HtmlImage.java | 2 +- .../htmlunit/html/HtmlLink.java | 2 +- .../htmlunit/html/HtmlPage.java | 2 +- ...owserCompatCookieHeaderValueFormatter.java | 95 ++- .../HtmlUnitBrowserCompatCookieSpec.java | 88 +-- .../HtmlUnitCookieSpecProvider.java | 8 +- .../httpclient/HtmlUnitCookieStore.java | 9 +- .../httpclient/HtmlUnitDomainHandler.java | 14 +- .../httpclient/HtmlUnitExpiresHandler.java | 8 +- .../httpclient/HtmlUnitHttpOnlyHandler.java | 14 +- .../httpclient/HtmlUnitPathHandler.java | 8 +- .../httpclient/HtmlUnitRedirectStrategie.java | 10 +- .../HtmlUnitSSLConnectionSocketFactory.java | 20 +- .../httpclient/HtmlUnitSameSiteHandler.java | 13 +- .../HtmlUnitVersionAttributeHandler.java | 16 +- .../SocksConnectionSocketFactory.java | 8 +- .../javascript/host/URLSearchParams.java | 5 +- .../javascript/host/xml/XMLHttpRequest.java | 8 +- .../htmlunit/util/Cookie.java | 21 +- .../htmlunit/util/NameValuePair.java | 14 +- .../htmlunit/util/StringUtils.java | 2 +- .../htmlunit/AbstractPageTest.java | 2 +- .../gargoylesoftware/htmlunit/CacheTest.java | 2 +- .../htmlunit/CookieManager3Test.java | 15 +- .../htmlunit/CookieManager5Test.java | 2 +- .../htmlunit/CookieManagerTest.java | 2 +- .../DefaultCredentialsProvider2Test.java | 16 +- .../DefaultCredentialsProvider3Test.java | 21 +- .../DefaultCredentialsProviderTest.java | 40 +- .../FailingHttpStatusCodeExceptionTest.java | 2 +- .../htmlunit/HtmlUnitCookieStoreTest.java | 46 +- .../htmlunit/HttpWebConnection3Test.java | 22 +- .../htmlunit/HttpWebConnectionTest.java | 31 +- .../htmlunit/SgmlPageTest.java | 2 +- .../htmlunit/WebClientTest.java | 2 +- .../htmlunit/WebRequestTest.java | 18 +- .../htmlunit/WebResponseDataTest.java | 2 +- .../htmlunit/WebResponseTest.java | 2 +- .../htmlunit/html/HtmlFileInput2Test.java | 21 +- .../htmlunit/html/HtmlScript2Test.java | 2 +- .../htmlunit/html/HtmlScriptTest.java | 2 +- .../htmlunit/html/XHtmlPageTest.java | 2 +- .../host/html/HTMLDocumentTest.java | 2 +- .../host/xml/XMLHttpRequestLifeCycleTest.java | 2 +- .../util/WebConnectionWrapperTest.java | 2 +- .../htmlunit/xml/XmlPageTest.java | 2 +- 57 files changed, 921 insertions(+), 613 deletions(-) create mode 100644 src/main/java/com/gargoylesoftware/htmlunit/NetscapeDraftHeaderParser.java diff --git a/pom.xml b/pom.xml index 63e8f7520b5..75bd3fe6549 100644 --- a/pom.xml +++ b/pom.xml @@ -29,7 +29,7 @@ 2.52.0 2.52.0 - 4.5.13 + 5.1 9.4.43.v20210629 2.14.1 3.141.59 @@ -1197,10 +1197,9 @@ - - org.apache.httpcomponents - httpmime + org.apache.httpcomponents.client5 + httpclient5 ${httpcomponents.version} @@ -1301,10 +1300,16 @@ org.apache.httpcomponents httpclient - ${httpcomponents.version} + 4.5.13 test-jar test + + org.apache.httpcomponents + httpclient + 4.5.13 + test + diff --git a/src/main/java/com/gargoylesoftware/htmlunit/Cache.java b/src/main/java/com/gargoylesoftware/htmlunit/Cache.java index d794a2803ea..f7c9da918a9 100644 --- a/src/main/java/com/gargoylesoftware/htmlunit/Cache.java +++ b/src/main/java/com/gargoylesoftware/htmlunit/Cache.java @@ -24,7 +24,7 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; -import org.apache.http.client.utils.DateUtils; +import org.apache.hc.client5.http.utils.DateUtils; import com.gargoylesoftware.css.dom.CSSStyleSheetImpl; import com.gargoylesoftware.htmlunit.util.HeaderUtils; diff --git a/src/main/java/com/gargoylesoftware/htmlunit/CookieManager.java b/src/main/java/com/gargoylesoftware/htmlunit/CookieManager.java index 2a9e045a952..388e5f3b3a1 100644 --- a/src/main/java/com/gargoylesoftware/htmlunit/CookieManager.java +++ b/src/main/java/com/gargoylesoftware/htmlunit/CookieManager.java @@ -24,7 +24,7 @@ import java.util.Set; import org.apache.commons.lang3.StringUtils; -import org.apache.http.cookie.CookieOrigin; +import org.apache.hc.client5.http.cookie.CookieOrigin; import com.gargoylesoftware.htmlunit.httpclient.HtmlUnitBrowserCompatCookieSpec; import com.gargoylesoftware.htmlunit.util.Cookie; diff --git a/src/main/java/com/gargoylesoftware/htmlunit/DefaultCredentialsProvider.java b/src/main/java/com/gargoylesoftware/htmlunit/DefaultCredentialsProvider.java index da42a1cefaf..0224845f350 100644 --- a/src/main/java/com/gargoylesoftware/htmlunit/DefaultCredentialsProvider.java +++ b/src/main/java/com/gargoylesoftware/htmlunit/DefaultCredentialsProvider.java @@ -21,11 +21,12 @@ import java.util.HashMap; import java.util.Map; -import org.apache.http.auth.AuthScope; -import org.apache.http.auth.Credentials; -import org.apache.http.auth.NTCredentials; -import org.apache.http.auth.UsernamePasswordCredentials; -import org.apache.http.client.CredentialsProvider; +import org.apache.hc.client5.http.auth.AuthScope; +import org.apache.hc.client5.http.auth.Credentials; +import org.apache.hc.client5.http.auth.CredentialsStore; +import org.apache.hc.client5.http.auth.NTCredentials; +import org.apache.hc.client5.http.auth.UsernamePasswordCredentials; +import org.apache.hc.core5.http.protocol.HttpContext; /** * Default HtmlUnit implementation of the CredentialsProvider interface. Provides @@ -38,7 +39,13 @@ * @author Ahmed Ashour * @author Nicolas Belisle */ -public class DefaultCredentialsProvider implements CredentialsProvider, Serializable { +public class DefaultCredentialsProvider implements CredentialsStore, Serializable { + + public static final String ANY_PROTOCOL = null; + public static final String ANY_HOST = null; + public static final int ANY_PORT = -1; + public static final String ANY_REALM = null; + public static final String ANY_SCHEME = null; private final Map credentialsMap_ = new HashMap<>(); @@ -51,8 +58,8 @@ public class DefaultCredentialsProvider implements CredentialsProvider, Serializ * @param username the username for the new credentials * @param password the password for the new credentials */ - public void addCredentials(final String username, final String password) { - addCredentials(username, password, AuthScope.ANY_HOST, AuthScope.ANY_PORT, AuthScope.ANY_REALM); + public void addCredentials(final String username, final char[] password) { + addCredentials(username, password, ANY_HOST, ANY_PORT, ANY_REALM); } /** @@ -65,9 +72,9 @@ public void addCredentials(final String username, final String password) { * @param port the port to which to the new credentials apply (negative if applicable to any port) * @param realm the realm to which to the new credentials apply ({@code null} if applicable to any realm) */ - public void addCredentials(final String username, final String password, final String host, + public void addCredentials(final String username, final char[] password, final String host, final int port, final String realm) { - final AuthScope authscope = new AuthScope(host, port, realm, AuthScope.ANY_SCHEME); + final AuthScope authscope = new AuthScope(ANY_PROTOCOL, host, port, realm, ANY_SCHEME); final Credentials credentials = new UsernamePasswordCredentials(username, password); setCredentials(authscope, credentials); } @@ -83,9 +90,9 @@ public void addCredentials(final String username, final String password, final S * Essentially, the computer name for this machine. * @param domain the domain to authenticate within */ - public void addNTLMCredentials(final String username, final String password, final String host, + public void addNTLMCredentials(final String username, final char[] password, final String host, final int port, final String workstation, final String domain) { - final AuthScope authscope = new AuthScope(host, port, AuthScope.ANY_REALM, AuthScope.ANY_SCHEME); + final AuthScope authscope = new AuthScope(ANY_PROTOCOL, host, port, ANY_REALM, ANY_SCHEME); final Credentials credentials = new NTCredentials(username, password, workstation, domain); setCredentials(authscope, credentials); } @@ -138,7 +145,7 @@ private static Credentials matchCredentials(final Map httpClientBuilder_ = new WeakHashMap<>(); + private final Map httpClientBuilder_ = new WeakHashMap<>(); private final WebClient webClient_; private String virtualHost_; - private final CookieSpecProvider htmlUnitCookieSpecProvider_; + private final CookieSpecFactory htmlUnitCookieSpecProvider_; private final WebClientOptions usedOptions_; - private PoolingHttpClientConnectionManager connectionManager_; + private PoolingAsyncClientConnectionManager connectionManager_; /** Authentication cache shared among all threads of a web client. */ private final AuthCache sharedAuthCache_ = new SynchronizedAuthCache(); @@ -169,9 +167,9 @@ public HttpWebConnection(final WebClient webClient) { */ @Override public WebResponse getResponse(final WebRequest webRequest) throws IOException { - final HttpClientBuilder builder = reconfigureHttpClientIfNeeded(getHttpClientBuilder(), webRequest); + final HttpAsyncClientBuilder builder = reconfigureHttpClientIfNeeded(getHttpClientBuilder(), webRequest); - HttpUriRequest httpMethod = null; + SimpleHttpRequest httpMethod = null; try { try { httpMethod = makeHttpMethod(webRequest, builder); @@ -181,23 +179,22 @@ public WebResponse getResponse(final WebRequest webRequest) throws IOException { + " (reason: " + e.getMessage() + ")", e); } - final URL url = webRequest.getUrl(); - final HttpHost httpHost = new HttpHost(url.getHost(), url.getPort(), url.getProtocol()); final long startTime = System.currentTimeMillis(); final HttpContext httpContext = getHttpContext(); HttpResponse httpResponse = null; try { - try (CloseableHttpClient closeableHttpClient = builder.build()) { - httpResponse = closeableHttpClient.execute(httpHost, httpMethod, httpContext); + try (CloseableHttpAsyncClient closeableHttpClient = builder.build()) { + httpResponse = execute(closeableHttpClient, httpMethod); } } catch (final SSLPeerUnverifiedException s) { + // TODO: not sure if we can still support this // Try to use only SSLv3 instead if (webClient_.getOptions().isUseInsecureSSL()) { HtmlUnitSSLConnectionSocketFactory.setUseSSL3Only(httpContext, true); - try (CloseableHttpClient closeableHttpClient = builder.build()) { - httpResponse = closeableHttpClient.execute(httpHost, httpMethod, httpContext); + try (CloseableHttpAsyncClient closeableHttpClient = builder.build()) { + httpResponse = execute(closeableHttpClient, httpMethod); } } else { @@ -224,12 +221,50 @@ public WebResponse getResponse(final WebRequest webRequest) throws IOException { } } + /** + * Executes the asynchronous operation and waits for the result. This effectively + * makes the asynchronous client working synchronously. + * @param httpClient the http client instance + * @param httpRequest the request + * @return the response + * @throws IOException if anything went wrongö + */ + private SimpleHttpResponse execute(final CloseableHttpAsyncClient httpClient, final SimpleHttpRequest httpRequest) + throws IOException { + // first start the client if not done so far + httpClient.start(); + + final Future future = + httpClient.execute(SimpleRequestProducer.create(httpRequest), SimpleResponseConsumer.create(), null); + + try { + return future.get(); + } + catch (final InterruptedException e) { + throw new IOException(e); + } + catch (final ExecutionException e) { + // throw the causing exception + final Throwable cause = e.getCause(); + if (cause instanceof RuntimeException) { + throw (RuntimeException) cause; + } + else if (cause instanceof IOException) { + throw (IOException) cause; + } + else { + // need to wrap the exception so we can throw it + throw new IOException(cause); + } + } + } + /** * Called when the response has been generated. Default action is to release * the HttpMethod's connection. Subclasses may override. * @param httpMethod the httpMethod used (can be null) */ - protected void onResponseGenerated(final HttpUriRequest httpMethod) { + protected void onResponseGenerated(final SimpleHttpRequest httpMethod) { } /** @@ -248,20 +283,27 @@ private synchronized HttpContext getHttpContext() { return httpClientContext; } - private void setProxy(final HttpRequestBase httpRequest, final WebRequest webRequest) { + private void setProxy(final SimpleHttpRequest httpRequest, final WebRequest webRequest, final HttpAsyncClientBuilder clientBuilder) { final InetAddress localAddress = webClient_.getOptions().getLocalAddress(); final RequestConfig.Builder requestBuilder = createRequestConfigBuilder(getTimeout(webRequest), localAddress); + // clean potential settings made for a previous request + clientBuilder.setIOReactorConfig(null); + if (webRequest.getProxyHost() == null) { requestBuilder.setProxy(null); httpRequest.setConfig(requestBuilder.build()); return; } - final HttpHost proxy = new HttpHost(webRequest.getProxyHost(), - webRequest.getProxyPort(), webRequest.getProxyScheme()); + final HttpHost proxy = new HttpHost(webRequest.getProxyScheme(), webRequest.getProxyHost(), + webRequest.getProxyPort()); if (webRequest.isSocksProxy()) { - SocksConnectionSocketFactory.setSocksProxy(getHttpContext(), proxy); + final IOReactorConfig.Builder configBuilder = IOReactorConfig.custom(); + configBuilder.setSocksProxyAddress(new InetSocketAddress(webRequest.getProxyHost(), webRequest.getProxyPort())); + final IOReactorConfig ioConfig = configBuilder.build(); + + clientBuilder.setIOReactorConfig(ioConfig); } else { requestBuilder.setProxy(proxy); @@ -277,7 +319,7 @@ private void setProxy(final HttpRequestBase httpRequest, final WebRequest webReq * @throws IOException * @throws URISyntaxException */ - private HttpUriRequest makeHttpMethod(final WebRequest webRequest, final HttpClientBuilder httpClientBuilder) + private SimpleHttpRequest makeHttpMethod(final WebRequest webRequest, final HttpAsyncClientBuilder httpClientBuilder) throws URISyntaxException { final HttpContext httpContext = getHttpContext(); @@ -292,40 +334,32 @@ private HttpUriRequest makeHttpMethod(final WebRequest webRequest, final HttpCli if (getVirtualHost() != null) { uri = URI.create(getVirtualHost()); } - final HttpRequestBase httpMethod = buildHttpMethod(webRequest.getHttpMethod(), uri); - setProxy(httpMethod, webRequest); + final SimpleHttpRequest httpMethod = buildHttpMethod(webRequest.getHttpMethod(), uri); + setProxy(httpMethod, webRequest, httpClientBuilder); - if (httpMethod instanceof HttpEntityEnclosingRequest) { + if (StringUtils.equalsAny(httpMethod.getMethod(), Method.POST.name(), Method.PUT.name(), Method.PATCH.name())) { // POST as well as PUT and PATCH - final HttpEntityEnclosingRequest method = (HttpEntityEnclosingRequest) httpMethod; - if (webRequest.getEncodingType() == FormEncodingType.URL_ENCODED && method instanceof HttpPost) { - final HttpPost postMethod = (HttpPost) method; + if (webRequest.getEncodingType() == FormEncodingType.URL_ENCODED && httpMethod.getMethod().equals(Method.POST.name())) { if (webRequest.getRequestBody() == null) { final List pairs = webRequest.getRequestParameters(); - final String query = URLEncodedUtils.format(NameValuePair.toHttpClient(pairs), charset); + final String query = WWWFormCodec.format(NameValuePair.toHttpClient(pairs), charset); - final StringEntity urlEncodedEntity; + final ContentType contentType; if (webRequest.hasHint(HttpHint.IncludeCharsetInContentTypeHeader)) { - urlEncodedEntity = new StringEntity(query, - ContentType.create(URLEncodedUtils.CONTENT_TYPE, charset)); - + contentType = ContentType.APPLICATION_FORM_URLENCODED.withCharset(charset); } else { - urlEncodedEntity = new StringEntity(query, charset); - urlEncodedEntity.setContentType(URLEncodedUtils.CONTENT_TYPE); + contentType = ContentType.APPLICATION_FORM_URLENCODED.withCharset((Charset) null); } - postMethod.setEntity(urlEncodedEntity); + httpMethod.setBody(query, contentType); } else { final String body = StringUtils.defaultString(webRequest.getRequestBody()); - final StringEntity urlEncodedEntity = new StringEntity(body, charset); - urlEncodedEntity.setContentType(URLEncodedUtils.CONTENT_TYPE); - postMethod.setEntity(urlEncodedEntity); + httpMethod.setBody(body, ContentType.APPLICATION_FORM_URLENCODED.withCharset(charset)); } } - else if (webRequest.getEncodingType() == FormEncodingType.TEXT_PLAIN && method instanceof HttpPost) { - final HttpPost postMethod = (HttpPost) method; + else if (webRequest.getEncodingType() == FormEncodingType.TEXT_PLAIN && httpMethod.getMethod().equals(Method.POST.name())) { if (webRequest.getRequestBody() == null) { final StringBuilder body = new StringBuilder(); for (final NameValuePair pair : webRequest.getRequestParameters()) { @@ -334,19 +368,18 @@ else if (webRequest.getEncodingType() == FormEncodingType.TEXT_PLAIN && method i .append(StringUtils.remove(StringUtils.remove(pair.getValue(), '\r'), '\n')) .append("\r\n"); } - final StringEntity bodyEntity = new StringEntity(body.toString(), charset); - bodyEntity.setContentType(MimeType.TEXT_PLAIN); - postMethod.setEntity(bodyEntity); + httpMethod.setBody(body.toString(), ContentType.TEXT_PLAIN.withCharset(charset)); } else { final String body = StringUtils.defaultString(webRequest.getRequestBody()); - final StringEntity bodyEntity = - new StringEntity(body, ContentType.create(MimeType.TEXT_PLAIN, charset)); - postMethod.setEntity(bodyEntity); + httpMethod.setBody(body, ContentType.TEXT_PLAIN.withCharset(charset)); } } else if (FormEncodingType.MULTIPART == webRequest.getEncodingType()) { final Charset c = getCharset(charset, webRequest.getRequestParameters()); + + // TODO: There ought to be a way to create a multi-part body for the asynchronous client. + // For now, use the classic classes. final MultipartEntityBuilder builder = MultipartEntityBuilder.create().setLaxMode(); builder.setCharset(c); @@ -356,15 +389,30 @@ else if (FormEncodingType.MULTIPART == webRequest.getEncodingType()) { } else { builder.addTextBody(pair.getName(), pair.getValue(), - ContentType.create(MimeType.TEXT_PLAIN, charset)); + ContentType.TEXT_PLAIN.withCharset(charset)); } } - method.setEntity(builder.build()); + + final HttpEntity entity = builder.build(); + + // convert the classic multi-part entity to bytes + final ByteArrayOutputStream buffer = new ByteArrayOutputStream(); + try { + entity.writeTo(buffer); + } + catch (IOException e) { + throw new RuntimeException(e); + } + final byte[] body = buffer.toByteArray(); + + httpMethod.setBody(body, ContentType.MULTIPART_FORM_DATA.withCharset(charset)); } else { // for instance a PUT or PATCH request final String body = webRequest.getRequestBody(); if (body != null) { - method.setEntity(new StringEntity(body, charset)); + // TODO: can't set charset without a content type + //httpMethod.setBody(body, ContentType.create(body, charset); + httpMethod.setBody(body, null); } } } @@ -372,9 +420,9 @@ else if (FormEncodingType.MULTIPART == webRequest.getEncodingType()) { // this is the case for GET as well as TRACE, DELETE, OPTIONS and HEAD if (!webRequest.getRequestParameters().isEmpty()) { final List pairs = webRequest.getRequestParameters(); - final String query = URLEncodedUtils.format(NameValuePair.toHttpClient(pairs), charset); + final String query = WWWFormCodec.format(NameValuePair.toHttpClient(pairs), charset); uri = UrlUtils.toURI(url, query); - httpMethod.setURI(uri); + httpMethod.setUri(uri); } } @@ -382,7 +430,7 @@ else if (FormEncodingType.MULTIPART == webRequest.getEncodingType()) { // Tell the client where to get its credentials from // (it may have changed on the webClient since last call to getHttpClientFor(...)) - final CredentialsProvider credentialsProvider = webClient_.getCredentialsProvider(); + final CredentialsStore credentialsProvider = webClient_.getCredentialsProvider(); // if the used url contains credentials, we have to add this final Credentials requestUrlCredentials = webRequest.getUrlCredentials(); @@ -392,6 +440,9 @@ else if (FormEncodingType.MULTIPART == webRequest.getEncodingType()) { final AuthScope authScope = new AuthScope(requestUrl.getHost(), requestUrl.getPort()); // updating our client to keep the credentials for the next request credentialsProvider.setCredentials(authScope, requestUrlCredentials); + // TODO: not sure if this is the correct way to invalidate a previous auth + HttpClientContext.adapt(httpContext).getAuthExchanges().remove(HttpHost.create(uri)); + sharedAuthCache_.remove(HttpHost.create(uri)); } // if someone has set credentials to this request, we have to add this @@ -401,10 +452,16 @@ else if (FormEncodingType.MULTIPART == webRequest.getEncodingType()) { final AuthScope authScope = new AuthScope(requestUrl.getHost(), requestUrl.getPort()); // updating our client to keep the credentials for the next request credentialsProvider.setCredentials(authScope, requestCredentials); + // TODO: not sure if this is the correct way to invalidate a previous auth + HttpClientContext.adapt(httpContext).getAuthExchanges().remove(HttpHost.create(uri)); + sharedAuthCache_.remove(HttpHost.create(uri)); } httpClientBuilder.setDefaultCredentialsProvider(credentialsProvider); httpContext.removeAttribute(HttpClientContext.CREDS_PROVIDER); - httpContext.removeAttribute(HttpClientContext.TARGET_AUTH_STATE); + + // TODO: don't know how to replace this??? + //httpContext.removeAttribute(HttpClientContext.TARGET_AUTH_STATE); + return httpMethod; } @@ -489,39 +546,40 @@ else if (pairWithFile.getFileName() == null) { * @param uri the uri being used * @return a new HttpClient HTTP method based on the specified parameters */ - private static HttpRequestBase buildHttpMethod(final HttpMethod submitMethod, final URI uri) { - final HttpRequestBase method; + private static SimpleHttpRequest buildHttpMethod(final HttpMethod submitMethod, final URI uri) { + final SimpleHttpRequest method; + switch (submitMethod) { case GET: - method = new HttpGet(uri); + method = new SimpleHttpRequest(Method.GET, uri); break; case POST: - method = new HttpPost(uri); + method = new SimpleHttpRequest(Method.POST, uri); break; case PUT: - method = new HttpPut(uri); + method = new SimpleHttpRequest(Method.PUT, uri); break; case DELETE: - method = new HttpDelete(uri); + method = new SimpleHttpRequest(Method.DELETE, uri); break; case OPTIONS: - method = new HttpOptions(uri); + method = new SimpleHttpRequest(Method.OPTIONS, uri); break; case HEAD: - method = new HttpHead(uri); + method = new SimpleHttpRequest(Method.HEAD, uri); break; case TRACE: - method = new HttpTrace(uri); + method = new SimpleHttpRequest(Method.TRACE, uri); break; case PATCH: - method = new HttpPatch(uri); + method = new SimpleHttpRequest(Method.PATCH, uri); break; default: @@ -535,16 +593,16 @@ private static HttpRequestBase buildHttpMethod(final HttpMethod submitMethod, fi * * @return the initialized HTTP client */ - protected HttpClientBuilder getHttpClientBuilder() { + protected HttpAsyncClientBuilder getHttpClientBuilder() { final Thread currentThread = Thread.currentThread(); - HttpClientBuilder builder = httpClientBuilder_.get(currentThread); + HttpAsyncClientBuilder builder = httpClientBuilder_.get(currentThread); if (builder == null) { builder = createHttpClientBuilder(); // this factory is required later // to be sure this is done, we do it outside the createHttpClient() call - final RegistryBuilder registeryBuilder - = RegistryBuilder.create() + final RegistryBuilder registeryBuilder + = RegistryBuilder.create() .register(HACKED_COOKIE_POLICY, htmlUnitCookieSpecProvider_); builder.setDefaultCookieSpecRegistry(registeryBuilder.build()); @@ -575,49 +633,80 @@ protected int getTimeout(final WebRequest webRequest) { * Creates the HttpClientBuilder that will be used by this WebClient. * Extensions may override this method in order to create a customized * HttpClientBuilder instance (e.g. with a custom - * {@link org.apache.http.conn.ClientConnectionManager} to perform + * {@link org.apache.hc.client5.http.io.HttpClientConnectionManager} to perform * some tracking; see feature request 1438216). * @return the HttpClientBuilder that will be used by this WebConnection */ - protected HttpClientBuilder createHttpClientBuilder() { - final HttpClientBuilder builder = HttpClientBuilder.create(); + protected HttpAsyncClientBuilder createHttpClientBuilder() { + final HttpAsyncClientBuilder builder = HttpAsyncClientBuilder.create(); builder.setRedirectStrategy(new HtmlUnitRedirectStrategie()); configureTimeout(builder, getTimeout(null)); - configureHttpsScheme(builder); - builder.setMaxConnPerRoute(6); + + //builder.disableContentCompression(); + builder.disableRedirectHandling(); + //builder.setVersionPolicy(HttpVersionPolicy.FORCE_HTTP_1); builder.setConnectionManagerShared(true); return builder; } - private void configureTimeout(final HttpClientBuilder builder, final int timeout) { + private void configureTimeout(final HttpAsyncClientBuilder builder, final int timeout) { final InetAddress localAddress = webClient_.getOptions().getLocalAddress(); final RequestConfig.Builder requestBuilder = createRequestConfigBuilder(timeout, localAddress); builder.setDefaultRequestConfig(requestBuilder.build()); - builder.setDefaultSocketConfig(createSocketConfigBuilder(timeout).build()); + // TODO: + //builder.setDefaultSocketConfig(createSocketConfigBuilder(timeout).build()); getHttpContext().removeAttribute(HttpClientContext.REQUEST_CONFIG); usedOptions_.setTimeout(timeout); } private static RequestConfig.Builder createRequestConfigBuilder(final int timeout, final InetAddress localAddress) { + + final Timeout timeoutMS = Timeout.ofMilliseconds(timeout); + final RequestConfig.Builder requestBuilder = RequestConfig.custom() .setCookieSpec(HACKED_COOKIE_POLICY) .setRedirectsEnabled(false) - .setLocalAddress(localAddress) + + // TODO: Has been moved elsewhere, but where? HttpRoutePlanner? + //.setLocalAddress(localAddress) // timeout - .setConnectTimeout(timeout) - .setConnectionRequestTimeout(timeout) - .setSocketTimeout(timeout); + .setConnectTimeout(timeoutMS) + .setConnectionRequestTimeout(timeoutMS) + .setResponseTimeout(timeoutMS); return requestBuilder; } + /** + * Creates the PoolingHttpClientConnectionManagerBuilder that will be used by this WebClient. + * Extensions may override this method in order to create a customized + * PoolingHttpClientConnectionManagerBuilder instance. + * @return the PoolingHttpClientConnectionManagerBuilder that will be used by this WebConnection + */ + protected PoolingAsyncClientConnectionManagerBuilder createConnectionManagerBuilder(final int timeout, final WebClientOptions options) { + final PoolingAsyncClientConnectionManagerBuilder builder = PoolingAsyncClientConnectionManagerBuilder.create(); + + configureHttpsScheme(builder); + + long connectionTimeToLive = options.getConnectionTimeToLive(); + if (connectionTimeToLive >= 0) { + final TimeValue timeToLive = TimeValue.ofMilliseconds(connectionTimeToLive); + builder.setConnectionTimeToLive(timeToLive); + } + + builder.setMaxConnPerRoute(6); + + return builder; + } + + // TODO: settings probably moved to IORecactorConfig? private static SocketConfig.Builder createSocketConfigBuilder(final int timeout) { final SocketConfig.Builder socketBuilder = SocketConfig.custom() // timeout - .setSoTimeout(timeout); + .setSoTimeout(Timeout.ofMilliseconds(timeout)); return socketBuilder; } @@ -625,52 +714,51 @@ private static SocketConfig.Builder createSocketConfigBuilder(final int timeout) * React on changes that may have occurred on the WebClient settings. * Registering as a listener would be probably better. */ - private HttpClientBuilder reconfigureHttpClientIfNeeded(final HttpClientBuilder httpClientBuilder, + private HttpAsyncClientBuilder reconfigureHttpClientIfNeeded(final HttpAsyncClientBuilder httpClientBuilder, final WebRequest webRequest) { final WebClientOptions options = webClient_.getOptions(); - // register new SSL factory only if settings have changed - if (options.isUseInsecureSSL() != usedOptions_.isUseInsecureSSL() - || options.getSSLClientCertificateStore() != usedOptions_.getSSLClientCertificateStore() - || options.getSSLTrustStore() != usedOptions_.getSSLTrustStore() - || options.getSSLClientCipherSuites() != usedOptions_.getSSLClientCipherSuites() - || options.getSSLClientProtocols() != usedOptions_.getSSLClientProtocols() - || options.getProxyConfig() != usedOptions_.getProxyConfig()) { - configureHttpsScheme(httpClientBuilder); - - if (connectionManager_ != null) { - connectionManager_.shutdown(); - connectionManager_ = null; - } - } - +// // register new SSL factory only if settings have changed +// if (options.isUseInsecureSSL() != usedOptions_.isUseInsecureSSL() +// || options.getSSLClientCertificateStore() != usedOptions_.getSSLClientCertificateStore() +// || options.getSSLTrustStore() != usedOptions_.getSSLTrustStore() +// || options.getSSLClientCipherSuites() != usedOptions_.getSSLClientCipherSuites() +// || options.getSSLClientProtocols() != usedOptions_.getSSLClientProtocols() +// || options.getProxyConfig() != usedOptions_.getProxyConfig()) { +// configureHttpsScheme(httpClientBuilder); +// +// if (connectionManager_ != null) { +// connectionManager_.close(); +// connectionManager_ = null; +// } +// } +// final int timeout = getTimeout(webRequest); if (timeout != usedOptions_.getTimeout()) { configureTimeout(httpClientBuilder, timeout); } - final long connectionTimeToLive = webClient_.getOptions().getConnectionTimeToLive(); - if (connectionTimeToLive != usedOptions_.getConnectionTimeToLive()) { - httpClientBuilder.setConnectionTimeToLive(connectionTimeToLive, TimeUnit.MILLISECONDS); - usedOptions_.setConnectionTimeToLive(connectionTimeToLive); - } +// final long connectionTimeToLive = webClient_.getOptions().getConnectionTimeToLive(); +// if (connectionTimeToLive != usedOptions_.getConnectionTimeToLive()) { +// // TODO +// //httpClientBuilder.setConnectionTimeToLive(connectionTimeToLive, TimeUnit.MILLISECONDS); +// usedOptions_.setConnectionTimeToLive(connectionTimeToLive); +// } if (connectionManager_ == null) { - connectionManager_ = createConnectionManager(httpClientBuilder); + connectionManager_ = createConnectionManagerBuilder(timeout, options).build(); } httpClientBuilder.setConnectionManager(connectionManager_); return httpClientBuilder; } - private void configureHttpsScheme(final HttpClientBuilder builder) { + private void configureHttpsScheme(final PoolingAsyncClientConnectionManagerBuilder builder) { final WebClientOptions options = webClient_.getOptions(); - final SSLConnectionSocketFactory socketFactory = - HtmlUnitSSLConnectionSocketFactory.buildSSLSocketFactory(options); - - builder.setSSLSocketFactory(socketFactory); - + final TlsStrategy tlsStrategy = createTlsStrategyBuilder(options).build(); + builder.setTlsStrategy(tlsStrategy); + usedOptions_.setUseInsecureSSL(options.isUseInsecureSSL()); usedOptions_.setSSLClientCertificateStore(options.getSSLClientCertificateStore()); usedOptions_.setSSLTrustStore(options.getSSLTrustStore()); @@ -679,22 +767,35 @@ private void configureHttpsScheme(final HttpClientBuilder builder) { usedOptions_.setProxyConfig(options.getProxyConfig()); } - private void configureHttpProcessorBuilder(final HttpClientBuilder builder, final WebRequest webRequest) { - final HttpProcessorBuilder b = HttpProcessorBuilder.create(); - for (final HttpRequestInterceptor i : getHttpRequestInterceptors(webRequest)) { - b.add(i); - } - - // These are the headers used in HttpClientBuilder, excluding the already added ones - // (RequestClientConnControl and RequestAddCookies) - b.addAll(new RequestDefaultHeaders(null), - new RequestContent(), - new RequestTargetHost(), - new RequestExpectContinue()); - b.add(new RequestAcceptEncoding()); - b.add(new RequestAuthCache()); - b.add(new ResponseProcessCookies()); - builder.setHttpProcessor(b.build()); + private void configureHttpProcessorBuilder(final HttpAsyncClientBuilder builder, final WebRequest webRequest) { + + // HttpProcessor is now created internally only, hence we have to configure + // the request interceptors directly at the builder. + + // Disable certain features so the related request interceptors are not added by default. + // We will add them on our own, in the order we want. + builder.disableConnectionState(); + builder.disableCookieManagement(); + + // first clear the current list of request interceptors at the builder + try { + // HACK: Seems there is no other way in the moment. + setField(builder, "requestInterceptors", null); + } + catch (IllegalAccessException e) { + throw new RuntimeException(e); + } + + // fill the list of request interceptors anew, specifically for the web request + final List interceptors = getHttpRequestInterceptors(webRequest); + + // reverse the list so we can use addRequestInterceptorFirst() and get the wanted result + Collections.reverse(interceptors); + for (final HttpRequestInterceptor i : interceptors) { + builder.addRequestInterceptorFirst(i); + } + + builder.addResponseInterceptorFirst(new ResponseProcessCookies()); } /** @@ -719,13 +820,13 @@ public String getVirtualHost() { private WebResponse makeWebResponse(final HttpResponse httpResponse, final WebRequest webRequest, final DownloadedContent responseBody, final long loadTime) { - String statusMessage = httpResponse.getStatusLine().getReasonPhrase(); + String statusMessage = new StatusLine(httpResponse).getReasonPhrase(); if (statusMessage == null) { statusMessage = "Unknown status message"; } - final int statusCode = httpResponse.getStatusLine().getStatusCode(); + final int statusCode = httpResponse.getCode(); final List headers = new ArrayList<>(); - for (final Header header : httpResponse.getAllHeaders()) { + for (final Header header : httpResponse.getHeaders()) { headers.add(new NameValuePair(header.getName(), header.getValue())); } final WebResponseData responseData = new WebResponseData(responseBody, statusCode, statusMessage, headers); @@ -739,12 +840,22 @@ private WebResponse makeWebResponse(final HttpResponse httpResponse, * @throws IOException in case of problem reading/saving the body */ protected DownloadedContent downloadResponseBody(final HttpResponse httpResponse) throws IOException { - final HttpEntity httpEntity = httpResponse.getEntity(); +// final HttpEntity httpEntity = ((CloseableHttpResponse) httpResponse).getEntity(); +// if (httpEntity == null) { +// return new DownloadedContent.InMemory(null); +// } +// +// try (InputStream is = httpEntity.getContent()) { +// return downloadContent(is, webClient_.getOptions().getMaxInMemory()); +// } + + // TODO: replace SimpleHttpResponse with a streaming response + final SimpleBody httpEntity = ((SimpleHttpResponse) httpResponse).getBody(); if (httpEntity == null) { return new DownloadedContent.InMemory(null); } - try (InputStream is = httpEntity.getContent()) { + try (InputStream is = new ByteArrayInputStream(httpEntity.getBodyBytes())) { return downloadContent(is, webClient_.getOptions().getMaxInMemory()); } } @@ -900,6 +1011,7 @@ else if (HttpHeader.REFERER.equals(header)) { } } else if (HttpHeader.CONNECTION.equals(header)) { + // TODO: add this only if not HTTP/2 list.add(new RequestClientConnControl()); } else if (HttpHeader.COOKIE.equals(header)) { @@ -931,7 +1043,7 @@ private static final class HostHeaderHttpRequestInterceptor implements HttpReque } @Override - public void process(final HttpRequest request, final HttpContext context) throws HttpException, IOException { + public void process(final HttpRequest request, final EntityDetails entity, final HttpContext context) throws HttpException, IOException { request.setHeader(HttpHeader.HOST, value_); } } @@ -944,7 +1056,7 @@ private static final class UserAgentHeaderHttpRequestInterceptor implements Http } @Override - public void process(final HttpRequest request, final HttpContext context) throws HttpException, IOException { + public void process(final HttpRequest request, final EntityDetails entity, final HttpContext context) throws HttpException, IOException { request.setHeader(HttpHeader.USER_AGENT, value_); } } @@ -957,7 +1069,7 @@ private static final class AcceptHeaderHttpRequestInterceptor implements HttpReq } @Override - public void process(final HttpRequest request, final HttpContext context) throws HttpException, IOException { + public void process(final HttpRequest request, final EntityDetails entity, final HttpContext context) throws HttpException, IOException { request.setHeader(HttpHeader.ACCEPT, value_); } } @@ -970,7 +1082,7 @@ private static final class AcceptLanguageHeaderHttpRequestInterceptor implements } @Override - public void process(final HttpRequest request, final HttpContext context) throws HttpException, IOException { + public void process(final HttpRequest request, final EntityDetails entity, final HttpContext context) throws HttpException, IOException { request.setHeader(HttpHeader.ACCEPT_LANGUAGE, value_); } } @@ -983,7 +1095,7 @@ private static final class UpgradeInsecureRequestHeaderHttpRequestInterceptor im } @Override - public void process(final HttpRequest request, final HttpContext context) throws HttpException, IOException { + public void process(final HttpRequest request, final EntityDetails entity, final HttpContext context) throws HttpException, IOException { request.setHeader(HttpHeader.UPGRADE_INSECURE_REQUESTS, value_); } } @@ -996,7 +1108,7 @@ private static final class AcceptEncodingHeaderHttpRequestInterceptor implements } @Override - public void process(final HttpRequest request, final HttpContext context) throws HttpException, IOException { + public void process(final HttpRequest request, final EntityDetails entity, final HttpContext context) throws HttpException, IOException { request.setHeader("Accept-Encoding", value_); } } @@ -1009,7 +1121,7 @@ private static final class RefererHeaderHttpRequestInterceptor implements HttpRe } @Override - public void process(final HttpRequest request, final HttpContext context) throws HttpException, IOException { + public void process(final HttpRequest request, final EntityDetails entity, final HttpContext context) throws HttpException, IOException { request.setHeader(HttpHeader.REFERER, value_); } } @@ -1022,7 +1134,7 @@ private static final class DntHeaderHttpRequestInterceptor implements HttpReques } @Override - public void process(final HttpRequest request, final HttpContext context) throws HttpException, IOException { + public void process(final HttpRequest request, final EntityDetails entity, final HttpContext context) throws HttpException, IOException { request.setHeader(HttpHeader.DNT, value_); } } @@ -1035,7 +1147,7 @@ private static final class SecFetchModeHeaderHttpRequestInterceptor implements H } @Override - public void process(final HttpRequest request, final HttpContext context) throws HttpException, IOException { + public void process(final HttpRequest request, final EntityDetails entity, final HttpContext context) throws HttpException, IOException { request.setHeader(HttpHeader.SEC_FETCH_MODE, value_); } } @@ -1048,7 +1160,7 @@ private static final class SecFetchSiteHeaderHttpRequestInterceptor implements H } @Override - public void process(final HttpRequest request, final HttpContext context) throws HttpException, IOException { + public void process(final HttpRequest request, final EntityDetails entity, final HttpContext context) throws HttpException, IOException { request.setHeader(HttpHeader.SEC_FETCH_SITE, value_); } } @@ -1061,7 +1173,7 @@ private static final class SecFetchUserHeaderHttpRequestInterceptor implements H } @Override - public void process(final HttpRequest request, final HttpContext context) throws HttpException, IOException { + public void process(final HttpRequest request, final EntityDetails entity, final HttpContext context) throws HttpException, IOException { request.setHeader(HttpHeader.SEC_FETCH_USER, value_); } } @@ -1074,7 +1186,7 @@ private static final class SecFetchDestHeaderHttpRequestInterceptor implements H } @Override - public void process(final HttpRequest request, final HttpContext context) throws HttpException, IOException { + public void process(final HttpRequest request, final EntityDetails entity, final HttpContext context) throws HttpException, IOException { request.setHeader(HttpHeader.SEC_FETCH_DEST, value_); } } @@ -1087,7 +1199,7 @@ private static final class SecClientHintUserAgentHeaderHttpRequestInterceptor im } @Override - public void process(final HttpRequest request, final HttpContext context) throws HttpException, IOException { + public void process(final HttpRequest request, final EntityDetails entity, final HttpContext context) throws HttpException, IOException { request.setHeader(HttpHeader.SEC_CH_UA, value_); } } @@ -1101,7 +1213,7 @@ private static final class SecClientHintUserAgentMobileHeaderHttpRequestIntercep } @Override - public void process(final HttpRequest request, final HttpContext context) throws HttpException, IOException { + public void process(final HttpRequest request, final EntityDetails entity, final HttpContext context) throws HttpException, IOException { request.setHeader(HttpHeader.SEC_CH_UA_MOBILE, value_); } } @@ -1114,7 +1226,7 @@ private static class MultiHttpRequestInterceptor implements HttpRequestIntercept } @Override - public void process(final HttpRequest request, final HttpContext context) + public void process(final HttpRequest request, final EntityDetails entity, final HttpContext context) throws HttpException, IOException { for (final Map.Entry entry : map_.entrySet()) { request.setHeader(entry.getKey(), entry.getValue()); @@ -1125,6 +1237,7 @@ public void process(final HttpRequest request, final HttpContext context) /** * An authentication cache that is synchronized. */ + // TODO: Looks like BasicAuthCache is already thread-safe so we can remove this class? private static final class SynchronizedAuthCache extends BasicAuthCache { SynchronizedAuthCache() { @@ -1164,107 +1277,91 @@ public void close() { httpClientBuilder_.clear(); if (connectionManager_ != null) { - connectionManager_.shutdown(); + connectionManager_.close(); connectionManager_ = null; } } /** - * Has the exact logic in {@link HttpClientBuilder#build()} which sets the {@code connManager} part, - * but with the ability to configure {@code socketFactory}. + * Creates a preconfigured builder for {@link TlsStrategy} objects. Sub classes + * may override this method to add further customizations to the builder. + * @param options the WebClient options + * @return the builder */ - private static PoolingHttpClientConnectionManager createConnectionManager(final HttpClientBuilder builder) { + protected ClientTlsStrategyBuilder createTlsStrategyBuilder(final WebClientOptions options) { try { - PublicSuffixMatcher publicSuffixMatcher = getField(builder, "publicSuffixMatcher"); - if (publicSuffixMatcher == null) { - publicSuffixMatcher = PublicSuffixMatcherLoader.getDefault(); - } - - LayeredConnectionSocketFactory sslSocketFactory = getField(builder, "sslSocketFactory"); - final SocketConfig defaultSocketConfig = getField(builder, "defaultSocketConfig"); - final ConnectionConfig defaultConnectionConfig = getField(builder, "defaultConnectionConfig"); - final boolean systemProperties = getField(builder, "systemProperties"); - final int maxConnTotal = getField(builder, "maxConnTotal"); - final int maxConnPerRoute = getField(builder, "maxConnPerRoute"); - HostnameVerifier hostnameVerifier = getField(builder, "hostnameVerifier"); - final SSLContext sslcontext = getField(builder, "sslContext"); - final DnsResolver dnsResolver = getField(builder, "dnsResolver"); - final long connTimeToLive = getField(builder, "connTimeToLive"); - final TimeUnit connTimeToLiveTimeUnit = getField(builder, "connTimeToLiveTimeUnit"); - - if (sslSocketFactory == null) { - final String[] supportedProtocols = systemProperties - ? split(System.getProperty("https.protocols")) : null; - final String[] supportedCipherSuites = systemProperties - ? split(System.getProperty("https.cipherSuites")) : null; - if (hostnameVerifier == null) { - hostnameVerifier = new DefaultHostnameVerifier(publicSuffixMatcher); - } - if (sslcontext == null) { - if (systemProperties) { - sslSocketFactory = new SSLConnectionSocketFactory( - (SSLSocketFactory) SSLSocketFactory.getDefault(), - supportedProtocols, supportedCipherSuites, hostnameVerifier); - } - else { - sslSocketFactory = new SSLConnectionSocketFactory( - SSLContexts.createDefault(), - hostnameVerifier); + final String[] sslClientProtocols = options.getSSLClientProtocols(); + final String[] sslClientCipherSuites = options.getSSLClientCipherSuites(); + final boolean useInsecureSSL = options.isUseInsecureSSL(); + + final SSLContext sslContext = createSslContextBuilder(options).build(); + + final ClientTlsStrategyBuilder tlsStrategyBuilder = ClientTlsStrategyBuilder.create(); + tlsStrategyBuilder.setSslContext(sslContext); + tlsStrategyBuilder.setCiphers(sslClientCipherSuites); + tlsStrategyBuilder.setTlsVersions(sslClientProtocols); + tlsStrategyBuilder.setHostnameVerifier(useInsecureSSL ? NoopHostnameVerifier.INSTANCE : new DefaultHostnameVerifier()); + + if (SystemUtils.isJavaVersionAtMost(JavaVersion.JAVA_9)) { + // From the async HttpClient examples: + + // IMPORTANT uncomment the following method when running Java 9 or older + // in order for ALPN support to work and avoid the illegal reflective + // access operation warning + tlsStrategyBuilder.setTlsDetailsFactory(new Factory() { + @Override + public TlsDetails create(final SSLEngine sslEngine) { + return new TlsDetails(sslEngine.getSession(), sslEngine.getApplicationProtocol()); } - } - else { - sslSocketFactory = new SSLConnectionSocketFactory( - sslcontext, supportedProtocols, supportedCipherSuites, hostnameVerifier); - } - } - - final PoolingHttpClientConnectionManager poolingmgr = new PoolingHttpClientConnectionManager( - RegistryBuilder.create() - .register("http", new SocksConnectionSocketFactory()) - .register("https", sslSocketFactory) - .build(), - null, - null, - dnsResolver, - connTimeToLive, - connTimeToLiveTimeUnit != null ? connTimeToLiveTimeUnit : TimeUnit.MILLISECONDS); - if (defaultSocketConfig != null) { - poolingmgr.setDefaultSocketConfig(defaultSocketConfig); - } - if (defaultConnectionConfig != null) { - poolingmgr.setDefaultConnectionConfig(defaultConnectionConfig); - } - if (systemProperties) { - String s = System.getProperty("http.keepAlive", "true"); - if ("true".equalsIgnoreCase(s)) { - s = System.getProperty("http.maxConnections", "5"); - final int max = Integer.parseInt(s); - poolingmgr.setDefaultMaxPerRoute(max); - poolingmgr.setMaxTotal(2 * max); - } - } - if (maxConnTotal > 0) { - poolingmgr.setMaxTotal(maxConnTotal); - } - if (maxConnPerRoute > 0) { - poolingmgr.setDefaultMaxPerRoute(maxConnPerRoute); + }); } - return poolingmgr; + + return tlsStrategyBuilder; } - catch (final IllegalAccessException e) { + catch (GeneralSecurityException e) { throw new RuntimeException(e); } } - private static String[] split(final String s) { - if (TextUtils.isBlank(s)) { - return null; + /** + * Creates a preconfigured builder for {@link SSLContext} objects. Sub classes + * may override this method to add further customizations to the builder. + * @param options the WebClient options + * @return the builder + */ + protected SSLContextBuilder createSslContextBuilder(final WebClientOptions options) { + try { + final SSLContextBuilder sslContextBuilder = SSLContexts.custom(); + + // custom key store + final KeyStore keyStore = options.getSSLClientCertificateStore(); + final char[] keyStorePassword = options.getSSLClientCertificatePassword(); + + sslContextBuilder.loadKeyMaterial(keyStore, keyStorePassword); + + // custom trust store + final boolean useInsecureSSL = options.isUseInsecureSSL(); + if (useInsecureSSL) { + sslContextBuilder.loadTrustMaterial(new TrustAllStrategy()); + } + else { + final KeyStore trustStore = options.getSSLTrustStore(); + sslContextBuilder.loadTrustMaterial(trustStore, null); + } + + return sslContextBuilder; + } + catch (GeneralSecurityException e) { + throw new RuntimeException(e); } - return s.split(" *, *"); } - + @SuppressWarnings("unchecked") private static T getField(final Object target, final String fieldName) throws IllegalAccessException { return (T) FieldUtils.readDeclaredField(target, fieldName, true); } + + private static void setField(final Object target, final String fieldName, T value) throws IllegalAccessException { + FieldUtils.writeDeclaredField(target, fieldName, value, true); + } } diff --git a/src/main/java/com/gargoylesoftware/htmlunit/NetscapeDraftHeaderParser.java b/src/main/java/com/gargoylesoftware/htmlunit/NetscapeDraftHeaderParser.java new file mode 100644 index 00000000000..262ce10c112 --- /dev/null +++ b/src/main/java/com/gargoylesoftware/htmlunit/NetscapeDraftHeaderParser.java @@ -0,0 +1,119 @@ +/* + * Copyright (c) 2002-2021 Gargoyle Software Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * https://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +/* + * ==================================================================== + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + * ==================================================================== + * + * This software consists of voluntary contributions made by many + * individuals on behalf of the Apache Software Foundation. For more + * information on the Apache Software Foundation, please see + * . + * + */ + +package com.gargoylesoftware.htmlunit; + + +import java.util.ArrayList; +import java.util.BitSet; +import java.util.List; + +import org.apache.hc.core5.annotation.Contract; +import org.apache.hc.core5.annotation.ThreadingBehavior; +import org.apache.hc.core5.http.HeaderElement; +import org.apache.hc.core5.http.NameValuePair; +import org.apache.hc.core5.http.ParseException; +import org.apache.hc.core5.http.message.BasicHeaderElement; +import org.apache.hc.core5.http.message.BasicNameValuePair; +import org.apache.hc.core5.http.message.ParserCursor; +import org.apache.hc.core5.http.message.TokenParser; +import org.apache.hc.core5.util.Args; +import org.apache.hc.core5.util.CharArrayBuffer; + + +/** + * @since 4.0 + */ +@Contract(threading = ThreadingBehavior.IMMUTABLE) +public class NetscapeDraftHeaderParser { + + public final static NetscapeDraftHeaderParser DEFAULT = new NetscapeDraftHeaderParser(); + + private final static char PARAM_DELIMITER = ';'; + + // IMPORTANT! + // These private static variables must be treated as immutable and never exposed outside this class + private static final BitSet TOKEN_DELIMS = TokenParser.INIT_BITSET('=', PARAM_DELIMITER); + private static final BitSet VALUE_DELIMS = TokenParser.INIT_BITSET(PARAM_DELIMITER); + + private final TokenParser tokenParser; + + public NetscapeDraftHeaderParser() { + super(); + this.tokenParser = TokenParser.INSTANCE; + } + + public HeaderElement parseHeader( + final CharArrayBuffer buffer, + final ParserCursor cursor) throws ParseException { + Args.notNull(buffer, "Char array buffer"); + Args.notNull(cursor, "Parser cursor"); + final NameValuePair nvp = parseNameValuePair(buffer, cursor); + final List params = new ArrayList(); + while (!cursor.atEnd()) { + final NameValuePair param = parseNameValuePair(buffer, cursor); + params.add(param); + } + return new BasicHeaderElement( + nvp.getName(), + nvp.getValue(), params.toArray(new NameValuePair[params.size()])); + } + + private NameValuePair parseNameValuePair( + final CharArrayBuffer buffer, final ParserCursor cursor) { + final String name = tokenParser.parseToken(buffer, cursor, TOKEN_DELIMS); + if (cursor.atEnd()) { + return new BasicNameValuePair(name, null); + } + final int delim = buffer.charAt(cursor.getPos()); + cursor.updatePos(cursor.getPos() + 1); + if (delim != '=') { + return new BasicNameValuePair(name, null); + } + final String value = tokenParser.parseToken(buffer, cursor, VALUE_DELIMS); + if (!cursor.atEnd()) { + cursor.updatePos(cursor.getPos() + 1); + } + return new BasicNameValuePair(name, value); + } + +} diff --git a/src/main/java/com/gargoylesoftware/htmlunit/StringWebResponse.java b/src/main/java/com/gargoylesoftware/htmlunit/StringWebResponse.java index d2d4bdb6ee6..9d40983eade 100644 --- a/src/main/java/com/gargoylesoftware/htmlunit/StringWebResponse.java +++ b/src/main/java/com/gargoylesoftware/htmlunit/StringWebResponse.java @@ -21,7 +21,7 @@ import java.util.ArrayList; import java.util.List; -import org.apache.http.HttpStatus; +import org.apache.hc.core5.http.HttpStatus; import com.gargoylesoftware.htmlunit.util.NameValuePair; import com.gargoylesoftware.htmlunit.util.TextUtils; diff --git a/src/main/java/com/gargoylesoftware/htmlunit/WebClient.java b/src/main/java/com/gargoylesoftware/htmlunit/WebClient.java index d9f7c137531..1dc3c133f53 100644 --- a/src/main/java/com/gargoylesoftware/htmlunit/WebClient.java +++ b/src/main/java/com/gargoylesoftware/htmlunit/WebClient.java @@ -62,16 +62,17 @@ import org.apache.commons.lang3.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.http.HttpStatus; -import org.apache.http.NoHttpResponseException; -import org.apache.http.client.CredentialsProvider; -import org.apache.http.client.utils.DateUtils; -import org.apache.http.cookie.ClientCookie; -import org.apache.http.cookie.CookieOrigin; -import org.apache.http.cookie.CookieSpec; -import org.apache.http.cookie.MalformedCookieException; -import org.apache.http.message.BufferedHeader; -import org.apache.http.util.CharArrayBuffer; +import org.apache.hc.client5.http.auth.CredentialsStore; +import org.apache.hc.client5.http.cookie.CookieOrigin; +import org.apache.hc.client5.http.cookie.CookieSpec; +import org.apache.hc.client5.http.cookie.MalformedCookieException; +import org.apache.hc.client5.http.utils.DateUtils; +import org.apache.hc.core5.http.ConnectionClosedException; +import org.apache.hc.core5.http.HttpStatus; +import org.apache.hc.core5.http.NoHttpResponseException; +import org.apache.hc.core5.http.ParseException; +import org.apache.hc.core5.http.message.BufferedHeader; +import org.apache.hc.core5.util.CharArrayBuffer; import com.gargoylesoftware.css.parser.CSSErrorHandler; import com.gargoylesoftware.htmlunit.activex.javascript.msxml.MSXMLActiveXObjectFactory; @@ -166,7 +167,7 @@ public class WebClient implements Serializable, AutoCloseable { 0, "No HTTP Response", Collections.emptyList()); private transient WebConnection webConnection_; - private CredentialsProvider credentialsProvider_ = new DefaultCredentialsProvider(); + private CredentialsStore credentialsProvider_ = new DefaultCredentialsProvider(); private CookieManager cookieManager_ = new CookieManager(); private transient AbstractJavaScriptEngine scriptEngine_; private transient List loadQueue_; @@ -492,7 +493,7 @@

P getPage(final WebWindow webWindow, final WebRequest webReques try { webResponse = loadWebResponse(webRequest); } - catch (final NoHttpResponseException e) { + catch (final ConnectionClosedException e) { webResponse = new WebResponse(RESPONSE_DATA_NO_HTTP_RESPONSE, webRequest, 0); } } @@ -789,7 +790,7 @@ public void removeRequestHeader(final String name) { * or Digest authentication. * @param credentialsProvider the new credentials provider to use to authenticate */ - public void setCredentialsProvider(final CredentialsProvider credentialsProvider) { + public void setCredentialsProvider(final CredentialsStore credentialsProvider) { WebAssert.notNull("credentialsProvider", credentialsProvider); credentialsProvider_ = credentialsProvider; } @@ -799,7 +800,7 @@ public void setCredentialsProvider(final CredentialsProvider credentialsProvider * method returns an instance of {@link DefaultCredentialsProvider}. * @return the credentials provider for this client instance */ - public CredentialsProvider getCredentialsProvider() { + public CredentialsStore getCredentialsProvider() { return credentialsProvider_; } @@ -2453,7 +2454,7 @@ public void download(final WebWindow requestingWindow, final String target, try { response = loadWebResponse(request); } - catch (final NoHttpResponseException e) { + catch (final ConnectionClosedException e) { LOG.error("NoHttpResponseException while downloading; generating a NoHttpResponse", e); response = new WebResponse(RESPONSE_DATA_NO_HTTP_RESPONSE, request, 0); } @@ -2602,13 +2603,13 @@ public synchronized Set getCookies(final URL url) { // discard expired cookies cookieManager.clearExpired(new Date()); - final List all = Cookie.toHttpClient(cookieManager.getCookies()); - final List matches = new ArrayList<>(); + final List all = Cookie.toHttpClient(cookieManager.getCookies()); + final List matches = new ArrayList<>(); if (all.size() > 0) { final CookieOrigin cookieOrigin = new CookieOrigin(host, port, path, secure); final CookieSpec cookieSpec = new HtmlUnitBrowserCompatCookieSpec(getBrowserVersion()); - for (final org.apache.http.cookie.Cookie cookie : all) { + for (final org.apache.hc.client5.http.cookie.Cookie cookie : all) { if (cookieSpec.match(cookie, cookieOrigin)) { matches.add(cookie); } @@ -2643,11 +2644,11 @@ public void addCookie(final String cookieString, final URL pageUrl, final Object final CookieSpec cookieSpec = new HtmlUnitBrowserCompatCookieSpec(getBrowserVersion()); try { - final List cookies = + final List cookies = cookieSpec.parse(new BufferedHeader(buffer), cookieManager.buildCookieOrigin(pageUrl)); - for (final org.apache.http.cookie.Cookie cookie : cookies) { - final Cookie htmlUnitCookie = new Cookie((ClientCookie) cookie); + for (final org.apache.hc.client5.http.cookie.Cookie cookie : cookies) { + final Cookie htmlUnitCookie = new Cookie(cookie); cookieManager.addCookie(htmlUnitCookie); if (LOG.isDebugEnabled()) { @@ -2655,7 +2656,7 @@ public void addCookie(final String cookieString, final URL pageUrl, final Object } } } - catch (final MalformedCookieException e) { + catch (final MalformedCookieException | ParseException e) { if (LOG.isDebugEnabled()) { LOG.warn("Adding cookie '" + cookieString + "' failed; reason: '" + e.getMessage() + "'."); } diff --git a/src/main/java/com/gargoylesoftware/htmlunit/WebRequest.java b/src/main/java/com/gargoylesoftware/htmlunit/WebRequest.java index 86480733101..a3615837ecb 100644 --- a/src/main/java/com/gargoylesoftware/htmlunit/WebRequest.java +++ b/src/main/java/com/gargoylesoftware/htmlunit/WebRequest.java @@ -32,8 +32,8 @@ import java.util.Set; import java.util.regex.Pattern; -import org.apache.http.auth.Credentials; -import org.apache.http.auth.UsernamePasswordCredentials; +import org.apache.hc.client5.http.auth.Credentials; +import org.apache.hc.client5.http.auth.UsernamePasswordCredentials; import com.gargoylesoftware.htmlunit.util.NameValuePair; import com.gargoylesoftware.htmlunit.util.UrlUtils; @@ -165,12 +165,12 @@ else if (path.contains("/.")) { if (userInfo != null) { final int splitPos = userInfo.indexOf(':'); if (splitPos == -1) { - urlCredentials_ = new UsernamePasswordCredentials(userInfo, ""); + urlCredentials_ = new UsernamePasswordCredentials(userInfo, "".toCharArray()); } else { final String username = userInfo.substring(0, splitPos); final String password = userInfo.substring(splitPos + 1); - urlCredentials_ = new UsernamePasswordCredentials(username, password); + urlCredentials_ = new UsernamePasswordCredentials(username, password.toCharArray()); } } } diff --git a/src/main/java/com/gargoylesoftware/htmlunit/activex/javascript/msxml/XMLHTTPRequest.java b/src/main/java/com/gargoylesoftware/htmlunit/activex/javascript/msxml/XMLHTTPRequest.java index d9898634186..c7b9e25d47b 100644 --- a/src/main/java/com/gargoylesoftware/htmlunit/activex/javascript/msxml/XMLHTTPRequest.java +++ b/src/main/java/com/gargoylesoftware/htmlunit/activex/javascript/msxml/XMLHTTPRequest.java @@ -35,7 +35,7 @@ import org.apache.commons.lang3.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.http.auth.UsernamePasswordCredentials; +import org.apache.hc.client5.http.auth.UsernamePasswordCredentials; import com.gargoylesoftware.htmlunit.AjaxController; import com.gargoylesoftware.htmlunit.FormEncodingType; @@ -426,7 +426,7 @@ public void open(final String method, final Object url, final Object asyncParam, passwordCred = password.toString(); } - request.setCredentials(new UsernamePasswordCredentials(userCred, passwordCred)); + request.setCredentials(new UsernamePasswordCredentials(userCred, passwordCred.toCharArray())); } webRequest_ = request; } diff --git a/src/main/java/com/gargoylesoftware/htmlunit/html/HtmlForm.java b/src/main/java/com/gargoylesoftware/htmlunit/html/HtmlForm.java index 64cbc09f09b..5b5c0ae3a96 100644 --- a/src/main/java/com/gargoylesoftware/htmlunit/html/HtmlForm.java +++ b/src/main/java/com/gargoylesoftware/htmlunit/html/HtmlForm.java @@ -39,7 +39,7 @@ import org.apache.commons.lang3.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.http.client.utils.URLEncodedUtils; +import org.apache.hc.core5.net.URLEncodedUtils; import com.gargoylesoftware.htmlunit.BrowserVersion; import com.gargoylesoftware.htmlunit.ElementNotFoundException; diff --git a/src/main/java/com/gargoylesoftware/htmlunit/html/HtmlImage.java b/src/main/java/com/gargoylesoftware/htmlunit/html/HtmlImage.java index ab6b0fbc3b1..0dd80f1c00a 100644 --- a/src/main/java/com/gargoylesoftware/htmlunit/html/HtmlImage.java +++ b/src/main/java/com/gargoylesoftware/htmlunit/html/HtmlImage.java @@ -43,7 +43,7 @@ import org.apache.commons.lang3.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.http.HttpStatus; +import org.apache.hc.core5.http.HttpStatus; import com.gargoylesoftware.htmlunit.BrowserVersion; import com.gargoylesoftware.htmlunit.Page; diff --git a/src/main/java/com/gargoylesoftware/htmlunit/html/HtmlLink.java b/src/main/java/com/gargoylesoftware/htmlunit/html/HtmlLink.java index a5943080354..daa92896ce9 100644 --- a/src/main/java/com/gargoylesoftware/htmlunit/html/HtmlLink.java +++ b/src/main/java/com/gargoylesoftware/htmlunit/html/HtmlLink.java @@ -23,7 +23,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.http.HttpStatus; +import org.apache.hc.core5.http.HttpStatus; import com.gargoylesoftware.htmlunit.BrowserVersion; import com.gargoylesoftware.htmlunit.SgmlPage; diff --git a/src/main/java/com/gargoylesoftware/htmlunit/html/HtmlPage.java b/src/main/java/com/gargoylesoftware/htmlunit/html/HtmlPage.java index ba1295dc1ca..843f61173da 100644 --- a/src/main/java/com/gargoylesoftware/htmlunit/html/HtmlPage.java +++ b/src/main/java/com/gargoylesoftware/htmlunit/html/HtmlPage.java @@ -49,7 +49,7 @@ import org.apache.commons.lang3.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.http.HttpStatus; +import org.apache.hc.core5.http.HttpStatus; import org.w3c.dom.Attr; import org.w3c.dom.Comment; import org.w3c.dom.DOMConfiguration; diff --git a/src/main/java/com/gargoylesoftware/htmlunit/httpclient/HtmlUnitBrowserCompatCookieHeaderValueFormatter.java b/src/main/java/com/gargoylesoftware/htmlunit/httpclient/HtmlUnitBrowserCompatCookieHeaderValueFormatter.java index 4c45b3a421a..fc872d288e7 100644 --- a/src/main/java/com/gargoylesoftware/htmlunit/httpclient/HtmlUnitBrowserCompatCookieHeaderValueFormatter.java +++ b/src/main/java/com/gargoylesoftware/htmlunit/httpclient/HtmlUnitBrowserCompatCookieHeaderValueFormatter.java @@ -14,7 +14,12 @@ */ package com.gargoylesoftware.htmlunit.httpclient; -import org.apache.http.message.BasicHeaderValueFormatter; +import org.apache.hc.core5.http.HeaderElement; +import org.apache.hc.core5.http.NameValuePair; +import org.apache.hc.core5.http.message.BasicHeaderValueFormatter; +import org.apache.hc.core5.http.message.HeaderValueFormatter; +import org.apache.hc.core5.util.Args; +import org.apache.hc.core5.util.CharArrayBuffer; /** * Customized BasicHeaderValueFormatter for HtmlUnit. @@ -23,10 +28,13 @@ * if the value contains special chars. * I guess this is something special for HttpClient because HttpClient also removes * the quotes from cookies (@see {@link HtmlUnitBrowserCompatCookieSpec}) + *

+ * The code is basically copied from {@link BasicHeaderValueFormatter} which does no + * longer allow overwriting the relevant methods. * * @author Ronald Brill */ -public class HtmlUnitBrowserCompatCookieHeaderValueFormatter extends BasicHeaderValueFormatter { +public class HtmlUnitBrowserCompatCookieHeaderValueFormatter implements HeaderValueFormatter { /** * Single instance as in BasicHeaderValueFormatter. @@ -34,21 +42,80 @@ public class HtmlUnitBrowserCompatCookieHeaderValueFormatter extends BasicHeader public static final HtmlUnitBrowserCompatCookieHeaderValueFormatter INSTANCE = new HtmlUnitBrowserCompatCookieHeaderValueFormatter(); - /** - * {@inheritDoc} - * Overwritten to disable automatic addition of quotes. - */ @Override - protected boolean isSeparator(final char ch) { - return false; + public void formatElements( + final CharArrayBuffer buffer, final HeaderElement[] elems, final boolean quote) { + Args.notNull(buffer, "Char array buffer"); + Args.notNull(elems, "Header element array"); + + for (int i = 0; i < elems.length; i++) { + if (i > 0) { + buffer.append(", "); + } + formatHeaderElement(buffer, elems[i], quote); + } } - /** - * Looks like browsers are not doing any escaping. - * {@inheritDoc} - */ @Override - protected boolean isUnsafe(final char ch) { - return false; + public void formatHeaderElement( + final CharArrayBuffer buffer, final HeaderElement elem, final boolean quote) { + Args.notNull(buffer, "Char array buffer"); + Args.notNull(elem, "Header element"); + + buffer.append(elem.getName()); + final String value = elem.getValue(); + if (value != null) { + buffer.append('='); + formatValue(buffer, value, quote); + } + + final int c = elem.getParameterCount(); + if (c > 0) { + for (int i = 0; i < c; i++) { + buffer.append("; "); + formatNameValuePair(buffer, elem.getParameter(i), quote); + } + } + } + + @Override + public void formatParameters( + final CharArrayBuffer buffer, final NameValuePair[] nvps, final boolean quote) { + Args.notNull(buffer, "Char array buffer"); + Args.notNull(nvps, "Header parameter array"); + + for (int i = 0; i < nvps.length; i++) { + if (i > 0) { + buffer.append("; "); + } + formatNameValuePair(buffer, nvps[i], quote); + } + } + + @Override + public void formatNameValuePair( + final CharArrayBuffer buffer, final NameValuePair nvp, final boolean quote) { + Args.notNull(buffer, "Char array buffer"); + Args.notNull(nvp, "Name / value pair"); + + buffer.append(nvp.getName()); + final String value = nvp.getValue(); + if (value != null) { + buffer.append('='); + formatValue(buffer, value, quote); + } + } + + void formatValue(final CharArrayBuffer buffer, final String value, final boolean quote) { + if (quote) { + buffer.append('"'); + } + for (int i = 0; i < value.length(); i++) { + final char ch = value.charAt(i); + buffer.append(ch); + } + if (quote) { + buffer.append('"'); + } } } diff --git a/src/main/java/com/gargoylesoftware/htmlunit/httpclient/HtmlUnitBrowserCompatCookieSpec.java b/src/main/java/com/gargoylesoftware/htmlunit/httpclient/HtmlUnitBrowserCompatCookieSpec.java index 09f0a4934c8..31186cabe86 100644 --- a/src/main/java/com/gargoylesoftware/htmlunit/httpclient/HtmlUnitBrowserCompatCookieSpec.java +++ b/src/main/java/com/gargoylesoftware/htmlunit/httpclient/HtmlUnitBrowserCompatCookieSpec.java @@ -23,30 +23,31 @@ import java.util.Locale; import org.apache.commons.lang3.StringUtils; -import org.apache.http.FormattedHeader; -import org.apache.http.Header; -import org.apache.http.HeaderElement; -import org.apache.http.NameValuePair; -import org.apache.http.client.utils.DateUtils; -import org.apache.http.cookie.Cookie; -import org.apache.http.cookie.CookieAttributeHandler; -import org.apache.http.cookie.CookieOrigin; -import org.apache.http.cookie.CookiePathComparator; -import org.apache.http.cookie.MalformedCookieException; -import org.apache.http.cookie.SM; -import org.apache.http.impl.cookie.BasicClientCookie; -import org.apache.http.impl.cookie.BasicCommentHandler; -import org.apache.http.impl.cookie.BasicMaxAgeHandler; -import org.apache.http.impl.cookie.BasicSecureHandler; -import org.apache.http.impl.cookie.CookieSpecBase; -import org.apache.http.impl.cookie.NetscapeDraftHeaderParser; -import org.apache.http.message.BasicHeader; -import org.apache.http.message.BasicHeaderElement; -import org.apache.http.message.BufferedHeader; -import org.apache.http.message.ParserCursor; -import org.apache.http.util.CharArrayBuffer; +import org.apache.hc.client5.http.cookie.Cookie; +import org.apache.hc.client5.http.cookie.CookieAttributeHandler; +import org.apache.hc.client5.http.cookie.CookieOrigin; +import org.apache.hc.client5.http.cookie.CookiePathComparator; +import org.apache.hc.client5.http.cookie.MalformedCookieException; +import org.apache.hc.client5.http.impl.cookie.BasicClientCookie; +import org.apache.hc.client5.http.impl.cookie.BasicMaxAgeHandler; +import org.apache.hc.client5.http.impl.cookie.BasicSecureHandler; +import org.apache.hc.client5.http.impl.cookie.CookieSpecBase; +import org.apache.hc.client5.http.utils.DateUtils; +import org.apache.hc.core5.http.FormattedHeader; +import org.apache.hc.core5.http.Header; +import org.apache.hc.core5.http.HeaderElement; +import org.apache.hc.core5.http.HttpHeaders; +import org.apache.hc.core5.http.NameValuePair; +import org.apache.hc.core5.http.ParseException; +import org.apache.hc.core5.http.message.BasicHeader; +import org.apache.hc.core5.http.message.BasicHeaderElement; +import org.apache.hc.core5.http.message.BasicHeaderValueParser; +import org.apache.hc.core5.http.message.BufferedHeader; +import org.apache.hc.core5.http.message.ParserCursor; +import org.apache.hc.core5.util.CharArrayBuffer; import com.gargoylesoftware.htmlunit.BrowserVersion; +import com.gargoylesoftware.htmlunit.NetscapeDraftHeaderParser; /** * Customized BrowserCompatSpec for HtmlUnit. @@ -96,12 +97,14 @@ public class HtmlUnitBrowserCompatCookieSpec extends CookieSpecBase { * @param browserVersion the {@link BrowserVersion} to simulate */ public HtmlUnitBrowserCompatCookieSpec(final BrowserVersion browserVersion) { - super(new HtmlUnitVersionAttributeHandler(), + super(// TODO + //new HtmlUnitVersionAttributeHandler(), new HtmlUnitDomainHandler(browserVersion), new HtmlUnitPathHandler(browserVersion), new BasicMaxAgeHandler(), new BasicSecureHandler(), - new BasicCommentHandler(), + // TODO + //new BasicCommentHandler(), new HtmlUnitExpiresHandler(browserVersion), new HtmlUnitHttpOnlyHandler(), new HtmlUnitSameSiteHandler()); @@ -137,10 +140,12 @@ else if (endPos == 0 || StringUtils.isBlank(text.substring(0, endPos))) { final List cookies; final String headername = header.getName(); - if (!headername.equalsIgnoreCase(SM.SET_COOKIE)) { + if (!headername.equalsIgnoreCase(HttpHeaders.SET_COOKIE)) { throw new MalformedCookieException("Unrecognized cookie header '" + header + "'"); } - final HeaderElement[] helems = header.getElements(); + final String headerValue = header.getValue(); + final ParserCursor cursor2 = new ParserCursor(0, headerValue.length()); + final HeaderElement[] helems = BasicHeaderValueParser.INSTANCE.parseElements(headerValue, cursor2); boolean versioned = false; boolean netscape = false; for (final HeaderElement helem: helems) { @@ -151,6 +156,7 @@ else if (endPos == 0 || StringUtils.isBlank(text.substring(0, endPos))) { netscape = true; } } + // TODO if (netscape || !versioned) { // Need to parse the header again, because Netscape style cookies do not correctly // support multiple header elements (comma cannot be treated as an element separator) @@ -172,7 +178,13 @@ else if (endPos == 0 || StringUtils.isBlank(text.substring(0, endPos))) { buffer.append(s); cursor = new ParserCursor(0, buffer.length()); } - final HeaderElement elem = parser.parseHeader(buffer, cursor); + HeaderElement elem; + try { + elem = parser.parseHeader(buffer, cursor); + } + catch (ParseException e) { + throw new RuntimeException(e); + } final String name = elem.getName(); final String value = elem.getValue(); if (name == null || name.isEmpty()) { @@ -195,7 +207,8 @@ else if (endPos == 0 || StringUtils.isBlank(text.substring(0, endPos))) { } // Override version for Netscape style cookies if (netscape) { - cookie.setVersion(0); + // TODO + //cookie.setVersion(0); } cookies = Collections.singletonList(cookie); } @@ -217,7 +230,7 @@ public List

formatCookies(final List cookies) { Collections.sort(cookies, COOKIE_COMPARATOR); final CharArrayBuffer buffer = new CharArrayBuffer(20 * cookies.size()); - buffer.append(SM.COOKIE); + buffer.append(HttpHeaders.COOKIE); buffer.append(": "); for (int i = 0; i < cookies.size(); i++) { final Cookie cookie = cookies.get(i); @@ -226,13 +239,16 @@ public List
formatCookies(final List cookies) { } final String cookieName = cookie.getName(); final String cookieValue = cookie.getValue(); - if (cookie.getVersion() > 0 && !isQuoteEnclosed(cookieValue)) { + // TODO + //if (cookie.getVersion() > 0 && !isQuoteEnclosed(cookieValue)) { + if (!isQuoteEnclosed(cookieValue)) { HtmlUnitBrowserCompatCookieHeaderValueFormatter.INSTANCE.formatHeaderElement( buffer, new BasicHeaderElement(cookieName, cookieValue), false); } else { + // TODO // Netscape style cookies do not support quoted values buffer.append(cookieName); buffer.append("="); @@ -242,7 +258,7 @@ public List
formatCookies(final List cookies) { } } final List
headers = new ArrayList<>(1); - headers.add(new BufferedHeader(buffer)); + headers.add(BufferedHeader.create(buffer)); return headers; } @@ -253,16 +269,6 @@ private static boolean isQuoteEnclosed(final String s) { && '\"' == s.charAt(s.length() - 1); } - @Override - public int getVersion() { - return 0; - } - - @Override - public Header getVersionHeader() { - return null; - } - @Override public String toString() { return "compatibility"; diff --git a/src/main/java/com/gargoylesoftware/htmlunit/httpclient/HtmlUnitCookieSpecProvider.java b/src/main/java/com/gargoylesoftware/htmlunit/httpclient/HtmlUnitCookieSpecProvider.java index 93dbb2224ed..e175609c6dd 100644 --- a/src/main/java/com/gargoylesoftware/htmlunit/httpclient/HtmlUnitCookieSpecProvider.java +++ b/src/main/java/com/gargoylesoftware/htmlunit/httpclient/HtmlUnitCookieSpecProvider.java @@ -14,9 +14,9 @@ */ package com.gargoylesoftware.htmlunit.httpclient; -import org.apache.http.cookie.CookieSpec; -import org.apache.http.cookie.CookieSpecProvider; -import org.apache.http.protocol.HttpContext; +import org.apache.hc.client5.http.cookie.CookieSpec; +import org.apache.hc.client5.http.cookie.CookieSpecFactory; +import org.apache.hc.core5.http.protocol.HttpContext; import com.gargoylesoftware.htmlunit.BrowserVersion; @@ -25,7 +25,7 @@ * * @author Ronald Brill */ -public final class HtmlUnitCookieSpecProvider implements CookieSpecProvider { +public final class HtmlUnitCookieSpecProvider implements CookieSpecFactory { private final BrowserVersion browserVersion_; diff --git a/src/main/java/com/gargoylesoftware/htmlunit/httpclient/HtmlUnitCookieStore.java b/src/main/java/com/gargoylesoftware/htmlunit/httpclient/HtmlUnitCookieStore.java index 1c6087903b0..5f7b5c75227 100644 --- a/src/main/java/com/gargoylesoftware/htmlunit/httpclient/HtmlUnitCookieStore.java +++ b/src/main/java/com/gargoylesoftware/htmlunit/httpclient/HtmlUnitCookieStore.java @@ -18,14 +18,13 @@ import java.util.Date; import java.util.List; -import org.apache.http.client.CookieStore; -import org.apache.http.cookie.ClientCookie; -import org.apache.http.cookie.Cookie; +import org.apache.hc.client5.http.cookie.Cookie; +import org.apache.hc.client5.http.cookie.CookieStore; import com.gargoylesoftware.htmlunit.CookieManager; /** - * Implementation of {@link CookieStore} like {@link org.apache.http.impl.client.BasicCookieStore} + * Implementation of {@link CookieStore} like {@link org.apache.hc.client5.http.cookie.BasicCookieStore} * BUT using our own {@link CookieManager} as back end. * * @author Marc Guillemot @@ -48,7 +47,7 @@ public HtmlUnitCookieStore(final CookieManager manager) { */ @Override public synchronized void addCookie(final Cookie cookie) { - manager_.addCookie(new com.gargoylesoftware.htmlunit.util.Cookie((ClientCookie) cookie)); + manager_.addCookie(new com.gargoylesoftware.htmlunit.util.Cookie(cookie)); } /** diff --git a/src/main/java/com/gargoylesoftware/htmlunit/httpclient/HtmlUnitDomainHandler.java b/src/main/java/com/gargoylesoftware/htmlunit/httpclient/HtmlUnitDomainHandler.java index 70fd8cbff18..b2fd2827882 100644 --- a/src/main/java/com/gargoylesoftware/htmlunit/httpclient/HtmlUnitDomainHandler.java +++ b/src/main/java/com/gargoylesoftware/htmlunit/httpclient/HtmlUnitDomainHandler.java @@ -20,13 +20,13 @@ import java.net.UnknownHostException; import java.util.Locale; -import org.apache.http.cookie.Cookie; -import org.apache.http.cookie.CookieOrigin; -import org.apache.http.cookie.MalformedCookieException; -import org.apache.http.cookie.SetCookie; -import org.apache.http.impl.cookie.BasicDomainHandler; -import org.apache.http.util.Args; -import org.apache.http.util.TextUtils; +import org.apache.hc.client5.http.cookie.Cookie; +import org.apache.hc.client5.http.cookie.CookieOrigin; +import org.apache.hc.client5.http.cookie.MalformedCookieException; +import org.apache.hc.client5.http.cookie.SetCookie; +import org.apache.hc.client5.http.impl.cookie.BasicDomainHandler; +import org.apache.hc.core5.util.Args; +import org.apache.hc.core5.util.TextUtils; import com.gargoylesoftware.htmlunit.BrowserVersion; import com.gargoylesoftware.htmlunit.HttpHeader; diff --git a/src/main/java/com/gargoylesoftware/htmlunit/httpclient/HtmlUnitExpiresHandler.java b/src/main/java/com/gargoylesoftware/htmlunit/httpclient/HtmlUnitExpiresHandler.java index 32262239c72..a906ab568aa 100644 --- a/src/main/java/com/gargoylesoftware/htmlunit/httpclient/HtmlUnitExpiresHandler.java +++ b/src/main/java/com/gargoylesoftware/htmlunit/httpclient/HtmlUnitExpiresHandler.java @@ -20,10 +20,10 @@ import java.util.Date; -import org.apache.http.client.utils.DateUtils; -import org.apache.http.cookie.MalformedCookieException; -import org.apache.http.cookie.SetCookie; -import org.apache.http.impl.cookie.BasicExpiresHandler; +import org.apache.hc.client5.http.cookie.MalformedCookieException; +import org.apache.hc.client5.http.cookie.SetCookie; +import org.apache.hc.client5.http.impl.cookie.BasicExpiresHandler; +import org.apache.hc.client5.http.utils.DateUtils; import com.gargoylesoftware.htmlunit.BrowserVersion; diff --git a/src/main/java/com/gargoylesoftware/htmlunit/httpclient/HtmlUnitHttpOnlyHandler.java b/src/main/java/com/gargoylesoftware/htmlunit/httpclient/HtmlUnitHttpOnlyHandler.java index 9c70672b287..d94e88a1eb1 100644 --- a/src/main/java/com/gargoylesoftware/htmlunit/httpclient/HtmlUnitHttpOnlyHandler.java +++ b/src/main/java/com/gargoylesoftware/htmlunit/httpclient/HtmlUnitHttpOnlyHandler.java @@ -14,6 +14,13 @@ */ package com.gargoylesoftware.htmlunit.httpclient; +import org.apache.hc.client5.http.cookie.CommonCookieAttributeHandler; +import org.apache.hc.client5.http.cookie.Cookie; +import org.apache.hc.client5.http.cookie.CookieOrigin; +import org.apache.hc.client5.http.cookie.MalformedCookieException; +import org.apache.hc.client5.http.cookie.SetCookie; +import org.apache.hc.client5.http.impl.cookie.BasicClientCookie; + /** * Customized CookieAttributeHandler for handling of the httponly attribute. * @@ -27,13 +34,6 @@ * @author Ronald Brill * @author John J Murdoch */ -import org.apache.http.cookie.CommonCookieAttributeHandler; -import org.apache.http.cookie.Cookie; -import org.apache.http.cookie.CookieOrigin; -import org.apache.http.cookie.MalformedCookieException; -import org.apache.http.cookie.SetCookie; -import org.apache.http.impl.cookie.BasicClientCookie; - final class HtmlUnitHttpOnlyHandler implements CommonCookieAttributeHandler { private static final String HTTPONLY_ATTR = "httponly"; diff --git a/src/main/java/com/gargoylesoftware/htmlunit/httpclient/HtmlUnitPathHandler.java b/src/main/java/com/gargoylesoftware/htmlunit/httpclient/HtmlUnitPathHandler.java index d0bb4205b3a..635ff843aa2 100644 --- a/src/main/java/com/gargoylesoftware/htmlunit/httpclient/HtmlUnitPathHandler.java +++ b/src/main/java/com/gargoylesoftware/htmlunit/httpclient/HtmlUnitPathHandler.java @@ -16,10 +16,10 @@ import static com.gargoylesoftware.htmlunit.BrowserVersionFeatures.HTTP_COOKIE_EXTRACT_PATH_FROM_LOCATION; -import org.apache.http.cookie.Cookie; -import org.apache.http.cookie.CookieOrigin; -import org.apache.http.cookie.MalformedCookieException; -import org.apache.http.impl.cookie.BasicPathHandler; +import org.apache.hc.client5.http.cookie.Cookie; +import org.apache.hc.client5.http.cookie.CookieOrigin; +import org.apache.hc.client5.http.cookie.MalformedCookieException; +import org.apache.hc.client5.http.impl.cookie.BasicPathHandler; import com.gargoylesoftware.htmlunit.BrowserVersion; diff --git a/src/main/java/com/gargoylesoftware/htmlunit/httpclient/HtmlUnitRedirectStrategie.java b/src/main/java/com/gargoylesoftware/htmlunit/httpclient/HtmlUnitRedirectStrategie.java index 2f8dbb30275..fe3369c8b7d 100644 --- a/src/main/java/com/gargoylesoftware/htmlunit/httpclient/HtmlUnitRedirectStrategie.java +++ b/src/main/java/com/gargoylesoftware/htmlunit/httpclient/HtmlUnitRedirectStrategie.java @@ -14,11 +14,11 @@ */ package com.gargoylesoftware.htmlunit.httpclient; -import org.apache.http.HttpRequest; -import org.apache.http.HttpResponse; -import org.apache.http.ProtocolException; -import org.apache.http.impl.client.DefaultRedirectStrategy; -import org.apache.http.protocol.HttpContext; +import org.apache.hc.client5.http.impl.DefaultRedirectStrategy; +import org.apache.hc.core5.http.HttpRequest; +import org.apache.hc.core5.http.HttpResponse; +import org.apache.hc.core5.http.ProtocolException; +import org.apache.hc.core5.http.protocol.HttpContext; /** * Customized DefaultRedirectStrategy for HtmlUnit. diff --git a/src/main/java/com/gargoylesoftware/htmlunit/httpclient/HtmlUnitSSLConnectionSocketFactory.java b/src/main/java/com/gargoylesoftware/htmlunit/httpclient/HtmlUnitSSLConnectionSocketFactory.java index 0741ebdc639..d10fd1fc281 100644 --- a/src/main/java/com/gargoylesoftware/htmlunit/httpclient/HtmlUnitSSLConnectionSocketFactory.java +++ b/src/main/java/com/gargoylesoftware/htmlunit/httpclient/HtmlUnitSSLConnectionSocketFactory.java @@ -40,13 +40,15 @@ import javax.net.ssl.SSLSocket; import javax.net.ssl.X509ExtendedTrustManager; -import org.apache.http.HttpHost; -import org.apache.http.conn.ConnectTimeoutException; -import org.apache.http.conn.ssl.DefaultHostnameVerifier; -import org.apache.http.conn.ssl.NoopHostnameVerifier; -import org.apache.http.conn.ssl.SSLConnectionSocketFactory; -import org.apache.http.protocol.HttpContext; -import org.apache.http.ssl.SSLContexts; +import org.apache.hc.client5.http.ConnectTimeoutException; +import org.apache.hc.client5.http.ssl.DefaultHostnameVerifier; +import org.apache.hc.client5.http.ssl.NoopHostnameVerifier; +import org.apache.hc.client5.http.ssl.SSLConnectionSocketFactory; +import org.apache.hc.core5.http.HttpHost; +import org.apache.hc.core5.http.protocol.HttpContext; +import org.apache.hc.core5.ssl.SSLContexts; +import org.apache.hc.core5.util.TimeValue; +import org.apache.hc.core5.util.TimeValue; import com.gargoylesoftware.htmlunit.WebClientOptions; @@ -154,7 +156,7 @@ private static void configureSocket(final SSLSocket sslSocket, final HttpContext */ @Override public Socket connectSocket( - final int connectTimeout, + final TimeValue connectTimeout, final Socket socket, final HttpHost host, final InetSocketAddress remoteAddress, @@ -169,7 +171,7 @@ public Socket connectSocket( socksProxy.getPort()); try { //underlying.setSoTimeout(soTimeout); - underlying.connect(remoteAddress, connectTimeout); + underlying.connect(remoteAddress, connectTimeout.toMillisecondsIntBound()); } catch (final SocketTimeoutException ex) { throw new ConnectTimeoutException("Connect to " + socksProxyAddress + " timed out"); diff --git a/src/main/java/com/gargoylesoftware/htmlunit/httpclient/HtmlUnitSameSiteHandler.java b/src/main/java/com/gargoylesoftware/htmlunit/httpclient/HtmlUnitSameSiteHandler.java index c0ca3c74d04..bb0121909cb 100644 --- a/src/main/java/com/gargoylesoftware/htmlunit/httpclient/HtmlUnitSameSiteHandler.java +++ b/src/main/java/com/gargoylesoftware/htmlunit/httpclient/HtmlUnitSameSiteHandler.java @@ -15,19 +15,18 @@ package com.gargoylesoftware.htmlunit.httpclient; import org.apache.commons.lang3.StringUtils; +import org.apache.hc.client5.http.cookie.CommonCookieAttributeHandler; +import org.apache.hc.client5.http.cookie.Cookie; +import org.apache.hc.client5.http.cookie.CookieOrigin; +import org.apache.hc.client5.http.cookie.MalformedCookieException; +import org.apache.hc.client5.http.cookie.SetCookie; +import org.apache.hc.client5.http.impl.cookie.BasicClientCookie; /** * Customized CookieAttributeHandler for handling of the samesite attribute. * * @author Ronald Brill */ -import org.apache.http.cookie.CommonCookieAttributeHandler; -import org.apache.http.cookie.Cookie; -import org.apache.http.cookie.CookieOrigin; -import org.apache.http.cookie.MalformedCookieException; -import org.apache.http.cookie.SetCookie; -import org.apache.http.impl.cookie.BasicClientCookie; - final class HtmlUnitSameSiteHandler implements CommonCookieAttributeHandler { private static final String SAMESITE_ATTR = "samesite"; diff --git a/src/main/java/com/gargoylesoftware/htmlunit/httpclient/HtmlUnitVersionAttributeHandler.java b/src/main/java/com/gargoylesoftware/htmlunit/httpclient/HtmlUnitVersionAttributeHandler.java index a9d9517ec03..6f46847d071 100644 --- a/src/main/java/com/gargoylesoftware/htmlunit/httpclient/HtmlUnitVersionAttributeHandler.java +++ b/src/main/java/com/gargoylesoftware/htmlunit/httpclient/HtmlUnitVersionAttributeHandler.java @@ -14,11 +14,10 @@ */ package com.gargoylesoftware.htmlunit.httpclient; -import org.apache.http.cookie.ClientCookie; -import org.apache.http.cookie.CommonCookieAttributeHandler; -import org.apache.http.cookie.MalformedCookieException; -import org.apache.http.cookie.SetCookie; -import org.apache.http.impl.cookie.AbstractCookieAttributeHandler; +import org.apache.hc.client5.http.cookie.CommonCookieAttributeHandler; +import org.apache.hc.client5.http.cookie.MalformedCookieException; +import org.apache.hc.client5.http.cookie.SetCookie; +import org.apache.hc.client5.http.impl.cookie.AbstractCookieAttributeHandler; /** * VersionAttributeHandler for HtmlUnit. @@ -43,11 +42,14 @@ public void parse(final SetCookie cookie, final String value) throws MalformedCo catch (final NumberFormatException e) { // ignore invalid versions } - cookie.setVersion(version); + // TODO + //cookie.setVersion(version); } @Override public String getAttributeName() { - return ClientCookie.VERSION_ATTR; + // TODO + //return SetCookie.VERSION_ATTR; + return null; } } diff --git a/src/main/java/com/gargoylesoftware/htmlunit/httpclient/SocksConnectionSocketFactory.java b/src/main/java/com/gargoylesoftware/htmlunit/httpclient/SocksConnectionSocketFactory.java index 989590f9fdf..5e4ac8c89da 100644 --- a/src/main/java/com/gargoylesoftware/htmlunit/httpclient/SocksConnectionSocketFactory.java +++ b/src/main/java/com/gargoylesoftware/htmlunit/httpclient/SocksConnectionSocketFactory.java @@ -19,12 +19,12 @@ import java.net.Proxy; import java.net.Socket; -import org.apache.http.HttpHost; -import org.apache.http.conn.socket.PlainConnectionSocketFactory; -import org.apache.http.protocol.HttpContext; +import org.apache.hc.client5.http.socket.PlainConnectionSocketFactory; +import org.apache.hc.core5.http.HttpHost; +import org.apache.hc.core5.http.protocol.HttpContext; /** - * SOCKS aware {@link org.apache.http.conn.socket.ConnectionSocketFactory}. + * SOCKS aware {@link org.apache.hc.client5.http.socket.ConnectionSocketFactory}. * * @author Ahmed Ashour * @author Ronald Brill diff --git a/src/main/java/com/gargoylesoftware/htmlunit/javascript/host/URLSearchParams.java b/src/main/java/com/gargoylesoftware/htmlunit/javascript/host/URLSearchParams.java index e3202f3502c..31ee913bb08 100644 --- a/src/main/java/com/gargoylesoftware/htmlunit/javascript/host/URLSearchParams.java +++ b/src/main/java/com/gargoylesoftware/htmlunit/javascript/host/URLSearchParams.java @@ -20,6 +20,7 @@ import static com.gargoylesoftware.htmlunit.javascript.configuration.SupportedBrowser.FF; import static com.gargoylesoftware.htmlunit.javascript.configuration.SupportedBrowser.FF78; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Collections; import java.util.Iterator; @@ -27,7 +28,7 @@ import java.util.ListIterator; import org.apache.commons.lang3.StringUtils; -import org.apache.http.client.utils.URLEncodedUtils; +import org.apache.hc.core5.net.URLEncodedUtils; import com.gargoylesoftware.htmlunit.FormEncodingType; import com.gargoylesoftware.htmlunit.WebRequest; @@ -333,7 +334,7 @@ public Object values() { */ @JsxFunction(functionName = "toString") public String jsToString() { - return URLEncodedUtils.format(NameValuePair.toHttpClient(params_), "UTF-8"); + return URLEncodedUtils.format(NameValuePair.toHttpClient(params_), StandardCharsets.UTF_8); } /** diff --git a/src/main/java/com/gargoylesoftware/htmlunit/javascript/host/xml/XMLHttpRequest.java b/src/main/java/com/gargoylesoftware/htmlunit/javascript/host/xml/XMLHttpRequest.java index a4e3856c5b3..8766942a534 100644 --- a/src/main/java/com/gargoylesoftware/htmlunit/javascript/host/xml/XMLHttpRequest.java +++ b/src/main/java/com/gargoylesoftware/htmlunit/javascript/host/xml/XMLHttpRequest.java @@ -51,9 +51,9 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; -import org.apache.http.HttpStatus; -import org.apache.http.NoHttpResponseException; -import org.apache.http.auth.UsernamePasswordCredentials; +import org.apache.hc.client5.http.auth.UsernamePasswordCredentials; +import org.apache.hc.core5.http.HttpStatus; +import org.apache.hc.core5.http.NoHttpResponseException; import com.gargoylesoftware.htmlunit.AjaxController; import com.gargoylesoftware.htmlunit.BrowserVersion; @@ -517,7 +517,7 @@ public void open(final String method, final Object urlParam, final Object asyncP passwordCred = password.toString(); } - request.setCredentials(new UsernamePasswordCredentials(userCred, passwordCred)); + request.setCredentials(new UsernamePasswordCredentials(userCred, passwordCred.toCharArray())); } webRequest_ = request; } diff --git a/src/main/java/com/gargoylesoftware/htmlunit/util/Cookie.java b/src/main/java/com/gargoylesoftware/htmlunit/util/Cookie.java index c9969705b0c..01aab29b907 100644 --- a/src/main/java/com/gargoylesoftware/htmlunit/util/Cookie.java +++ b/src/main/java/com/gargoylesoftware/htmlunit/util/Cookie.java @@ -23,8 +23,7 @@ import org.apache.commons.lang3.builder.EqualsBuilder; import org.apache.commons.lang3.builder.HashCodeBuilder; -import org.apache.http.cookie.ClientCookie; -import org.apache.http.impl.cookie.BasicClientCookie; +import org.apache.hc.client5.http.impl.cookie.BasicClientCookie; /** * A cookie. This class is immutable. @@ -36,7 +35,7 @@ */ public class Cookie implements Serializable { - private ClientCookie httpClientCookie_; + private org.apache.hc.client5.http.cookie.Cookie httpClientCookie_; /** * Creates a new cookie with the specified name and value which applies to the specified domain. @@ -85,7 +84,7 @@ public Cookie(final String domain, final String name, final String value, final cookie.setDomain(domain); // BasicDomainHandler.match(Cookie, CookieOrigin) checks the attib also (see #333) - cookie.setAttribute(ClientCookie.DOMAIN_ATTR, domain); + cookie.setAttribute(BasicClientCookie.DOMAIN_ATTR, domain); cookie.setPath(path); cookie.setExpiryDate(expires); @@ -101,7 +100,7 @@ public Cookie(final String domain, final String name, final String value, final * Creates a new HtmlUnit cookie from the HttpClient cookie provided. * @param clientCookie the HttpClient cookie */ - public Cookie(final ClientCookie clientCookie) { + public Cookie(final org.apache.hc.client5.http.cookie.Cookie clientCookie) { httpClientCookie_ = clientCookie; } @@ -246,7 +245,7 @@ public int hashCode() { * Converts this cookie to an HttpClient cookie. * @return an HttpClient version of this cookie */ - public org.apache.http.cookie.Cookie toHttpClient() { + public org.apache.hc.client5.http.cookie.Cookie toHttpClient() { return httpClientCookie_; } @@ -255,8 +254,8 @@ public org.apache.http.cookie.Cookie toHttpClient() { * @param cookies the cookies to be converted * @return the specified cookies, as HttpClient cookies */ - public static List toHttpClient(final Collection cookies) { - final ArrayList array = new ArrayList<>(cookies.size()); + public static List toHttpClient(final Collection cookies) { + final ArrayList array = new ArrayList<>(cookies.size()); final Iterator it = cookies.iterator(); while (it.hasNext()) { array.add(it.next().toHttpClient()); @@ -269,10 +268,10 @@ public static List toHttpClient(final Collection< * @param cookies the cookies to be converted * @return the specified HttpClient cookies, as cookies */ - public static List fromHttpClient(final List cookies) { + public static List fromHttpClient(final List cookies) { final List list = new ArrayList<>(cookies.size()); - for (final org.apache.http.cookie.Cookie c : cookies) { - list.add(new Cookie((ClientCookie) c)); + for (final org.apache.hc.client5.http.cookie.Cookie c : cookies) { + list.add(new Cookie(c)); } return list; } diff --git a/src/main/java/com/gargoylesoftware/htmlunit/util/NameValuePair.java b/src/main/java/com/gargoylesoftware/htmlunit/util/NameValuePair.java index 5c3b6a15087..be597cbc5ea 100644 --- a/src/main/java/com/gargoylesoftware/htmlunit/util/NameValuePair.java +++ b/src/main/java/com/gargoylesoftware/htmlunit/util/NameValuePair.java @@ -18,8 +18,8 @@ import java.util.ArrayList; import java.util.List; -import org.apache.http.message.BasicNameValuePair; -import org.apache.http.util.LangUtils; +import org.apache.hc.core5.http.message.BasicNameValuePair; +import org.apache.hc.core5.util.LangUtils; /** * A name/value pair. @@ -98,9 +98,9 @@ public String toString() { * @param pairs the name/value pairs to convert * @return the converted name/value pairs */ - public static org.apache.http.NameValuePair[] toHttpClient(final NameValuePair[] pairs) { - final org.apache.http.NameValuePair[] pairs2 = - new org.apache.http.NameValuePair[pairs.length]; + public static org.apache.hc.core5.http.NameValuePair[] toHttpClient(final NameValuePair[] pairs) { + final org.apache.hc.core5.http.NameValuePair[] pairs2 = + new org.apache.hc.core5.http.NameValuePair[pairs.length]; for (int i = 0; i < pairs.length; i++) { final NameValuePair pair = pairs[i]; pairs2[i] = new BasicNameValuePair(pair.getName(), pair.getValue()); @@ -113,8 +113,8 @@ public static org.apache.http.NameValuePair[] toHttpClient(final NameValuePair[] * @param pairs the name/value pairs to convert * @return the converted name/value pairs */ - public static List toHttpClient(final List pairs) { - final List resultingPairs = new ArrayList<>(pairs.size()); + public static List toHttpClient(final List pairs) { + final List resultingPairs = new ArrayList<>(pairs.size()); for (int i = 0; i < pairs.size(); i++) { final NameValuePair pair = pairs.get(i); resultingPairs.add(new BasicNameValuePair(pair.getName(), pair.getValue())); diff --git a/src/main/java/com/gargoylesoftware/htmlunit/util/StringUtils.java b/src/main/java/com/gargoylesoftware/htmlunit/util/StringUtils.java index 969eaa88da1..d8a07cc3652 100644 --- a/src/main/java/com/gargoylesoftware/htmlunit/util/StringUtils.java +++ b/src/main/java/com/gargoylesoftware/htmlunit/util/StringUtils.java @@ -18,7 +18,7 @@ import java.util.regex.Matcher; import java.util.regex.Pattern; -import org.apache.http.client.utils.DateUtils; +import org.apache.hc.client5.http.utils.DateUtils; import com.gargoylesoftware.htmlunit.html.impl.Color; diff --git a/src/test/java/com/gargoylesoftware/htmlunit/AbstractPageTest.java b/src/test/java/com/gargoylesoftware/htmlunit/AbstractPageTest.java index 9c72d861a82..baeef59c536 100644 --- a/src/test/java/com/gargoylesoftware/htmlunit/AbstractPageTest.java +++ b/src/test/java/com/gargoylesoftware/htmlunit/AbstractPageTest.java @@ -14,7 +14,7 @@ */ package com.gargoylesoftware.htmlunit; -import static org.apache.http.client.utils.DateUtils.formatDate; +import static org.apache.hc.client5.http.utils.DateUtils.formatDate; import java.util.ArrayList; import java.util.Date; diff --git a/src/test/java/com/gargoylesoftware/htmlunit/CacheTest.java b/src/test/java/com/gargoylesoftware/htmlunit/CacheTest.java index a3784c88a1b..97be9525764 100644 --- a/src/test/java/com/gargoylesoftware/htmlunit/CacheTest.java +++ b/src/test/java/com/gargoylesoftware/htmlunit/CacheTest.java @@ -14,7 +14,7 @@ */ package com.gargoylesoftware.htmlunit; -import static org.apache.http.client.utils.DateUtils.formatDate; +import static org.apache.hc.client5.http.utils.DateUtils.formatDate; import static org.easymock.EasyMock.createMock; import static org.easymock.EasyMock.expect; import static org.easymock.EasyMock.expectLastCall; diff --git a/src/test/java/com/gargoylesoftware/htmlunit/CookieManager3Test.java b/src/test/java/com/gargoylesoftware/htmlunit/CookieManager3Test.java index 081c8e467ed..8b6ea872ab7 100644 --- a/src/test/java/com/gargoylesoftware/htmlunit/CookieManager3Test.java +++ b/src/test/java/com/gargoylesoftware/htmlunit/CookieManager3Test.java @@ -21,10 +21,11 @@ import java.util.Date; import java.util.List; -import org.apache.http.Header; -import org.apache.http.cookie.CookieOrigin; -import org.apache.http.impl.cookie.DefaultCookieSpec; -import org.apache.http.message.BasicHeader; +import org.apache.hc.client5.http.cookie.CookieOrigin; +import org.apache.hc.client5.http.cookie.CookieSpec; +import org.apache.hc.client5.http.impl.cookie.RFC6265StrictSpec; +import org.apache.hc.core5.http.Header; +import org.apache.hc.core5.http.message.BasicHeader; import org.junit.Test; import com.gargoylesoftware.htmlunit.util.Cookie; @@ -106,9 +107,11 @@ public void basicBehavior() { @Test public void httpClientParsesCookiesQuotedValuesCorrectly() throws Exception { final Header header = new BasicHeader("Set-Cookie", "first=\"hello world\""); - final DefaultCookieSpec spec = new DefaultCookieSpec(); + // TODO + //final DefaultCookieSpec spec = new DefaultCookieSpec(); + final CookieSpec spec = new RFC6265StrictSpec(); final CookieOrigin origin = new CookieOrigin("localhost", 80, "/", false); - final List list = spec.parse(header, origin); + final List list = spec.parse(header, origin); assertEquals(1, list.size()); assertEquals("\"hello world\"", list.get(0).getValue()); } diff --git a/src/test/java/com/gargoylesoftware/htmlunit/CookieManager5Test.java b/src/test/java/com/gargoylesoftware/htmlunit/CookieManager5Test.java index 1e06ad8fdf3..f897a4a77f7 100644 --- a/src/test/java/com/gargoylesoftware/htmlunit/CookieManager5Test.java +++ b/src/test/java/com/gargoylesoftware/htmlunit/CookieManager5Test.java @@ -26,7 +26,7 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; -import org.apache.http.client.utils.DateUtils; +import org.apache.hc.client5.http.utils.DateUtils; import org.junit.Test; import org.junit.runner.RunWith; diff --git a/src/test/java/com/gargoylesoftware/htmlunit/CookieManagerTest.java b/src/test/java/com/gargoylesoftware/htmlunit/CookieManagerTest.java index 1202301df15..296f4f5eb92 100644 --- a/src/test/java/com/gargoylesoftware/htmlunit/CookieManagerTest.java +++ b/src/test/java/com/gargoylesoftware/htmlunit/CookieManagerTest.java @@ -22,7 +22,7 @@ import java.util.List; import java.util.Map; -import org.apache.http.client.utils.DateUtils; +import org.apache.hc.client5.http.utils.DateUtils; import org.junit.After; import org.junit.Test; import org.junit.runner.RunWith; diff --git a/src/test/java/com/gargoylesoftware/htmlunit/DefaultCredentialsProvider2Test.java b/src/test/java/com/gargoylesoftware/htmlunit/DefaultCredentialsProvider2Test.java index 524de695210..85e9c779b25 100644 --- a/src/test/java/com/gargoylesoftware/htmlunit/DefaultCredentialsProvider2Test.java +++ b/src/test/java/com/gargoylesoftware/htmlunit/DefaultCredentialsProvider2Test.java @@ -60,7 +60,7 @@ public void basicAuthenticationWrongUserName() throws Exception { // wrong user name getWebClient().getCredentialsProvider().clear(); - ((DefaultCredentialsProvider) getWebClient().getCredentialsProvider()).addCredentials("joe", "jetty"); + ((DefaultCredentialsProvider) getWebClient().getCredentialsProvider()).addCredentials("joe", "jetty".toCharArray()); try { loadPage("Hi There"); @@ -80,7 +80,7 @@ public void basicAuthenticationWrongPassword() throws Exception { // wrong user name getWebClient().getCredentialsProvider().clear(); - ((DefaultCredentialsProvider) getWebClient().getCredentialsProvider()).addCredentials("jetty", "secret"); + ((DefaultCredentialsProvider) getWebClient().getCredentialsProvider()).addCredentials("jetty", "secret".toCharArray()); try { loadPage("Hi There"); @@ -111,7 +111,7 @@ public void basicAuthentication_singleAuthenticaiton() throws Exception { logger.addAppender(writerAppender); try { - ((DefaultCredentialsProvider) getWebClient().getCredentialsProvider()).addCredentials("jetty", "jetty"); + ((DefaultCredentialsProvider) getWebClient().getCredentialsProvider()).addCredentials("jetty", "jetty".toCharArray()); loadPage("Hi There"); int unauthorizedCount = StringUtils.countMatches(stringWriter.toString(), "HTTP/1.1 401"); @@ -310,7 +310,7 @@ public void basicAuthenticationUserFromUrlOverwriteDefaultCredentials() throws E getMockWebConnection().setDefaultResponse(html); getWebClient().getCredentialsProvider().clear(); - ((DefaultCredentialsProvider) getWebClient().getCredentialsProvider()).addCredentials("jetty", "jetty"); + ((DefaultCredentialsProvider) getWebClient().getCredentialsProvider()).addCredentials("jetty", "jetty".toCharArray()); // use default credentials loadPageWithAlerts(URL_FIRST); @@ -345,7 +345,7 @@ public void basicAuthenticationUserFromUrlOverwriteWrongDefaultCredentials() thr getMockWebConnection().setDefaultResponse(html); getWebClient().getCredentialsProvider().clear(); - ((DefaultCredentialsProvider) getWebClient().getCredentialsProvider()).addCredentials("joe", "hack"); + ((DefaultCredentialsProvider) getWebClient().getCredentialsProvider()).addCredentials("joe", "hack".toCharArray()); // use default wrong credentials try { @@ -390,7 +390,7 @@ public void basicAuthenticationXHR() throws Exception { + "xhr.send('');\n" + ""; - ((DefaultCredentialsProvider) getWebClient().getCredentialsProvider()).addCredentials("jetty", "jetty"); + ((DefaultCredentialsProvider) getWebClient().getCredentialsProvider()).addCredentials("jetty", "jetty".toCharArray()); getMockWebConnection().setResponse(URL_FIRST, html); getMockWebConnection().setResponse(URL_SECOND, "Hello World"); loadPageWithAlerts(html, URL_FIRST, DEFAULT_WAIT_TIME); @@ -418,7 +418,7 @@ public void basicAuthenticationXHRWithUsername() throws Exception { + "xhr.send('');\n" + ""; - ((DefaultCredentialsProvider) getWebClient().getCredentialsProvider()).addCredentials("jetty", "jetty"); + ((DefaultCredentialsProvider) getWebClient().getCredentialsProvider()).addCredentials("jetty", "jetty".toCharArray()); getMockWebConnection().setResponse(URL_FIRST, html); getMockWebConnection().setResponse(URL_SECOND, "Hello World"); loadPageWithAlerts(html, URL_FIRST, DEFAULT_WAIT_TIME); @@ -444,7 +444,7 @@ public void basicAuthenticationXHRWithUser() throws Exception { + "xhr.send('');\n" + ""; - ((DefaultCredentialsProvider) getWebClient().getCredentialsProvider()).addCredentials("jetty", "jetty"); + ((DefaultCredentialsProvider) getWebClient().getCredentialsProvider()).addCredentials("jetty", "jetty".toCharArray()); getMockWebConnection().setResponse(URL_FIRST, html); getMockWebConnection().setResponse(URL_SECOND, "Hello World"); loadPageWithAlerts(html, URL_FIRST, DEFAULT_WAIT_TIME); diff --git a/src/test/java/com/gargoylesoftware/htmlunit/DefaultCredentialsProvider3Test.java b/src/test/java/com/gargoylesoftware/htmlunit/DefaultCredentialsProvider3Test.java index 142dd17b0bb..725ed24575d 100644 --- a/src/test/java/com/gargoylesoftware/htmlunit/DefaultCredentialsProvider3Test.java +++ b/src/test/java/com/gargoylesoftware/htmlunit/DefaultCredentialsProvider3Test.java @@ -18,7 +18,7 @@ import static org.junit.Assert.assertNull; import org.apache.commons.lang3.SerializationUtils; -import org.apache.http.auth.AuthScope; +import org.apache.hc.client5.http.auth.AuthScope; import org.junit.Test; /** @@ -33,8 +33,9 @@ public class DefaultCredentialsProvider3Test { */ @Test public void serialization() { + final String protocol = "https"; final String username = "foo"; - final String password = "password"; + final char[] password = "password".toCharArray(); final String host = "my.host"; final int port = 1234; final String realm = "blah"; @@ -43,16 +44,16 @@ public void serialization() { DefaultCredentialsProvider provider = new DefaultCredentialsProvider(); provider.addCredentials(username, password, host, port, realm); - assertNotNull(provider.getCredentials(new AuthScope(host, port, realm, scheme))); - assertNull(provider.getCredentials(new AuthScope("invalidHost", port, realm, scheme))); - assertNotNull(provider.getCredentials(new AuthScope(host, port, realm, scheme))); - assertNull(provider.getCredentials(new AuthScope("invalidHost", port, realm, scheme))); + assertNotNull(provider.getCredentials(new AuthScope(protocol, host, port, realm, scheme), null)); + assertNull(provider.getCredentials(new AuthScope(protocol, "invalidHost", port, realm, scheme), null)); + assertNotNull(provider.getCredentials(new AuthScope(protocol, host, port, realm, scheme), null)); + assertNull(provider.getCredentials(new AuthScope(protocol, "invalidHost", port, realm, scheme), null)); provider = SerializationUtils.clone(provider); - assertNotNull(provider.getCredentials(new AuthScope(host, port, realm, scheme))); - assertNull(provider.getCredentials(new AuthScope("invalidHost", port, realm, scheme))); - assertNotNull(provider.getCredentials(new AuthScope(host, port, realm, scheme))); - assertNull(provider.getCredentials(new AuthScope("invalidHost", port, realm, scheme))); + assertNotNull(provider.getCredentials(new AuthScope(protocol, host, port, realm, scheme), null)); + assertNull(provider.getCredentials(new AuthScope(protocol, "invalidHost", port, realm, scheme), null)); + assertNotNull(provider.getCredentials(new AuthScope(protocol, host, port, realm, scheme), null)); + assertNull(provider.getCredentials(new AuthScope(protocol, "invalidHost", port, realm, scheme), null)); } } diff --git a/src/test/java/com/gargoylesoftware/htmlunit/DefaultCredentialsProviderTest.java b/src/test/java/com/gargoylesoftware/htmlunit/DefaultCredentialsProviderTest.java index 4435e163d4b..356d56a3cfe 100644 --- a/src/test/java/com/gargoylesoftware/htmlunit/DefaultCredentialsProviderTest.java +++ b/src/test/java/com/gargoylesoftware/htmlunit/DefaultCredentialsProviderTest.java @@ -14,9 +14,9 @@ */ package com.gargoylesoftware.htmlunit; -import org.apache.http.auth.AuthScope; -import org.apache.http.auth.UsernamePasswordCredentials; -import org.apache.http.impl.auth.BasicScheme; +import org.apache.hc.client5.http.auth.AuthScope; +import org.apache.hc.client5.http.auth.UsernamePasswordCredentials; +import org.apache.hc.client5.http.impl.auth.BasicScheme; import org.junit.Test; import org.junit.runner.RunWith; @@ -38,27 +38,27 @@ public class DefaultCredentialsProviderTest extends SimpleWebTestCase { @Test public void overwrite() throws Exception { final String realm = "blah"; - final String scheme = new BasicScheme().getSchemeName(); + final String scheme = new BasicScheme().getName(); final DefaultCredentialsProvider provider = new DefaultCredentialsProvider(); - provider.addCredentials("username", "password"); + provider.addCredentials("username", "password".toCharArray()); UsernamePasswordCredentials credentials = - (UsernamePasswordCredentials) provider.getCredentials(new AuthScope(HttpHeader.HOST_LC, 80, realm, scheme)); + (UsernamePasswordCredentials) provider.getCredentials(new AuthScope(null, HttpHeader.HOST_LC, 80, realm, scheme), null); assertEquals("username", credentials.getUserName()); - assertEquals("password", credentials.getPassword()); + assertEquals("password", new String(credentials.getPassword())); - provider.addCredentials("username", "new password"); + provider.addCredentials("username", "new password".toCharArray()); credentials = (UsernamePasswordCredentials) provider - .getCredentials(new AuthScope(HttpHeader.HOST_LC, 80, realm, scheme)); + .getCredentials(new AuthScope(null, HttpHeader.HOST_LC, 80, realm, scheme), null); assertEquals("username", credentials.getUserName()); - assertEquals("new password", credentials.getPassword()); + assertEquals("new password", new String(credentials.getPassword())); - provider.addCredentials("new username", "other password"); + provider.addCredentials("new username", "other password".toCharArray()); credentials = (UsernamePasswordCredentials) provider - .getCredentials(new AuthScope(HttpHeader.HOST_LC, 80, realm, scheme)); + .getCredentials(new AuthScope(null, HttpHeader.HOST_LC, 80, realm, scheme), null); assertEquals("new username", credentials.getUserName()); - assertEquals("other password", credentials.getPassword()); + assertEquals("other password", new String(credentials.getPassword())); } /** @@ -66,7 +66,7 @@ public void overwrite() throws Exception { */ @Test public void basicAuthenticationTwice() throws Exception { - ((DefaultCredentialsProvider) getWebClient().getCredentialsProvider()).addCredentials("jetty", "jetty"); + ((DefaultCredentialsProvider) getWebClient().getCredentialsProvider()).addCredentials("jetty", "jetty".toCharArray()); getMockWebConnection().setResponse(URL_SECOND, "Hello World"); HtmlPage page = loadPage("Hi There"); @@ -81,20 +81,20 @@ public void basicAuthenticationTwice() throws Exception { @Test public void removeCredentials() throws Exception { final String realm = "blah"; - final String scheme = new BasicScheme().getSchemeName(); + final String scheme = new BasicScheme().getName(); final DefaultCredentialsProvider provider = new DefaultCredentialsProvider(); - provider.addCredentials("username", "password", HttpHeader.HOST_LC, 80, realm); + provider.addCredentials("username", "password".toCharArray(), HttpHeader.HOST_LC, 80, realm); UsernamePasswordCredentials credentials = - (UsernamePasswordCredentials) provider.getCredentials(new AuthScope(HttpHeader.HOST_LC, 80, realm, scheme)); + (UsernamePasswordCredentials) provider.getCredentials(new AuthScope(null, HttpHeader.HOST_LC, 80, realm, scheme), null); assertEquals("username", credentials.getUserName()); - assertEquals("password", credentials.getPassword()); + assertEquals("password", new String(credentials.getPassword())); - provider.removeCredentials(new AuthScope(HttpHeader.HOST_LC, 80, realm, scheme)); + provider.removeCredentials(new AuthScope(null, HttpHeader.HOST_LC, 80, realm, scheme)); credentials = (UsernamePasswordCredentials) provider - .getCredentials(new AuthScope(HttpHeader.HOST_LC, 80, realm, scheme)); + .getCredentials(new AuthScope(null, HttpHeader.HOST_LC, 80, realm, scheme), null); assertNull(credentials); } diff --git a/src/test/java/com/gargoylesoftware/htmlunit/FailingHttpStatusCodeExceptionTest.java b/src/test/java/com/gargoylesoftware/htmlunit/FailingHttpStatusCodeExceptionTest.java index 48161d4942e..ca5d481b6ca 100644 --- a/src/test/java/com/gargoylesoftware/htmlunit/FailingHttpStatusCodeExceptionTest.java +++ b/src/test/java/com/gargoylesoftware/htmlunit/FailingHttpStatusCodeExceptionTest.java @@ -20,7 +20,7 @@ import java.util.List; import org.apache.commons.lang3.ArrayUtils; -import org.apache.http.HttpStatus; +import org.apache.hc.core5.http.HttpStatus; import org.junit.Test; import org.junit.runner.RunWith; diff --git a/src/test/java/com/gargoylesoftware/htmlunit/HtmlUnitCookieStoreTest.java b/src/test/java/com/gargoylesoftware/htmlunit/HtmlUnitCookieStoreTest.java index e44af734e45..c88af332dbf 100644 --- a/src/test/java/com/gargoylesoftware/htmlunit/HtmlUnitCookieStoreTest.java +++ b/src/test/java/com/gargoylesoftware/htmlunit/HtmlUnitCookieStoreTest.java @@ -21,8 +21,7 @@ import java.util.Date; import java.util.List; -import org.apache.commons.lang3.builder.EqualsBuilder; -import org.apache.http.impl.cookie.BasicClientCookie; +import org.apache.hc.client5.http.impl.cookie.BasicClientCookie; import org.junit.Before; import org.junit.Test; @@ -69,11 +68,11 @@ public void getCookies() { mgr_.addCookie(new Cookie("localhost", "myname", "myvalue")); mgr_.addCookie(new Cookie("localhost", "myname2", "myvalue2")); - final List cookies = store_.getCookies(); + final List cookies = store_.getCookies(); assertEquals(2, cookies.size()); - assertTrue(cookies.contains(new MyCookie("myname", "myvalue"))); - assertTrue(cookies.contains(new MyCookie("myname2", "myvalue2"))); - + assertTrue(containsCookie(cookies, "myname", "myvalue")); + assertTrue(containsCookie(cookies, "myname2", "myvalue2")); + mgr_.setCookiesEnabled(false); assertTrue(store_.getCookies().isEmpty()); } @@ -91,9 +90,9 @@ public void clearExpired() { assertTrue(store_.clearExpired(new Date(System.currentTimeMillis() + 20_000))); assertFalse(store_.clearExpired(new Date(System.currentTimeMillis() + 20_000))); - final List cookies = store_.getCookies(); + final List cookies = store_.getCookies(); assertEquals(1, cookies.size()); - assertTrue(cookies.contains(new MyCookie("myname", "myvalue"))); + assertTrue(containsCookie(cookies, "myname", "myvalue")); } /** @@ -110,23 +109,20 @@ public void clear() { assertTrue(store_.getCookies().isEmpty()); } - private static final class MyCookie extends BasicClientCookie { - MyCookie(final String name, final String value) { - super(name, value); - } - - @Override - public boolean equals(final Object obj) { - return obj instanceof org.apache.http.cookie.Cookie - && new EqualsBuilder() - .append(getName(), ((org.apache.http.cookie.Cookie) obj).getName()) - .append(getValue(), ((org.apache.http.cookie.Cookie) obj).getValue()) - .isEquals(); - } - - @Override - public int hashCode() { - return super.hashCode(); + /** + * Checks if the given list of cookies contains a cookie with the passed name and value. + * @param cookies the list of cookies to search + * @param name the cookie name + * @param value the cookie value + * @return true if such a cookie was found, false otherwise + */ + private static boolean containsCookie(final List cookies, + final String name, final String value) { + for (org.apache.hc.client5.http.cookie.Cookie cookie : cookies) { + if (cookie.getName().equals(name) && cookie.getValue().equals(value)) { + return true; + } } + return false; } } diff --git a/src/test/java/com/gargoylesoftware/htmlunit/HttpWebConnection3Test.java b/src/test/java/com/gargoylesoftware/htmlunit/HttpWebConnection3Test.java index f94f655ce48..23c38464b02 100644 --- a/src/test/java/com/gargoylesoftware/htmlunit/HttpWebConnection3Test.java +++ b/src/test/java/com/gargoylesoftware/htmlunit/HttpWebConnection3Test.java @@ -394,10 +394,10 @@ public void queryString() throws Exception { "User-Agent: §§USER_AGENT§§", "Accept-Encoding: gzip, deflate", "Host: localhost:§§PORT§§", - "Connection: Keep-Alive"}) + "Connection: keep-alive"}) @HtmlUnitNYI(CHROME = {"GET /foo?text1=me+%26amp%3B+you&text2=Hello%0D%0Aworld%21 HTTP/1.1", "Host: localhost:§§PORT§§", - "Connection: Keep-Alive", + "Connection: keep-alive", "sec-ch-ua: \" Not;A Brand\";v=\"99\", \"Google Chrome\";v=\"91\", \"Chromium\";v=\"91\"", "sec-ch-ua-mobile: ?0", "Upgrade-Insecure-Requests: 1", @@ -412,7 +412,7 @@ public void queryString() throws Exception { "Accept-Language: en-US,en;q=0.9"}, EDGE = {"GET /foo?text1=me+%26amp%3B+you&text2=Hello%0D%0Aworld%21 HTTP/1.1", "Host: localhost:§§PORT§§", - "Connection: Keep-Alive", + "Connection: keep-alive", "sec-ch-ua: \" Not;A Brand\";v=\"99\", \"Microsoft Edge\";v=\"91\", \"Chromium\";v=\"91\"", "sec-ch-ua-mobile: ?0", "Upgrade-Insecure-Requests: 1", @@ -431,7 +431,7 @@ public void queryString() throws Exception { "Accept: §§ACCEPT§§", "Accept-Language: en-US,en;q=0.5", "Accept-Encoding: gzip, deflate", - "Connection: Keep-Alive", + "Connection: keep-alive", "Referer: http://localhost:§§PORT§§/", "Upgrade-Insecure-Requests: 1", "Sec-Fetch-Dest: document", @@ -444,7 +444,7 @@ public void queryString() throws Exception { "Accept: §§ACCEPT§§", "Accept-Language: en-US,en;q=0.5", "Accept-Encoding: gzip, deflate", - "Connection: Keep-Alive", + "Connection: keep-alive", "Referer: http://localhost:§§PORT§§/", "Upgrade-Insecure-Requests: 1"}) public void formGet() throws Exception { @@ -573,13 +573,13 @@ public void formGet() throws Exception { "Accept-Encoding: gzip, deflate", "Host: localhost:§§PORT§§", "Content-Length: 48", - "Connection: Keep-Alive", + "Connection: keep-alive", "Cache-Control: no-cache", "", "text1=me+%26amp%3B+you&text2=Hello%0D%0Aworld%21"}) @HtmlUnitNYI(CHROME = {"POST /foo HTTP/1.1", "Host: localhost:§§PORT§§", - "Connection: Keep-Alive", + "Connection: keep-alive", "sec-ch-ua: \" Not;A Brand\";v=\"99\", \"Google Chrome\";v=\"91\", \"Chromium\";v=\"91\"", "sec-ch-ua-mobile: ?0", "Upgrade-Insecure-Requests: 1", @@ -600,7 +600,7 @@ public void formGet() throws Exception { "text1=me+%26amp%3B+you&text2=Hello%0D%0Aworld%21"}, EDGE = {"POST /foo HTTP/1.1", "Host: localhost:§§PORT§§", - "Connection: Keep-Alive", + "Connection: keep-alive", "sec-ch-ua: \" Not;A Brand\";v=\"99\", \"Microsoft Edge\";v=\"91\", \"Chromium\";v=\"91\"", "sec-ch-ua-mobile: ?0", "Upgrade-Insecure-Requests: 1", @@ -625,7 +625,7 @@ public void formGet() throws Exception { "Accept: §§ACCEPT§§", "Accept-Language: en-US,en;q=0.5", "Accept-Encoding: gzip, deflate", - "Connection: Keep-Alive", + "Connection: keep-alive", "Referer: http://localhost:§§PORT§§/", "Upgrade-Insecure-Requests: 1", "Sec-Fetch-Dest: document", @@ -643,7 +643,7 @@ public void formGet() throws Exception { "Accept: §§ACCEPT§§", "Accept-Language: en-US,en;q=0.5", "Accept-Encoding: gzip, deflate", - "Connection: Keep-Alive", + "Connection: keep-alive", "Referer: http://localhost:§§PORT§§/", "Origin: http://localhost:§§PORT§§", "Upgrade-Insecure-Requests: 1", @@ -658,7 +658,7 @@ public void formGet() throws Exception { "User-Agent: §§USER_AGENT§§", "Accept-Encoding: gzip, deflate", "Host: localhost:§§PORT§§", - "Connection: Keep-Alive", + "Connection: keep-alive", "Cache-Control: no-cache", "Content-Length: 48", "Content-Type: application/x-www-form-urlencoded", diff --git a/src/test/java/com/gargoylesoftware/htmlunit/HttpWebConnectionTest.java b/src/test/java/com/gargoylesoftware/htmlunit/HttpWebConnectionTest.java index b32c5ee6c80..610ae83d35e 100644 --- a/src/test/java/com/gargoylesoftware/htmlunit/HttpWebConnectionTest.java +++ b/src/test/java/com/gargoylesoftware/htmlunit/HttpWebConnectionTest.java @@ -40,16 +40,17 @@ import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; -import org.apache.http.HttpEntity; -import org.apache.http.HttpResponse; -import org.apache.http.HttpStatus; -import org.apache.http.ProtocolVersion; -import org.apache.http.StatusLine; -import org.apache.http.entity.StringEntity; -import org.apache.http.entity.mime.MultipartEntityBuilder; -import org.apache.http.impl.client.HttpClientBuilder; -import org.apache.http.message.BasicHttpResponse; -import org.apache.http.message.BasicStatusLine; +import org.apache.hc.client5.http.entity.mime.MultipartEntityBuilder; +import org.apache.hc.client5.http.impl.async.HttpAsyncClientBuilder; +import org.apache.hc.client5.http.impl.classic.HttpClientBuilder; +import org.apache.hc.core5.http.ClassicHttpResponse; +import org.apache.hc.core5.http.HttpEntity; +import org.apache.hc.core5.http.HttpResponse; +import org.apache.hc.core5.http.HttpStatus; +import org.apache.hc.core5.http.ProtocolVersion; +import org.apache.hc.core5.http.io.entity.StringEntity; +import org.apache.hc.core5.http.message.BasicClassicHttpResponse; +import org.apache.hc.core5.http.message.StatusLine; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.core.Logger; @@ -187,8 +188,8 @@ public void makeWebResponse() throws Exception { final long loadTime = 500L; final ProtocolVersion protocolVersion = new ProtocolVersion("HTTP", 1, 0); - final StatusLine statusLine = new BasicStatusLine(protocolVersion, HttpStatus.SC_OK, null); - final HttpResponse httpResponse = new BasicHttpResponse(statusLine); + final StatusLine statusLine = new StatusLine(protocolVersion, HttpStatus.SC_OK, null); + final ClassicHttpResponse httpResponse = new BasicClassicHttpResponse(HttpStatus.SC_OK); final HttpEntity responseEntity = new StringEntity(content); httpResponse.setEntity(responseEntity); @@ -237,10 +238,10 @@ public void designedForExtension() throws Exception { final boolean[] tabCalled = {false}; final WebConnection myWebConnection = new HttpWebConnection(webClient) { @Override - protected HttpClientBuilder createHttpClientBuilder() { + protected HttpAsyncClientBuilder createHttpClientBuilder() { tabCalled[0] = true; - final HttpClientBuilder builder = HttpClientBuilder.create(); + final HttpAsyncClientBuilder builder = HttpAsyncClientBuilder.create(); builder.setConnectionManagerShared(true); return builder; } @@ -594,7 +595,7 @@ public void contentLengthLargerThanContent() throws Exception { public void userAgent() throws Exception { final WebClient webClient = getWebClient(); final HttpWebConnection connection = (HttpWebConnection) webClient.getWebConnection(); - final HttpClientBuilder builder = connection.getHttpClientBuilder(); + final HttpAsyncClientBuilder builder = connection.getHttpClientBuilder(); final String userAgent = get(builder, "userAgent"); assertEquals(webClient.getBrowserVersion().getUserAgent(), userAgent); } diff --git a/src/test/java/com/gargoylesoftware/htmlunit/SgmlPageTest.java b/src/test/java/com/gargoylesoftware/htmlunit/SgmlPageTest.java index 536b8b31c60..c4eaa34d3b8 100644 --- a/src/test/java/com/gargoylesoftware/htmlunit/SgmlPageTest.java +++ b/src/test/java/com/gargoylesoftware/htmlunit/SgmlPageTest.java @@ -14,7 +14,7 @@ */ package com.gargoylesoftware.htmlunit; -import static org.apache.http.client.utils.DateUtils.formatDate; +import static org.apache.hc.client5.http.utils.DateUtils.formatDate; import java.util.ArrayList; import java.util.Date; diff --git a/src/test/java/com/gargoylesoftware/htmlunit/WebClientTest.java b/src/test/java/com/gargoylesoftware/htmlunit/WebClientTest.java index edf1ddbefb8..63cf9e8c6eb 100644 --- a/src/test/java/com/gargoylesoftware/htmlunit/WebClientTest.java +++ b/src/test/java/com/gargoylesoftware/htmlunit/WebClientTest.java @@ -41,7 +41,7 @@ import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; import org.apache.commons.lang3.mutable.MutableInt; -import org.apache.http.HttpStatus; +import org.apache.hc.core5.http.HttpStatus; import org.junit.Test; import org.junit.runner.RunWith; diff --git a/src/test/java/com/gargoylesoftware/htmlunit/WebRequestTest.java b/src/test/java/com/gargoylesoftware/htmlunit/WebRequestTest.java index ea4a996b926..32be209d62e 100644 --- a/src/test/java/com/gargoylesoftware/htmlunit/WebRequestTest.java +++ b/src/test/java/com/gargoylesoftware/htmlunit/WebRequestTest.java @@ -20,8 +20,8 @@ import java.net.URL; -import org.apache.http.auth.BasicUserPrincipal; -import org.apache.http.auth.Credentials; +import org.apache.hc.client5.http.auth.BasicUserPrincipal; +import org.apache.hc.client5.http.auth.Credentials; import org.junit.Test; /** @@ -116,7 +116,7 @@ public void credentials() throws Exception { final WebRequest request = new WebRequest(url); final Credentials credentials = request.getUrlCredentials(); assertEquals(new BasicUserPrincipal("john.smith"), credentials.getUserPrincipal()); - assertEquals("secret", credentials.getPassword()); + assertEquals("secret", new String(credentials.getPassword())); } /** @@ -129,7 +129,7 @@ public void credentialsAndEmptyPath() throws Exception { final Credentials credentials = request.getUrlCredentials(); assertNotNull("Credentials object is null", credentials); assertEquals(new BasicUserPrincipal("john.smith"), credentials.getUserPrincipal()); - assertEquals("secret", credentials.getPassword()); + assertEquals("secret", new String(credentials.getPassword())); } /** @@ -142,7 +142,7 @@ public void credentialsAndPathWithDots() throws Exception { final Credentials credentials = request.getUrlCredentials(); assertNotNull("Credentials object is null", credentials); assertEquals(new BasicUserPrincipal("john.smith"), credentials.getUserPrincipal()); - assertEquals("secret", credentials.getPassword()); + assertEquals("secret", new String(credentials.getPassword())); } /** @@ -155,7 +155,7 @@ public void credentialsAndInternationalizedDomainName() throws Exception { final Credentials credentials = request.getUrlCredentials(); assertNotNull("Credentials object is null", credentials); assertEquals(new BasicUserPrincipal("john.smith"), credentials.getUserPrincipal()); - assertEquals("secret", credentials.getPassword()); + assertEquals("secret", new String(credentials.getPassword())); } /** @@ -167,7 +167,7 @@ public void credentialsOnlyUsernameInURL() throws Exception { final WebRequest request = new WebRequest(url); final Credentials credentials = request.getUrlCredentials(); assertEquals(new BasicUserPrincipal("john.smith"), credentials.getUserPrincipal()); - assertEquals("", credentials.getPassword()); + assertEquals("", new String(credentials.getPassword())); } /** @@ -179,7 +179,7 @@ public void credentialsOnlyPasswordInURL() throws Exception { final WebRequest request = new WebRequest(url); final Credentials credentials = request.getUrlCredentials(); assertEquals(new BasicUserPrincipal(""), credentials.getUserPrincipal()); - assertEquals("secret", credentials.getPassword()); + assertEquals("secret", new String(credentials.getPassword())); } /** @@ -191,7 +191,7 @@ public void credentialsEmptyURL() throws Exception { final WebRequest request = new WebRequest(url); final Credentials credentials = request.getUrlCredentials(); assertEquals(new BasicUserPrincipal(""), credentials.getUserPrincipal()); - assertEquals("", credentials.getPassword()); + assertEquals("", new String(credentials.getPassword())); } /** diff --git a/src/test/java/com/gargoylesoftware/htmlunit/WebResponseDataTest.java b/src/test/java/com/gargoylesoftware/htmlunit/WebResponseDataTest.java index 4242fb693e5..0ec46ceca52 100644 --- a/src/test/java/com/gargoylesoftware/htmlunit/WebResponseDataTest.java +++ b/src/test/java/com/gargoylesoftware/htmlunit/WebResponseDataTest.java @@ -32,7 +32,7 @@ import org.apache.commons.io.IOUtils; import org.apache.commons.lang3.StringUtils; -import org.apache.http.HttpStatus; +import org.apache.hc.core5.http.HttpStatus; import org.junit.Test; import org.junit.runner.RunWith; diff --git a/src/test/java/com/gargoylesoftware/htmlunit/WebResponseTest.java b/src/test/java/com/gargoylesoftware/htmlunit/WebResponseTest.java index 14d7e3b91dd..162fff70a86 100644 --- a/src/test/java/com/gargoylesoftware/htmlunit/WebResponseTest.java +++ b/src/test/java/com/gargoylesoftware/htmlunit/WebResponseTest.java @@ -33,7 +33,7 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; -import org.apache.http.HttpStatus; +import org.apache.hc.core5.http.HttpStatus; import org.junit.After; import org.junit.Test; import org.junit.runner.RunWith; diff --git a/src/test/java/com/gargoylesoftware/htmlunit/html/HtmlFileInput2Test.java b/src/test/java/com/gargoylesoftware/htmlunit/html/HtmlFileInput2Test.java index ac8dfbc7ddb..fadaa16f67a 100644 --- a/src/test/java/com/gargoylesoftware/htmlunit/html/HtmlFileInput2Test.java +++ b/src/test/java/com/gargoylesoftware/htmlunit/html/HtmlFileInput2Test.java @@ -44,14 +44,15 @@ import org.apache.commons.fileupload.disk.DiskFileItemFactory; import org.apache.commons.fileupload.servlet.ServletFileUpload; import org.apache.commons.io.IOUtils; -import org.apache.http.HttpEntity; -import org.apache.http.HttpResponse; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.entity.ContentType; -import org.apache.http.entity.mime.HttpMultipartMode; -import org.apache.http.entity.mime.MultipartEntityBuilder; -import org.apache.http.entity.mime.content.FileBody; -import org.apache.http.impl.client.HttpClientBuilder; +import org.apache.hc.client5.http.classic.methods.HttpPost; +import org.apache.hc.client5.http.entity.mime.FileBody; +import org.apache.hc.client5.http.entity.mime.HttpMultipartMode; +import org.apache.hc.client5.http.entity.mime.MultipartEntityBuilder; +import org.apache.hc.client5.http.impl.classic.HttpClientBuilder; +import org.apache.hc.core5.http.ContentType; +import org.apache.hc.core5.http.HttpEntity; +import org.apache.hc.core5.http.HttpEntityContainer; +import org.apache.hc.core5.http.HttpResponse; import org.junit.Test; import org.junit.runner.RunWith; @@ -384,13 +385,13 @@ public void uploadFileWithNonASCIIName_HttpClient() throws Exception { final HttpPost filePost = new HttpPost(URL_FIRST + "upload2"); final MultipartEntityBuilder builder = MultipartEntityBuilder.create(); - builder.setMode(HttpMultipartMode.BROWSER_COMPATIBLE).setCharset(UTF_8); + builder.setMode(HttpMultipartMode.LEGACY).setCharset(UTF_8); builder.addPart("myInput", new FileBody(file, ContentType.APPLICATION_OCTET_STREAM)); filePost.setEntity(builder.build()); final HttpClientBuilder clientBuilder = HttpClientBuilder.create(); - final HttpResponse httpResponse = clientBuilder.build().execute(filePost); + final HttpEntityContainer httpResponse = clientBuilder.build().execute(filePost); try (InputStream content = httpResponse.getEntity().getContent()) { final String response = new String(IOUtils.toByteArray(content)); diff --git a/src/test/java/com/gargoylesoftware/htmlunit/html/HtmlScript2Test.java b/src/test/java/com/gargoylesoftware/htmlunit/html/HtmlScript2Test.java index c1dc6d8d412..aa018b5249e 100644 --- a/src/test/java/com/gargoylesoftware/htmlunit/html/HtmlScript2Test.java +++ b/src/test/java/com/gargoylesoftware/htmlunit/html/HtmlScript2Test.java @@ -21,7 +21,7 @@ import java.util.Map; import org.apache.commons.io.ByteOrderMark; -import org.apache.http.HttpStatus; +import org.apache.hc.core5.http.HttpStatus; import org.junit.Test; import org.junit.runner.RunWith; import org.openqa.selenium.By; diff --git a/src/test/java/com/gargoylesoftware/htmlunit/html/HtmlScriptTest.java b/src/test/java/com/gargoylesoftware/htmlunit/html/HtmlScriptTest.java index 9808c5def2b..e99d7fe2ae7 100644 --- a/src/test/java/com/gargoylesoftware/htmlunit/html/HtmlScriptTest.java +++ b/src/test/java/com/gargoylesoftware/htmlunit/html/HtmlScriptTest.java @@ -20,7 +20,7 @@ import java.util.ArrayList; import java.util.List; -import org.apache.http.HttpStatus; +import org.apache.hc.core5.http.HttpStatus; import org.junit.Test; import org.junit.runner.RunWith; diff --git a/src/test/java/com/gargoylesoftware/htmlunit/html/XHtmlPageTest.java b/src/test/java/com/gargoylesoftware/htmlunit/html/XHtmlPageTest.java index ef313f02a44..1dcdc79ae77 100644 --- a/src/test/java/com/gargoylesoftware/htmlunit/html/XHtmlPageTest.java +++ b/src/test/java/com/gargoylesoftware/htmlunit/html/XHtmlPageTest.java @@ -17,7 +17,7 @@ import java.util.ArrayList; import java.util.List; -import org.apache.http.HttpStatus; +import org.apache.hc.core5.http.HttpStatus; import org.junit.Test; import org.junit.runner.RunWith; diff --git a/src/test/java/com/gargoylesoftware/htmlunit/javascript/host/html/HTMLDocumentTest.java b/src/test/java/com/gargoylesoftware/htmlunit/javascript/host/html/HTMLDocumentTest.java index 9d118a21e18..0edda930af3 100644 --- a/src/test/java/com/gargoylesoftware/htmlunit/javascript/host/html/HTMLDocumentTest.java +++ b/src/test/java/com/gargoylesoftware/htmlunit/javascript/host/html/HTMLDocumentTest.java @@ -27,7 +27,7 @@ import java.util.List; import java.util.Locale; -import org.apache.http.client.utils.DateUtils; +import org.apache.hc.client5.http.utils.DateUtils; import org.junit.Test; import org.junit.runner.RunWith; import org.openqa.selenium.By; diff --git a/src/test/java/com/gargoylesoftware/htmlunit/javascript/host/xml/XMLHttpRequestLifeCycleTest.java b/src/test/java/com/gargoylesoftware/htmlunit/javascript/host/xml/XMLHttpRequestLifeCycleTest.java index 3c6c1f47481..08e960550d1 100644 --- a/src/test/java/com/gargoylesoftware/htmlunit/javascript/host/xml/XMLHttpRequestLifeCycleTest.java +++ b/src/test/java/com/gargoylesoftware/htmlunit/javascript/host/xml/XMLHttpRequestLifeCycleTest.java @@ -31,7 +31,7 @@ import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpServletResponse; -import org.apache.http.HttpStatus; +import org.apache.hc.core5.http.HttpStatus; import org.junit.After; import org.junit.Before; import org.junit.Test; diff --git a/src/test/java/com/gargoylesoftware/htmlunit/util/WebConnectionWrapperTest.java b/src/test/java/com/gargoylesoftware/htmlunit/util/WebConnectionWrapperTest.java index 25c48c57bd2..44ccf12a74c 100644 --- a/src/test/java/com/gargoylesoftware/htmlunit/util/WebConnectionWrapperTest.java +++ b/src/test/java/com/gargoylesoftware/htmlunit/util/WebConnectionWrapperTest.java @@ -17,7 +17,7 @@ import java.util.Collections; import java.util.List; -import org.apache.http.HttpStatus; +import org.apache.hc.core5.http.HttpStatus; import org.junit.Test; import com.gargoylesoftware.htmlunit.HttpMethod; diff --git a/src/test/java/com/gargoylesoftware/htmlunit/xml/XmlPageTest.java b/src/test/java/com/gargoylesoftware/htmlunit/xml/XmlPageTest.java index b1c12ee326a..989217c1bcc 100644 --- a/src/test/java/com/gargoylesoftware/htmlunit/xml/XmlPageTest.java +++ b/src/test/java/com/gargoylesoftware/htmlunit/xml/XmlPageTest.java @@ -27,7 +27,7 @@ import javax.servlet.http.HttpServletResponse; import org.apache.commons.io.ByteOrderMark; -import org.apache.http.HttpStatus; +import org.apache.hc.core5.http.HttpStatus; import org.junit.Test; import org.junit.runner.RunWith; import org.w3c.dom.Node;