Make it easier to work with server and client certificates in the client (#2647)

* Make it easier to work with server and client certificates in the client

Rather then making folks register their ServerCertificateValidation callback globally on the static `ServicePointManager` or subclass `HttpConnection` to set it on the request/itself
we now expose it on `ConnectionSettings` this callback is fired for each unique endpoint (node) until it returns true after which its cached for the duration of that servicepoint.

We also ship with handy baked in validations on `CertificateValidations`:

* `CertificateValidations.AllowAll` simply returns true
* `CertificateValidations.DenyAll` simply returns false

If your client application however has access to the public CA certificate locally Elasticsearch.NET/NEST ships with handy helpers that assert
that the certificate that the server presented was one that came from our local CA certificate. If you use x-pack's `certgen` tool to
[generate SSL certificates](https://www.elastic.co/guide/en/x-pack/current/ssl-tls.html) the generated node certificate does not include the CA in the
certificate chain. This to cut back on SSL handshake size. In those case you can use `CertificateValidations.AuthorityIsRoot` and pass it your local copy
of the CA public key to assert that the certificate the server presented was generated off that.

If you go for a vendor generated SSL certificate its common practice for them to include the CA and any intermediary CA's in the certificate chain
in those case use `CertificateValidations.AuthorityPartOfChain` which validates that the local CA certificate is part of that chain and was used to
generate the servers key.

`ConnectionSettings` now also accepts `ClientCertificates` as a collection or `ClientCertificate` as a single certificate to be used as the user authentication for ALL requests.
`RequestConfiguration` accepts the same but will be the client certificate for that single request only.

The client certificate should be a certificate that has the public and private key available (`pfx` or `p12`) however x-pack `certgen` generates two separate `cer` and `key` files.

For .NET 4.5/4.6 we ship with a helper that creates a proper self contained certificate from these two files `ClientCertificate.LoadWithPrivateKey` but because we can no longer update a
certificates `PublicKey` algorithm in .NET core this is not available there. Its typically recommended to generate a single pfx or p12 file since those can just be passed to `X509Certificate`'s
constructor

* spacing and visibillity changes

* try fix mono build of .net 4.* HttpConnection

* make sure in unit test mode we skip the certificate tests since they rely on a disk on file, also make sure cluster base does not do the desiredport check when running in unit test mode

* only throw when attempting to set callback on mono when callback is not null

* callback not in ifdef scope

Conflicts:
	src/Elasticsearch.Net/Configuration/ConnectionConfiguration.cs
	src/Elasticsearch.Net/Configuration/IConnectionConfigurationValues.cs
	src/Elasticsearch.Net/Configuration/RequestConfiguration.cs
	src/Elasticsearch.Net/Transport/Pipeline/RequestData.cs
	src/Tests/Framework/ManagedElasticsearch/Nodes/ElasticsearchNode.cs
	src/Tests/Tests.csproj

Author: Mpdreamz

docs/client-concepts/high-level/mapping/auto-map.asciidoc

@@ -698,6 +698,51 @@ var expected = new
 Expect(expected).WhenSerializing((ICreateIndexRequest)descriptor);
 ----
 
+[source,csharp]
+----
+public class ParentWithStringId : Parent
+{
+    public new string Id { get; set; }
+}
+----
+
+[source,csharp]
+----
+var descriptor = new CreateIndexDescriptor("myindex")
+    .Mappings(ms => ms
+        .Map<ParentWithStringId>(m => m
+            .AutoMap()
+        )
+    );
+
+var expected = new
+{
+    mappings = new
+    {
+        parent = new
+        {
+            properties = new
+            {
+                id = new
+                {
+                    type = "string",
+                }
+            }
+        }
+    }
+};
+
+var settings = WithConnectionSettings(s => s
+    .InferMappingFor<ParentWithStringId>(m => m
+        .TypeName("parent")
+        .Ignore(p => p.Description)
+        .Ignore(p => p.IgnoreMe)
+    )
+);
+
+settings.Expect(expected).WhenSerializing((ICreateIndexRequest) descriptor);
+----
+
 [[ignoring-properties]]
 [float]
 == Ignoring Properties

src/Elasticsearch.Net/Configuration/ConnectionConfiguration.cs

@@ -3,6 +3,8 @@
 using System.ComponentModel;
 using System.Diagnostics;
 using System.Diagnostics.CodeAnalysis;
+using System.Net.Security;
+using System.Security.Cryptography.X509Certificates;
 using System.Threading;
 
 #if DOTNETCORE
@@ -143,6 +145,11 @@ private static void DefaultRequestDataCreated(RequestData response) { }
 		private Action<RequestData> _onRequestDataCreated = DefaultRequestDataCreated;
 		Action<RequestData> IConnectionConfigurationValues.OnRequestDataCreated => _onRequestDataCreated;
 
+		private Func<object, X509Certificate,X509Chain,SslPolicyErrors, bool> _serverCertificateValidationCallback;
+		Func<object, X509Certificate, X509Chain, SslPolicyErrors, bool> IConnectionConfigurationValues.ServerCertificateValidationCallback => _serverCertificateValidationCallback;
+
+		private X509CertificateCollection _clientCertificates;
+		X509CertificateCollection IConnectionConfigurationValues.ClientCertificates => _clientCertificates;
 		private readonly NameValueCollection _queryString = new NameValueCollection();
 		NameValueCollection IConnectionConfigurationValues.QueryStringParameters => _queryString;
 
@@ -388,6 +395,35 @@ public T EnableDebugMode(Action<IApiCallDetails> onRequestCompleted = null)
 			return (T)this;
 		}
 
+		/// <summary>
+		/// Register a ServerCertificateValidationCallback, this is called per endpoint until it returns true.
+		/// After this callback returns true that endpoint is validated for the lifetime of the ServiceEndpoint
+		/// for that host.
+		/// </summary>
+		public T ServerCertificateValidationCallback(Func<object, X509Certificate, X509Chain, SslPolicyErrors, bool> callback) =>
+			Assign(a => a._serverCertificateValidationCallback = callback);
+
+		/// <summary>
+		/// Use the following certificates to authenticate all HTTP requests. You can also set them on individual
+		/// request using <see cref="RequestConfiguration.ClientCertificates"/>
+		/// </summary>
+		public T ClientCertificates(X509CertificateCollection certificates) =>
+			Assign(a => a._clientCertificates = certificates);
+
+		/// <summary>
+		/// Use the following certificate to authenticate all HTTP requests. You can also set them on individual
+		/// request using <see cref="RequestConfiguration.ClientCertificates"/>
+		/// </summary>
+		public T ClientCertificate(X509Certificate certificate) =>
+			Assign(a => a._clientCertificates = new X509Certificate2Collection { certificate });
+
+		/// <summary>
+		/// Use the following certificate to authenticate all HTTP requests. You can also set them on individual
+		/// request using <see cref="RequestConfiguration.ClientCertificates"/>
+		/// </summary>
+		public T ClientCertificate(string certificatePath) =>
+			Assign(a => a._clientCertificates = new X509Certificate2Collection { new X509Certificate(certificatePath) });
+
 		void IDisposable.Dispose() => this.DisposeManagedResources();
 
 		protected virtual void DisposeManagedResources()

src/Elasticsearch.Net/Configuration/IConnectionConfigurationValues.cs

@@ -1,5 +1,7 @@
 using System;
 using System.Collections.Specialized;
+using System.Net.Security;
+using System.Security.Cryptography.X509Certificates;
 using System.Threading;
 
 namespace Elasticsearch.Net
@@ -171,5 +173,17 @@ public interface IConnectionConfigurationValues : IDisposable
 		/// received.
 		/// </summary>
 		TimeSpan? KeepAliveInterval { get; }
+
+		/// <summary>
+		/// Register a ServerCertificateValidationCallback per request
+		/// </summary>
+		Func<object, X509Certificate,X509Chain,SslPolicyErrors, bool>  ServerCertificateValidationCallback { get; }
+
+
+		/// <summary>
+		/// Use the following certificates to authenticate all HTTP requests. You can also set them on individual
+		/// request using <see cref="RequestConfiguration.ClientCertificates"/>
+		/// </summary>
+		X509CertificateCollection ClientCertificates { get; }
 	}
 }

src/Elasticsearch.Net/Configuration/RequestConfiguration.cs

@@ -1,5 +1,7 @@
 using System;
 using System.Collections.Generic;
+using System.Net.Security;
+using System.Security.Cryptography.X509Certificates;
 using System.Threading;
 
 namespace Elasticsearch.Net
@@ -74,6 +76,11 @@ public interface IRequestConfiguration
 		/// <pre/>https://www.elastic.co/guide/en/shield/current/submitting-requests-for-other-users.html
 		/// </summary>
 		string RunAs { get; set; }
+
+		/// <summary>
+		/// Use the following client certificates to authenticate this single request
+		/// </summary>
+		X509CertificateCollection ClientCertificates { get; set; }
 	}
 
 	public class RequestConfiguration : IRequestConfiguration
@@ -95,6 +102,8 @@ public class RequestConfiguration : IRequestConfiguration
 		/// https://www.elastic.co/guide/en/shield/current/submitting-requests-for-other-users.html
 		/// </summary>
 		public string RunAs { get; set; }
+
+		public X509CertificateCollection ClientCertificates { get; set; }
 	}
 
 	public class RequestConfigurationDescriptor : IRequestConfiguration
@@ -114,6 +123,7 @@ public class RequestConfigurationDescriptor : IRequestConfiguration
 		bool IRequestConfiguration.EnableHttpPipelining { get; set; } = true;
 		CancellationToken IRequestConfiguration.CancellationToken { get; set; }
 		string IRequestConfiguration.RunAs { get; set; }
+		X509CertificateCollection IRequestConfiguration.ClientCertificates { get; set; }
 
 		public RequestConfigurationDescriptor(IRequestConfiguration config)
 		{
@@ -130,6 +140,7 @@ public RequestConfigurationDescriptor(IRequestConfiguration config)
 			Self.EnableHttpPipelining = config?.EnableHttpPipelining ?? true;
 			Self.CancellationToken = config?.CancellationToken ?? default(CancellationToken);
 			Self.RunAs = config?.RunAs;
+			Self.ClientCertificates = config?.ClientCertificates;
 		}
 
 		/// <summary>
@@ -195,6 +206,7 @@ public RequestConfigurationDescriptor ForceNode(Uri uri)
 			Self.ForceNode = uri;
 			return this;
 		}
+
 		public RequestConfigurationDescriptor MaxRetries(int retry)
 		{
 			Self.MaxRetries = retry;
@@ -220,5 +232,20 @@ public RequestConfigurationDescriptor EnableHttpPipelining(bool enable = true)
 			Self.EnableHttpPipelining = enable;
 			return this;
 		}
+
+		/// <summary> Use the following client certificates to authenticate this request to Elasticsearch </summary>
+		public RequestConfigurationDescriptor ClientCertificates(X509CertificateCollection certificates)
+		{
+			Self.ClientCertificates = certificates;
+			return this;
+		}
+
+		/// <summary> Use the following client certificate to authenticate this request to Elasticsearch </summary>
+		public RequestConfigurationDescriptor ClientCertificate(X509Certificate certificate) =>
+			this.ClientCertificates(new X509Certificate2Collection { certificate });
+
+		/// <summary> Use the following client certificate to authenticate this request to Elasticsearch </summary>
+		public RequestConfigurationDescriptor ClientCertificate(string certificatePath) =>
+			this.ClientCertificates(new X509Certificate2Collection {new X509Certificate(certificatePath)});
 	}
 }

src/Elasticsearch.Net/Connection/CertificateValidations.cs

@@ -0,0 +1,129 @@
+using System;
+using System.Collections.Concurrent;
+using System.Net;
+using System.Net.Security;
+using System.Security.Cryptography.X509Certificates;
+
+namespace Elasticsearch.Net
+{
+	/// <summary>
+	/// A collection of handy baked in server certificate validation callbacks
+	/// </summary>
+	public static class CertificateValidations
+	{
+		/// <summary>
+		/// DANGEROUS, never use this in production validates ALL certificates to true.
+		/// </summary>
+		/// <returns>Always true, allowing ALL certificates</returns>
+		public static bool AllowAll(object sender, X509Certificate certificate, X509Chain chain, SslPolicyErrors errors) => true;
+
+		/// <summary>
+		/// Always false, in effect blocking ALL certificates
+		/// </summary>
+		/// <returns>Always false, always blocking ALL certificates</returns>
+		public static bool DenyAll(object sender, X509Certificate certificate, X509Chain chain, SslPolicyErrors errors) => false;
+
+		/// <summary>
+		/// Helper to create a certificate validation callback based on the certificate authority certificate that we used to
+		/// generate the nodes certificates with. This callback expects the CA to be part of the chain as intermediate CA.
+		/// </summary>
+		/// <param name="caCertificate">The ca certificate used to generate the nodes certificate </param>
+		/// <param name="trustRoot">Custom CA are never trusted by default unless they are in the machines trusted store, set this to true
+		/// if you've added the CA to the machines trusted store. In which case UntrustedRoot should not be accepted.
+		/// </param>
+		/// <param name="revocationMode">By default we do not check revocation, it is however recommended to check this (either offline or online).</param>
+		public static Func<object, X509Certificate, X509Chain, SslPolicyErrors, bool> AuthorityPartOfChain(
+			X509Certificate caCertificate, bool trustRoot = true, X509RevocationMode revocationMode = X509RevocationMode.NoCheck) =>
+			(sender, cert, chain, errors) =>
+				errors == SslPolicyErrors.None
+				|| ValidIntermediateCa(caCertificate, cert, chain, trustRoot, revocationMode);
+
+		/// <summary>
+		/// Helper to create a certificate validation callback based on the certificate authority certificate that we used to
+		/// generate the nodes certificates with. This callback does NOT expect the CA to be part of the chain presented by the server.
+		/// Including the root certificate in the chain increases the SSL handshake size and Elasticsearch's certgen by default does not include
+		/// the CA in the certificate chain.
+		/// </summary>
+		/// <param name="caCertificate">The ca certificate used to generate the nodes certificate </param>
+		/// <param name="trustRoot">Custom CA are never trusted by default unless they are in the machines trusted store, set this to true
+		/// if you've added the CA to the machines trusted store. In which case UntrustedRoot should not be accepted.
+		/// </param>
+		/// <param name="revocationMode">By default we do not check revocation, it is however recommended to check this (either offline or online).</param>
+		public static Func<object, X509Certificate, X509Chain, SslPolicyErrors, bool> AuthorityIsRoot(
+			X509Certificate caCertificate, bool trustRoot = true, X509RevocationMode revocationMode = X509RevocationMode.NoCheck) =>
+			(sender, cert, chain, errors) =>
+				errors == SslPolicyErrors.None
+				|| ValidRootCa(caCertificate, cert, chain, trustRoot, revocationMode);
+
+		private static X509Certificate2 to2(X509Certificate certificate)
+		{
+			#if DOTNETCORE
+				return new X509Certificate2(certificate.Export(X509ContentType.Cert));
+			#else
+				return new X509Certificate2(certificate);
+			#endif
+		}
+
+		private static bool ValidRootCa(X509Certificate caCertificate, X509Certificate certificate, X509Chain chain, bool trustRoot,
+			X509RevocationMode revocationMode)
+		{
+			var ca = to2(caCertificate);
+			var privateChain = new X509Chain {ChainPolicy = {RevocationMode = revocationMode}};
+			privateChain.ChainPolicy.ExtraStore.Add(ca);
+			privateChain.Build(to2(certificate));
+
+			//lets validate the our chain status
+			foreach (var chainStatus in privateChain.ChainStatus)
+			{
+				//custom CA's that are not in the machine trusted store will always have this status
+				//by setting trustRoot = true (default) we skip this error
+				if (chainStatus.Status == X509ChainStatusFlags.UntrustedRoot && trustRoot) continue;
+				//trustRoot is false so we expected our CA to be in the machines trusted store
+				if (chainStatus.Status == X509ChainStatusFlags.UntrustedRoot) return false;
+				//otherwise if the chain has any error of any sort return false
+				if (chainStatus.Status != X509ChainStatusFlags.NoError) return false;
+			}
+			return true;
+
+		}
+
+		private static bool ValidIntermediateCa(X509Certificate caCertificate, X509Certificate certificate, X509Chain chain, bool trustRoot,
+			X509RevocationMode revocationMode)
+		{
+			var ca = to2(caCertificate);
+			var privateChain = new X509Chain {ChainPolicy = {RevocationMode = revocationMode}};
+			privateChain.ChainPolicy.ExtraStore.Add(ca);
+			privateChain.Build(to2(certificate));
+
+			//Assert our chain has the same number of elements as the certifcate presented by the server
+			if (chain.ChainElements.Count != privateChain.ChainElements.Count) return false;
+
+			//lets validate the our chain status
+			foreach (var chainStatus in privateChain.ChainStatus)
+			{
+				//custom CA's that are not in the machine trusted store will always have this status
+				//by setting trustRoot = true (default) we skip this error
+				if (chainStatus.Status == X509ChainStatusFlags.UntrustedRoot && trustRoot) continue;
+				//trustRoot is false so we expected our CA to be in the machines trusted store
+				if (chainStatus.Status == X509ChainStatusFlags.UntrustedRoot) return false;
+				//otherwise if the chain has any error of any sort return false
+				if (chainStatus.Status != X509ChainStatusFlags.NoError) return false;
+			}
+
+			var found = false;
+			//We are going to walk both chains and make sure the thumbprints align
+			//while making sure one of the chains certificates presented by the server has our expected CA thumbprint
+			for (var i = 0; i < chain.ChainElements.Count; i++)
+			{
+				var c = chain.ChainElements[i].Certificate.Thumbprint;
+				var cPrivate = privateChain.ChainElements[i].Certificate.Thumbprint;
+				if (c == ca.Thumbprint) found = true;
+
+				//mis aligned certificate chain, return false so we do not accept this certificate
+				if (c != cPrivate) return false;
+				i++;
+			}
+			return found;
+		}
+	}
+}