DEV-344

src/Configuration/ServiceCollectionExtensions.cs

@@ -3,6 +3,8 @@
 using Serilog;
 using System;
 using System.Net.Http;
+using Microsoft.Extensions.Http.Resilience;
+using Polly;
 
 namespace MultiFactor.Ldap.Adapter.Configuration
 {
@@ -39,7 +41,16 @@ public static void AddHttpClientWithProxy(this IServiceCollection services)
 
                 return handler;
             })
-            .AddHttpMessageHandler<MfTraceIdHeaderSetter>();
+            .AddHttpMessageHandler<MfTraceIdHeaderSetter>()
+            .AddResilienceHandler("mf-api-pipeline", x =>
+            {
+                x.AddRetry(new HttpRetryStrategyOptions
+                {
+                    MaxRetryAttempts = 2,
+                    Delay = TimeSpan.FromSeconds(1),
+                    BackoffType = DelayBackoffType.Exponential
+                });
+            });
         }
     }
 }

src/Configuration/ServiceConfiguration.cs

@@ -1,5 +1,5 @@
 //Copyright(c) 2021 MultiFactor
-//Please see licence at 
+//Please see licence at
 //https://github.com/MultifactorLab/multifactor-ldap-adapter/blob/main/LICENSE.md
 
 using MultiFactor.Ldap.Adapter.Configuration.Core;
@@ -64,7 +64,7 @@ public ClientConfiguration GetClient(IPAddress ip)
         /// <summary>
         /// Multifactor API URL
         /// </summary>
-        public string ApiUrl { get; set; }
+        public string[] ApiUrls { get; set; }
 
         /// <summary>
         /// HTTP Proxy for API
@@ -90,7 +90,7 @@ public ClientConfiguration GetClient(IPAddress ip)
         /// Certificate for TLS
         /// </summary>
         public X509Certificate2 X509Certificate { get; set; }
-        
+
         /// <summary>
         /// Certificate Password
         /// </summary>
@@ -127,7 +127,7 @@ private void Load()
                 throw new Exception("Configuration error: 'logging-level' element not found");
             }
 
-            ApiUrl = apiUrlSetting;
+            ApiUrls = apiUrlSetting.Split(';', StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries).Distinct().ToArray();
             ApiTimeout = apiTimeout;
             ApiProxy = apiProxySetting;
             LogLevel = logLevelSetting;
@@ -137,7 +137,7 @@ private void Load()
             {
                 throw new Exception("Configuration error: Neither 'adapter-ldap-endpoint' or 'adapter-ldaps-endpoint' configured");
             }
-            
+
             ServerConfig = ldapServerConfig;
 
             if (!string.IsNullOrEmpty(certificatePassword))
@@ -230,12 +230,12 @@ private static ClientConfiguration Load(string name, AppSettingsSection appSetti
             }
 
             LdapIdentityFormat transformLdapIdentityFormat = LdapIdentityFormat.None;
-            if (!string.IsNullOrEmpty(transformLdapIdentityString) && 
+            if (!string.IsNullOrEmpty(transformLdapIdentityString) &&
                 !Enum.TryParse<LdapIdentityFormat>(transformLdapIdentityString, true, out transformLdapIdentityFormat))
             {
                 throw new Exception("Configuration error: 'transform-ldap-identity' element has a wrong value");
             }
-            
+
             var configuration = new ClientConfiguration
             {
                 Name = name,

src/MultiFactor.Ldap.Adapter.csproj

@@ -12,6 +12,7 @@
   <ItemGroup>
     <PackageReference Include="BouncyCastle.NetCore" Version="2.2.1" />
     <PackageReference Include="IPAddressRange" Version="6.0.0" />
+    <PackageReference Include="Microsoft.Extensions.Http.Resilience" Version="9.2.0" />
     <PackageReference Include="Newtonsoft.Json" Version="13.0.3" />
     <PackageReference Include="Serilog" Version="3.1.1" />
     <PackageReference Include="Serilog.Formatting.Compact" Version="2.0.0" />

src/Program.cs

@@ -1,4 +1,3 @@
-using Microsoft.AspNetCore.Hosting;
 using Microsoft.Extensions.DependencyInjection;
 using Microsoft.Extensions.Hosting;
 using MultiFactor.Ldap.Adapter.Configuration;

src/Services/MfTraceIdHeaderSetter.cs

@@ -25,7 +25,7 @@ protected override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage
             }
             else
             {
-                request.Headers.Add(_key, $"ldl-{Guid.NewGuid()}");
+                request.Headers.Add(_key, $"ldp-{Guid.NewGuid()}");
             }
             var resp = await base.SendAsync(request, cancellationToken);
 

src/Services/MultiFactorApiClient.cs

@@ -1,11 +1,12 @@
 //Copyright(c) 2021 MultiFactor
-//Please see licence at 
+//Please see licence at
 //https://github.com/MultifactorLab/multifactor-ldap-adapter/blob/main/LICENSE.md
 
 using MultiFactor.Ldap.Adapter.Configuration;
 using MultiFactor.Ldap.Adapter.Services.Caching;
 using Serilog;
 using System;
+using System.Linq;
 using System.Net;
 using System.Net.Http;
 using System.Text;
@@ -63,13 +64,12 @@ public async Task<bool> Authenticate(ConnectedClientInfo connectedClient)
                 return true;
             }
 
-            var url = _configuration.ApiUrl + "/access/requests/la";
             var payload = new
             {
                 Identity = connectedClient.Username,
             };
 
-            var response = await SendRequest(connectedClient.ClientConfiguration, url, payload);
+            var response = await SendRequest(connectedClient.ClientConfiguration, _configuration.ApiUrls, payload);
 
             if (response == null)
             {
@@ -78,7 +78,7 @@ public async Task<bool> Authenticate(ConnectedClientInfo connectedClient)
 
             if (response.Granted && !response.Bypassed)
             {
-                _logger.Information("Second factor for user '{user:l}' verified successfully. Authenticator '{authenticator:l}', account '{account:l}'", 
+                _logger.Information("Second factor for user '{user:l}' verified successfully. Authenticator '{authenticator:l}', account '{account:l}'",
                     connectedClient.Username, response?.Authenticator, response?.Account);
                 _clientCache.SetCache(connectedClient.Username, connectedClient.ClientConfiguration);
             }
@@ -87,80 +87,96 @@ public async Task<bool> Authenticate(ConnectedClientInfo connectedClient)
             {
                 var reason = response?.ReplyMessage;
                 var phone = response?.Phone;
-                _logger.Warning("Second factor verification for user '{user:l}' failed with reason='{reason:l}'. User phone {phone:l}", 
+                _logger.Warning("Second factor verification for user '{user:l}' failed with reason='{reason:l}'. User phone {phone:l}",
                     connectedClient.Username, reason, phone);
             }
 
             return response.Granted;
         }
 
-        private async Task<MultiFactorAccessRequest> SendRequest(ClientConfiguration clientConfig, string url, object payload)
+        private async Task<MultiFactorAccessRequest> SendRequest(ClientConfiguration clientConfig, string[] baseUrls, object payload)
         {
-            try
-            {
-                //make sure we can communicate securely
-                ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12;
-                ServicePointManager.DefaultConnectionLimit = 100;
+            //make sure we can communicate securely
+            ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12;
+            ServicePointManager.DefaultConnectionLimit = 100;
 
-                var json = JsonSerializer.Serialize(payload, _serialazerOptions);
+            var json = JsonSerializer.Serialize(payload, _serialazerOptions);
 
-                _logger.Debug($"Sending request to API: {json}");
+            _logger.Debug("Sending request to API: {Body}.", json);
 
-                //basic authorization
-                var auth = Convert.ToBase64String(Encoding.ASCII.GetBytes(clientConfig.MultifactorApiKey + ":" + clientConfig.MultifactorApiSecret));
-                var httpClient = _httpClientFactory.CreateClient(nameof(MultiFactorApiClient));
+            //basic authorization
+            var auth = Convert.ToBase64String(Encoding.ASCII.GetBytes($"{clientConfig.MultifactorApiKey}:{clientConfig.MultifactorApiSecret}"));
+            var httpClient = _httpClientFactory.CreateClient(nameof(MultiFactorApiClient));
 
-                StringContent jsonContent = new StringContent(json, Encoding.UTF8, "application/json");
-                HttpRequestMessage message = new HttpRequestMessage(HttpMethod.Post, url)
-                {
-                    Content = jsonContent
-                };
-                message.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Basic", auth);
-                var res = await httpClient.SendAsync(message);
-
-                if (res.StatusCode == HttpStatusCode.TooManyRequests)
+            foreach (var url in baseUrls.Select(baseUrl => $"{baseUrl}/access/requests/la"))
+            {
+                _logger.Information("Sending request to API '{ApiUrl:l}'.", url);
+                try
                 {
-                    _logger.Warning("Got unsuccessful response from API: {@response}", res.ReasonPhrase);
-                    return new MultiFactorAccessRequest() { Status = "Denied", ReplyMessage = "Too many requests"};
-                }
+                    var message = PrepareHttpRequestMessage(json, url, auth);
+                    var res = await TrySendRequest(httpClient, message);
+                    if (res == null)
+                        continue;
 
-                var jsonResponse = await res.Content.ReadAsStringAsync();
-                var response = JsonSerializer.Deserialize<MultiFactorApiResponse<MultiFactorAccessRequest>>(jsonResponse, _serialazerOptions);
+                    if (res.StatusCode == HttpStatusCode.TooManyRequests)
+                    {
+                        _logger.Warning("Got unsuccessful response from API '{ApiUrl:l}': {@response}", url, res.ReasonPhrase);
+                        return new MultiFactorAccessRequest { Status = "Denied", ReplyMessage = "Too many requests" };
+                    }
 
-                _logger.Debug("Received response from API: {@response}", response);
+                    var jsonResponse = await res.Content.ReadAsStringAsync();
+                    var response = JsonSerializer.Deserialize<MultiFactorApiResponse<MultiFactorAccessRequest>>(jsonResponse, _serialazerOptions);
 
-                if (!response.Success)
-                {
-                    _logger.Warning("Got unsuccessful response from API: {@response}", response);
-                }
+                    _logger.Debug("Received response from API '{ApiUrl:l}': {@response}", url, response);
 
-                return response.Model;
-            }
-            catch (TaskCanceledException tce)
-            {
-                _logger.Error(tce, $"Multifactor API host unreachable {url}: timeout");
+                    if (!response.Success)
+                    {
+                        _logger.Warning("Got unsuccessful response from API: {@response}", response);
+                        throw new HttpRequestException($"Got unsuccessful response from API. Status code: {res.StatusCode}.");
+                    }
+
+                    return response.Model;
 
-                if (clientConfig.BypassSecondFactorWhenApiUnreachable)
+                }
+                catch (Exception ex)
                 {
-                    _logger.Warning("Bypass second factor");
-                    return MultiFactorAccessRequest.Bypass;
+                    _logger.Error(ex, "Multifactor API host '{ApiUrl:l}' unreachable: {Message:l}", url, ex.Message);
                 }
-
-                return null;
             }
-            catch (Exception ex)
-            {
-                _logger.Error(ex, $"Multifactor API host unreachable {url}: {ex.Message}");
 
-                if (clientConfig.BypassSecondFactorWhenApiUnreachable)
-                {
-                    _logger.Warning("Bypass second factor");
-                    return MultiFactorAccessRequest.Bypass;
-                }
+            _logger.Error("Multifactor API Cloud unreachable");
 
+            if (clientConfig.BypassSecondFactorWhenApiUnreachable)
+            {
+                _logger.Warning("Bypass second factor");
+                return MultiFactorAccessRequest.Bypass;
+            }
+            return null;
+        }
+
+        private async Task<HttpResponseMessage> TrySendRequest(HttpClient httpClient, HttpRequestMessage message)
+        {
+            try
+            {
+                return await httpClient.SendAsync(message);
+            }
+            catch (HttpRequestException exception)
+            {
+                _logger.Warning("Failed to send request to API '{ApiUrl:l}': {Message:l}", message.RequestUri, exception.Message);
                 return null;
             }
         }
+
+        private static HttpRequestMessage PrepareHttpRequestMessage(string json, string url, string auth)
+        {
+            var jsonContent = new StringContent(json, Encoding.UTF8, "application/json");
+            var message = new HttpRequestMessage(HttpMethod.Post, url)
+            {
+                Content = jsonContent
+            };
+            message.Headers.Authorization = new System.Net.Http.Headers.AuthenticationHeaderValue("Basic", auth);
+            return message;
+        }
     }
 
     public class MultiFactorApiResponse<TModel>
@@ -190,6 +206,6 @@ public static MultiFactorAccessRequest Bypass
             {
                 return new MultiFactorAccessRequest { Status = "Granted", Bypassed = true };
             }
-        } 
+        }
     }
 }