feat: add timer trigger to refresh org cache

src/Dan.Plugin.Tilda/Functions/Timers.cs

@@ -5,19 +5,22 @@
 using Dan.Plugin.Tilda.Clients;
 using Dan.Plugin.Tilda.Exceptions;
 using Dan.Plugin.Tilda.Interfaces;
-using Dan.Plugin.Tilda.Models.AlertMessages;
 using Dan.Plugin.Tilda.Services;
 using Dan.Plugin.Tilda.Utils;
 using Dan.Tilda.Models.Alerts;
 using Microsoft.Azure.Functions.Worker;
-using Microsoft.Azure.Functions.Worker.Http;
+using Microsoft.Extensions.Logging;
+using StackExchange.Redis;
 
 namespace Dan.Plugin.Tilda.Functions;
 
 public class Timers(
     ITildaSourceProvider tildaSourceProvider,
     IMtamCounterClient mtamCounterClient,
-    IAlertMessageSender alertMessageSender)
+    IAlertMessageSender alertMessageSender,
+    IConnectionMultiplexer connectionMultiplexer,
+    IBrregService brregService,
+    ILogger<Timers> logger)
 {
     // MTAM - Melding til annen myndighet (message to other authority/auditor)
     [Function("MtamTimer")]
@@ -36,7 +39,7 @@ public async Task MessageToOtherAuditorsTimer(
             {
                 messages = await mtamSource.GetAlertMessagesAsync(from);
             }
-            catch (FailedToFetchDataException e)
+            catch (FailedToFetchDataException)
             {
                 // If we fail to fetch alert messages from a source, we should continue with the rest of sources
                 // but not update this source's counter so we'll try to fetch from the same time again next attempt
@@ -54,4 +57,96 @@ public async Task MessageToOtherAuditorsTimer(
             await mtamCounterClient.UpsertMtamCounter(mtamCounter);
         }
     }
+
+    [Function("CacheRefreshTimer")]
+    public async Task CacheRefreshTimer(
+        //[HttpTrigger(AuthorizationLevel.Function, "get", "post")] HttpRequestData req, // easier to test locally using http trigger
+        [TimerTrigger("%CacheRefreshCron%")] TimerInfo timerInfo,
+        FunctionContext context)
+    {
+        var db = connectionMultiplexer.GetDatabase();
+        var now =  DateTime.UtcNow;
+        var mainunitKeys = new List<string>();
+        var subunitKeys = new List<string>();
+        const string mainKeyPrefix = "Tilda-Cache_Absolute_GET_https://data.brreg.no/enhetsregisteret/api/enheter/";
+        const string subunitKeyPrefix = "Tilda-Cache_Absolute_GET_https://data.brreg.no/enhetsregisteret/api/enheter/?overordnetEnhet=";
+
+        var keys = GetKeysAsync("Tilda-Cache_Absolute*");
+        await foreach (var key in keys)
+        {
+            var expiry = await db.KeyExpireTimeAsync(key);
+            if (expiry is null)
+            {
+                logger.LogInformation("Expiry for key {key} is null", key);
+                continue;
+            }
+            var timeLeft = expiry - now;
+            if (!(timeLeft <= TimeSpan.FromMinutes(11)))
+            {
+                continue;
+            }
+
+            if (key.StartsWith(subunitKeyPrefix))
+            {
+                var subKey =  key.Replace(subunitKeyPrefix, "");
+                subunitKeys.Add(subKey);
+            }
+            else if (key.StartsWith(mainKeyPrefix))
+            {
+                var mainKey = key.Replace(mainKeyPrefix, "");
+                mainunitKeys.Add(mainKey);
+            }
+        }
+
+        if (mainunitKeys.Count == 0 && subunitKeys.Count == 0)
+        {
+            return;
+        }
+
+        logger.LogInformation("Refreshing {mainKeys} main keys and {subunitKeys} subunit keys", mainunitKeys.Count, subunitKeys.Count);
+
+        var tasks = new List<Task>();
+        foreach (var key in mainunitKeys)
+        {
+            tasks.Add(RefreshOrganisationEntry(key, includeSubunits: false));
+        }
+
+        foreach (var key in subunitKeys)
+        {
+            tasks.Add(RefreshOrganisationEntry(key, includeSubunits:true));
+        }
+
+        await Task.WhenAll(tasks);
+    }
+
+    private async IAsyncEnumerable<string> GetKeysAsync(string pattern)
+    {
+        if (string.IsNullOrWhiteSpace(pattern))
+        {
+            throw new ArgumentException("Value cannot be null or whitespace.", nameof(pattern));
+        }
+
+        foreach (var endpoint in connectionMultiplexer.GetEndPoints())
+        {
+            var server = connectionMultiplexer.GetServer(endpoint);
+            await foreach (var key in server.KeysAsync(pattern: pattern))
+            {
+                yield return key.ToString();
+            }
+        }
+    }
+
+    private async Task RefreshOrganisationEntry(string org, bool includeSubunits)
+    {
+        try
+        {
+            await brregService.GetFromBr(org, includeSubunits, skipCache: true);
+        }
+        catch (Exception e)
+        {
+            // Logging warning would be flooding the alerts/dashboards a bit too much, it's not really an issue if
+            // something fails to refresh, this is just a feature to prevent regular calls being too long too often
+            logger.LogInformation("Failed to refresh organisation cache entry for {org}, exception message: {exMessage}", org, e.Message);
+        }
+    }
 }

src/Dan.Plugin.Tilda/Program.cs

@@ -13,6 +13,7 @@
 using System.Linq;
 using System.Net.Http;
 using System.Reflection;
+using System.Threading.Tasks;
 using Altinn.ApiClients.Maskinporten.Interfaces;
 using Altinn.ApiClients.Maskinporten.Services;
 using Azure.Identity;
@@ -52,21 +53,20 @@
             {
                 option.Configuration = settings.RedisConnectionString;
             });
+            services.AddSingleton<IConnectionMultiplexer>(ConnectionMultiplexer.Connect(settings.RedisConnectionString));
         }
         else
         {
+            var configurationOptions = ConfigurationOptions
+                .Parse(settings.RedisConnectionString)
+                .ConfigureForAzureWithTokenCredentialAsync(credentials)
+                .GetAwaiter().GetResult();
+
+            IConnectionMultiplexer connectionMultiplexer = ConnectionMultiplexer.Connect(configurationOptions);
+            services.AddSingleton(connectionMultiplexer);
             services.AddStackExchangeRedisCache(option =>
             {
-                option.ConnectionMultiplexerFactory = async () =>
-                {
-                    var configurationOptions = await ConfigurationOptions
-                        .Parse(settings.RedisConnectionString)
-                        .ConfigureForAzureWithTokenCredentialAsync(credentials);
-
-                    var connectionMultiplexer = await ConnectionMultiplexer.ConnectAsync(configurationOptions);
-
-                    return connectionMultiplexer;
-                };
+                option.ConnectionMultiplexerFactory = () => Task.FromResult(connectionMultiplexer);
             });
         }
 

src/Dan.Plugin.Tilda/Services/BrregService.cs

@@ -19,7 +19,7 @@ public interface IBrregService
 {
     Task<AccountsInformation> GetAnnualTurnoverFromBr(string organizationNumber);
 
-    Task<List<BREntityRegisterEntry>> GetFromBr(string organization, bool? includeSubunits);
+    Task<List<BREntityRegisterEntry>> GetFromBr(string organization, bool? includeSubunits, bool skipCache = false);
 
     Task<List<string>> GetKofuviAddresses(string organizationNumber);
 }
@@ -92,7 +92,7 @@ public async Task<AccountsInformation> GetAnnualTurnoverFromBr(string organizati
         return result;
     }
 
-    public async Task<List<BREntityRegisterEntry>> GetFromBr(string organization, bool? includeSubunits)
+    public async Task<List<BREntityRegisterEntry>> GetFromBr(string organization, bool? includeSubunits, bool skipCache = false)
     {
         var result = new List<BREntityRegisterEntry>();
 
@@ -110,11 +110,11 @@ public async Task<List<BREntityRegisterEntry>> GetFromBr(string organization, bo
 
         if (includeSubunits == true)
         {
-            result.AddRange(await GetAllUnitsFromBr(organization));
+            result.AddRange(await GetAllUnitsFromBr(organization, skipCache));
         }
         else
         {
-            result.Add(await GetOrganizationInfoFromBr(organization));
+            result.Add(await GetOrganizationInfoFromBr(organization, skipCache));
         }
 
         return result;
@@ -207,7 +207,7 @@ public async Task<List<string>> GetKofuviAddresses(string organizationNumber)
         }
     }
 
-    private async Task<BREntityRegisterEntry> GetOrganizationInfoFromBr(string organizationNumber)
+    private async Task<BREntityRegisterEntry> GetOrganizationInfoFromBr(string organizationNumber, bool skipCache = false)
     {
         var mainUnitUrl = $"https://data.brreg.no/enhetsregisteret/api/enheter/{organizationNumber}";
         var subUnitUrl = $"https://data.brreg.no/enhetsregisteret/api/underenheter/{organizationNumber}";
@@ -217,19 +217,22 @@ private async Task<BREntityRegisterEntry> GetOrganizationInfoFromBr(string organ
         string rawResult;
         try
         {
-            try
+            if (!skipCache)
             {
-                result = await cache.GetValueAsync<BREntityRegisterEntry>(cacheKey);
-                if (result is not null)
+                try
                 {
-                    return result;
+                    result = await cache.GetValueAsync<BREntityRegisterEntry>(cacheKey);
+                    if (result is not null)
+                    {
+                        return result;
+                    }
+                }
+                catch (Exception e)
+                {
+                    logger.LogWarning("Failed to get GetOrganizationInfoFromBr from cache for {orgNumber}, fetching from source directly. Exception message: {message}",
+                        organizationNumber,
+                        e.Message);
                 }
-            }
-            catch (Exception e)
-            {
-                logger.LogWarning("Failed to get GetOrganizationInfoFromBr from cache for {orgNumber}, fetching from source directly. Exception message: {message}",
-                    organizationNumber,
-                    e.Message);
             }
 
             var response = await _erClient.GetAsync(mainUnitUrl);
@@ -270,7 +273,7 @@ private async Task<BREntityRegisterEntry> GetOrganizationInfoFromBr(string organ
         return result;
     }
 
-    private async Task<List<BREntityRegisterEntry>> GetAllUnitsFromBr(string organizationNumber)
+    private async Task<List<BREntityRegisterEntry>> GetAllUnitsFromBr(string organizationNumber, bool skipCache = false)
     {
         List<BREntityRegisterEntry> result;
         string rawResult;
@@ -279,19 +282,22 @@ private async Task<List<BREntityRegisterEntry>> GetAllUnitsFromBr(string organiz
 
         try
         {
-            try
+            if (!skipCache)
             {
-                result = await cache.GetValueAsync<List<BREntityRegisterEntry>>(cacheKey);
-                if (result is not null)
+                try
                 {
-                    return result;
+                    result = await cache.GetValueAsync<List<BREntityRegisterEntry>>(cacheKey);
+                    if (result is not null)
+                    {
+                        return result;
+                    }
+                }
+                catch (Exception e)
+                {
+                    logger.LogWarning("Failed to get GetAllUnitsFromBr from cache for {orgNumber}, fetching from source directly. Exception message: {message}",
+                        organizationNumber,
+                        e.Message);
                 }
-            }
-            catch (Exception e)
-            {
-                logger.LogWarning("Failed to get GetAllUnitsFromBr from cache for {orgNumber}, fetching from source directly. Exception message: {message}",
-                    organizationNumber,
-                    e.Message);
             }
 
             result = [];