Add support for logrotated and compressed access.log files

DeveLanCacheUI_Backend.Tests/DeveLanCacheUI_Backend.Tests.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk">
 
   <PropertyGroup>
-    <TargetFramework>net9.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
 
@@ -16,6 +16,7 @@
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>
+    <PackageReference Include="ZstdNet" Version="1.4.5" />
   </ItemGroup>
 
   <ItemGroup>

DeveLanCacheUI_Backend.Tests/LogReading/LogRotationTests.cs

@@ -0,0 +1,193 @@
+using DeveLanCacheUI_Backend.LogReading;
+using System.IO.Compression;
+using System.Text;
+using ZstdNet;
+
+namespace DeveLanCacheUI_Backend.Tests.LogReading
+{
+    [TestClass]
+    public class LogRotationTests
+    {
+        private string _tempDirectory = null!;
+
+        [TestInitialize]
+        public void Setup()
+        {
+            _tempDirectory = Path.Combine(Path.GetTempPath(), Guid.NewGuid().ToString());
+            Directory.CreateDirectory(_tempDirectory);
+        }
+
+        [TestCleanup]
+        public void Cleanup()
+        {
+            if (Directory.Exists(_tempDirectory))
+            {
+                Directory.Delete(_tempDirectory, true);
+            }
+        }
+
+        [TestMethod]
+        public void GetLogFiles_ReturnsCorrectOrderWithBasicFiles()
+        {
+            // Arrange
+            var sut = new LanCacheLogReaderHostedService(null!, null!, null!, null!);
+
+            // Create test files
+            File.WriteAllText(Path.Combine(_tempDirectory, "access.log"), "current log");
+            File.WriteAllText(Path.Combine(_tempDirectory, "access.log.1"), "rotated 1");
+            File.WriteAllText(Path.Combine(_tempDirectory, "access.log.2"), "rotated 2");
+
+            // Act
+            var result = InvokePrivateMethod<List<string>>(sut, "GetLogFiles", _tempDirectory);
+
+            // Assert
+            Assert.AreEqual(3, result.Count);
+            Assert.IsTrue(result[0].EndsWith("access.log"));
+            Assert.IsTrue(result[1].EndsWith("access.log.1"));
+            Assert.IsTrue(result[2].EndsWith("access.log.2"));
+        }
+
+        [TestMethod]
+        public void GetLogFiles_ReturnsCorrectOrderWithCompressedFiles()
+        {
+            // Arrange
+            var sut = new LanCacheLogReaderHostedService(null!, null!, null!, null!);
+
+            // Create test files
+            File.WriteAllText(Path.Combine(_tempDirectory, "access.log"), "current log");
+            File.WriteAllText(Path.Combine(_tempDirectory, "access.log.1.gz"), "compressed 1");
+            File.WriteAllText(Path.Combine(_tempDirectory, "access.log.2.zst"), "compressed 2");
+
+            // Act
+            var result = InvokePrivateMethod<List<string>>(sut, "GetLogFiles", _tempDirectory);
+
+            // Assert
+            Assert.AreEqual(3, result.Count);
+            Assert.IsTrue(result[0].EndsWith("access.log"));
+            Assert.IsTrue(result[1].EndsWith("access.log.1.gz"));
+            Assert.IsTrue(result[2].EndsWith("access.log.2.zst"));
+        }
+
+        [TestMethod]
+        public void GetLogFiles_HandlesGapInNumbers()
+        {
+            // Arrange
+            var sut = new LanCacheLogReaderHostedService(null!, null!, null!, null!);
+
+            // Create test files with gaps
+            File.WriteAllText(Path.Combine(_tempDirectory, "access.log"), "current log");
+            File.WriteAllText(Path.Combine(_tempDirectory, "access.log.1"), "rotated 1");
+            // Skip access.log.2
+            File.WriteAllText(Path.Combine(_tempDirectory, "access.log.3"), "rotated 3");
+
+            // Act
+            var result = InvokePrivateMethod<List<string>>(sut, "GetLogFiles", _tempDirectory);
+
+            // Assert - should stop at the first gap
+            Assert.AreEqual(2, result.Count);
+            Assert.IsTrue(result[0].EndsWith("access.log"));
+            Assert.IsTrue(result[1].EndsWith("access.log.1"));
+        }
+
+        [TestMethod]
+        public void OpenLogFileStream_HandlesRegularFile()
+        {
+            // Arrange
+            var sut = new LanCacheLogReaderHostedService(null!, null!, null!, null!);
+            var testFile = Path.Combine(_tempDirectory, "test.log");
+            var testContent = "test line 1\ntest line 2\n";
+            File.WriteAllText(testFile, testContent);
+
+            // Act
+            using var stream = InvokePrivateMethod<Stream>(sut, "OpenLogFileStream", testFile);
+            using var reader = new StreamReader(stream);
+            var content = reader.ReadToEnd();
+
+            // Assert
+            Assert.AreEqual(testContent, content);
+        }
+
+        [TestMethod]
+        public void OpenLogFileStream_HandlesGzipFile()
+        {
+            // Arrange
+            var sut = new LanCacheLogReaderHostedService(null!, null!, null!, null!);
+            var testFile = Path.Combine(_tempDirectory, "test.log.gz");
+            var testContent = "test line 1\ntest line 2\n";
+
+            // Create gzipped file
+            using (var fileStream = File.Create(testFile))
+            using (var gzipStream = new GZipStream(fileStream, CompressionMode.Compress))
+            using (var writer = new StreamWriter(gzipStream))
+            {
+                writer.Write(testContent);
+            }
+
+            // Act
+            using var stream = InvokePrivateMethod<Stream>(sut, "OpenLogFileStream", testFile);
+            using var reader = new StreamReader(stream);
+            var content = reader.ReadToEnd();
+
+            // Assert
+            Assert.AreEqual(testContent, content);
+        }
+
+        [TestMethod]
+        public void OpenLogFileStream_HandlesZstdFile()
+        {
+            // Arrange
+            var sut = new LanCacheLogReaderHostedService(null!, null!, null!, null!);
+            var testFile = Path.Combine(_tempDirectory, "test.log.zst");
+            var testContent = "test line 1\ntest line 2\n";
+
+            // Create zstd compressed file
+            using (var compressor = new Compressor())
+            {
+                var originalBytes = Encoding.UTF8.GetBytes(testContent);
+                var compressedBytes = compressor.Wrap(originalBytes);
+                File.WriteAllBytes(testFile, compressedBytes);
+            }
+
+            // Act
+            using var stream = InvokePrivateMethod<Stream>(sut, "OpenLogFileStream", testFile);
+            using var reader = new StreamReader(stream);
+            var content = reader.ReadToEnd();
+
+            // Assert
+            Assert.AreEqual(testContent, content);
+        }
+
+        [TestMethod]
+        public void ReadAllLinesFromStream_ReadsAllLines()
+        {
+            // Arrange
+            var sut = new LanCacheLogReaderHostedService(null!, null!, null!, null!);
+            var testContent = "line1\nline2\nline3\n";
+            using var stream = new MemoryStream(Encoding.UTF8.GetBytes(testContent));
+            var cts = new CancellationTokenSource();
+
+            // Act
+            var result = InvokePrivateMethod<IEnumerable<string>>(sut, "ReadAllLinesFromStream", stream, cts.Token).ToList();
+
+            // Assert
+            Assert.AreEqual(3, result.Count);
+            Assert.AreEqual("line1", result[0]);
+            Assert.AreEqual("line2", result[1]);
+            Assert.AreEqual("line3", result[2]);
+        }
+
+        /// <summary>
+        /// Helper method to invoke private methods for testing
+        /// </summary>
+        private T InvokePrivateMethod<T>(object obj, string methodName, params object[] parameters)
+        {
+            var type = obj.GetType();
+            var method = type.GetMethod(methodName, System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance);
+            if (method == null)
+                throw new ArgumentException($"Method {methodName} not found");
+
+            var result = method.Invoke(obj, parameters);
+            return (T)result!;
+        }
+    }
+}

DeveLanCacheUI_Backend/Db/DbModels/DbSetting.cs

@@ -9,5 +9,6 @@ public class DbSetting
         public const string SettingKey_DepotVersion = nameof(SettingKey_DepotVersion);
         public const string SettingKey_SteamChangeNumber = nameof(SettingKey_SteamChangeNumber);
         public const string SettingKey_TotalBytesRead = nameof(SettingKey_TotalBytesRead);
+        public const string SettingKey_ProcessedLogFiles = nameof(SettingKey_ProcessedLogFiles);
     }
 }

DeveLanCacheUI_Backend/DeveLanCacheUI_Backend.csproj

@@ -1,7 +1,7 @@
 <Project Sdk="Microsoft.NET.Sdk.Web">
 
   <PropertyGroup>
-    <TargetFramework>net9.0</TargetFramework>
+    <TargetFramework>net8.0</TargetFramework>
     <Nullable>enable</Nullable>
     <ImplicitUsings>enable</ImplicitUsings>
     <UserSecretsId>5be89f33-044f-41d5-b192-06d2df23e484</UserSecretsId>
@@ -22,18 +22,19 @@
 
   <ItemGroup>
     <PackageReference Include="DeveHashImageGenerator" Version="1.0.12" />
-    <PackageReference Include="Microsoft.AspNetCore.OpenApi" Version="9.0.4" />
-    <PackageReference Include="Microsoft.EntityFrameworkCore" Version="9.0.4" />
-    <PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="9.0.4">
+    <PackageReference Include="Microsoft.AspNetCore.OpenApi" Version="8.0.10" />
+    <PackageReference Include="Microsoft.EntityFrameworkCore" Version="8.0.10" />
+    <PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="8.0.10">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>
-    <PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="9.0.4" />
+    <PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="8.0.10" />
     <PackageReference Include="Microsoft.VisualStudio.Azure.Containers.Tools.Targets" Version="1.21.2" />
     <PackageReference Include="Polly" Version="8.5.2" />
     <PackageReference Include="protobuf-net" Version="3.2.52" />
     <PackageReference Include="SteamKit2" Version="3.2.0" />
     <PackageReference Include="Swashbuckle.AspNetCore" Version="8.1.1" />
+    <PackageReference Include="ZstdNet" Version="1.4.5" />
   </ItemGroup>
 
   <ItemGroup>