From 38f4d1da3d2d837189dfc0e3ae25ee911022713c Mon Sep 17 00:00:00 2001
From: Lilian ARAGO <larago@lucca.fr>
Date: Fri, 10 Feb 2023 15:49:01 +0100
Subject: [PATCH 01/16] Uploading files to S3

---
 TCC.Lib/Options/CompressOption.cs       |  9 +++--
 TCC.Lib/Storage/RemoteStorageFactory.cs | 23 +++++++++++-
 TCC.Lib/Storage/S3RemoteStorage.cs      | 48 +++++++++++++++++++++++++
 TCC.Lib/TCC.Lib.csproj                  |  1 +
 TCC.Lib/UploadMode.cs                   |  5 +--
 TCC/Parser/CompressCommand.cs           |  7 +++-
 6 files changed, 87 insertions(+), 6 deletions(-)
 create mode 100644 TCC.Lib/Storage/S3RemoteStorage.cs

diff --git a/TCC.Lib/Options/CompressOption.cs b/TCC.Lib/Options/CompressOption.cs
index f64aef4..391a1f1 100644
--- a/TCC.Lib/Options/CompressOption.cs
+++ b/TCC.Lib/Options/CompressOption.cs
@@ -21,7 +21,12 @@ public class CompressOption : TccOption
         public string AzBlobSaS { get; set; }
         public int? AzThread { get; set; }
         public string GoogleStorageBucketName { get; set; }
-        public string GoogleStorageCredential { get; set; }
-        public UploadMode? UploadMode { get; set; }
+        public string GoogleStorageCredential { get; set; }
+        public string S3AccessKeyId { get; set; }
+        public string S3SecretAcessKey { get; set; }
+        public string S3Host { get; set; }
+        public string S3BucketName { get; set; }
+        public string S3Region { get; set; }
+        public UploadMode? UploadMode { get; set; }
     }
 }
\ No newline at end of file
diff --git a/TCC.Lib/Storage/RemoteStorageFactory.cs b/TCC.Lib/Storage/RemoteStorageFactory.cs
index 8d97434..f72763e 100644
--- a/TCC.Lib/Storage/RemoteStorageFactory.cs
+++ b/TCC.Lib/Storage/RemoteStorageFactory.cs
@@ -1,4 +1,6 @@
-using Azure.Storage.Blobs;
+using Amazon.Runtime;
+using Amazon.S3;
+using Azure.Storage.Blobs;
 using Google.Apis.Auth.OAuth2;
 using Google.Cloud.Storage.V1;
 using Microsoft.Extensions.Logging;
@@ -42,6 +44,25 @@ public static async Task<IRemoteStorage> GetRemoteStorageAsync(this CompressOpti
                     StorageClient storage = await GoogleAuthHelper.GetGoogleStorageClientAsync(option.GoogleStorageCredential, token);
                     return new GoogleRemoteStorage(storage, option.GoogleStorageBucketName);
                 }
+                case UploadMode.S3:
+                    if (string.IsNullOrEmpty(option.S3AccessKeyId)
+                        || string.IsNullOrEmpty(option.S3Host)
+                        || string.IsNullOrEmpty(option.S3Region)
+                        || string.IsNullOrEmpty(option.S3BucketName)
+                        || string.IsNullOrEmpty(option.S3SecretAcessKey))
+                    {
+                        logger.LogCritical("Configuration error for S3 upload");
+                        return new NoneRemoteStorage();
+                    }
+
+                    var credentials = new BasicAWSCredentials(option.S3AccessKeyId, option.S3SecretAcessKey);
+                    var s3Config = new AmazonS3Config()
+                    {
+                        AuthenticationRegion = option.S3Region,
+                        ServiceURL = option.S3Host,
+                    };
+
+                    return new S3RemoteStorage(new AmazonS3Client(credentials, s3Config), option.S3BucketName);
                 case UploadMode.None:
                 case null:
                     return new NoneRemoteStorage();
diff --git a/TCC.Lib/Storage/S3RemoteStorage.cs b/TCC.Lib/Storage/S3RemoteStorage.cs
new file mode 100644
index 0000000..2e0533f
--- /dev/null
+++ b/TCC.Lib/Storage/S3RemoteStorage.cs
@@ -0,0 +1,48 @@
+using Amazon;
+using Amazon.Runtime;
+using Amazon.S3;
+using System;
+using System.IO;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace TCC.Lib.Storage
+{
+    public class S3RemoteStorage : IRemoteStorage
+    {
+        internal string BucketName { get; }
+        private readonly AmazonS3Client _s3Client;
+
+        public S3RemoteStorage(AmazonS3Client s3Client, string bucketName)
+        {
+            BucketName = bucketName;
+            _s3Client = s3Client;
+        }
+
+        public async Task<UploadResponse> UploadAsync(string targetPath, Stream data, CancellationToken token)
+        {
+            try
+            {
+                await _s3Client.PutObjectAsync(new Amazon.S3.Model.PutObjectRequest() {
+                    BucketName = BucketName,
+                    Key = targetPath,
+                    InputStream = data,
+                }, token);
+            }
+            catch (Exception e)
+            {
+                return new UploadResponse
+                {
+                    IsSuccess = false,
+                    RemoteFilePath = targetPath,
+                    ErrorMessage = e.Message
+                };
+            }
+            return new UploadResponse
+            {
+                IsSuccess = true,
+                RemoteFilePath = targetPath
+            };
+        }
+    }
+}
\ No newline at end of file
diff --git a/TCC.Lib/TCC.Lib.csproj b/TCC.Lib/TCC.Lib.csproj
index 8534aae..25043fa 100644
--- a/TCC.Lib/TCC.Lib.csproj
+++ b/TCC.Lib/TCC.Lib.csproj
@@ -6,6 +6,7 @@
   </PropertyGroup>
 
   <ItemGroup>
+    <PackageReference Include="AWSSDK.S3" Version="3.7.10.1" />
     <PackageReference Include="Azure.Storage.Blobs" Version="12.13.0" />
     <PackageReference Include="Google.Cloud.Storage.V1" Version="3.7.0" />
     <PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="6.0.7">
diff --git a/TCC.Lib/UploadMode.cs b/TCC.Lib/UploadMode.cs
index 6ff5896..ebfae59 100644
--- a/TCC.Lib/UploadMode.cs
+++ b/TCC.Lib/UploadMode.cs
@@ -1,9 +1,10 @@
-namespace TCC.Lib
+namespace TCC.Lib
 {
     public enum UploadMode
     {
         None,
         AzureSdk,
-        GoogleCloudStorage
+        GoogleCloudStorage,
+        S3
     }
 }
\ No newline at end of file
diff --git a/TCC/Parser/CompressCommand.cs b/TCC/Parser/CompressCommand.cs
index 8814060..9cbcc87 100644
--- a/TCC/Parser/CompressCommand.cs
+++ b/TCC/Parser/CompressCommand.cs
@@ -64,7 +64,12 @@ protected override IEnumerable<Option> CreateOptions()
             yield return new Option<string>(new[] { "--azBlobSaS" }, "Azure blob storage SaS token");
             yield return new Option<int?>(new[] { "--azThread" }, "Azure blob maximum parallel threads");
             yield return new Option<string>(new[] { "--googleStorageBucketName" }, "Google Cloud Storage destination bucket");
-            yield return new Option<string>(new[] { "--googleStorageCredential" }, "Google Cloud Storage credential json, either full path or base64");
+            yield return new Option<string>(new[] { "--googleStorageCredential" }, "Google Cloud Storage credential json, either full path or base64");
+            yield return new Option<string>(new[] { "--s3AccessKeyId" }, "S3 Access Key ID");
+            yield return new Option<string>(new[] { "--s3Region" }, "S3 Region");
+            yield return new Option<string>(new[] { "--s3SecretAcessKey" }, "S3 Access Key Secret");
+            yield return new Option<string>(new[] { "--s3Host" }, "S3 Host");
+            yield return new Option<string>(new[] { "--s3BucketName" }, "S3 destination bucket");
         }
         protected override Task RunAsync(ITccController controller, CompressOptionBinding option) => controller.CompressAsync(option);
     }

From 58b5adeba4af275507170b802a8947460e4aabde Mon Sep 17 00:00:00 2001
From: Lilian ARAGO <larago@lucca.fr>
Date: Tue, 14 Feb 2023 16:50:26 +0100
Subject: [PATCH 02/16] Multiple uploadModes

---
 TCC.Lib/OperationBlock.cs               |   1 +
 TCC.Lib/Options/CompressOption.cs       |   2 +
 TCC.Lib/Storage/AzureRemoteStorage.cs   |   5 +-
 TCC.Lib/Storage/GoogleRemoteStorage.cs  |   4 +-
 TCC.Lib/Storage/IRemoteStorage.cs       |   6 +-
 TCC.Lib/Storage/NoneRemoteStorage.cs    |   4 +-
 TCC.Lib/Storage/RemoteStorageFactory.cs | 120 ++++++++++--------
 TCC.Lib/Storage/S3RemoteStorage.cs      |   2 +
 TCC.Lib/TarCompressCrypt.cs             | 159 ++++++++++++------------
 TCC.Tests/Upload/BlobStorageTests.cs    |  13 +-
 TCC/Parser/CompressCommand.cs           |   3 +-
 11 files changed, 176 insertions(+), 143 deletions(-)

diff --git a/TCC.Lib/OperationBlock.cs b/TCC.Lib/OperationBlock.cs
index f5e7955..ebbda5e 100644
--- a/TCC.Lib/OperationBlock.cs
+++ b/TCC.Lib/OperationBlock.cs
@@ -236,6 +236,7 @@ public class StepResult
         public string Errors { get; set; }
         public string Warning { get; set; }
         public string Infos { get; set; }
+        public UploadMode? UploadMode { get; set; }
         public bool IsSuccess => !HasError && !HasWarning;
         public bool HasError => !string.IsNullOrWhiteSpace(Errors);
         public bool HasWarning => !string.IsNullOrWhiteSpace(Warning);
diff --git a/TCC.Lib/Options/CompressOption.cs b/TCC.Lib/Options/CompressOption.cs
index 391a1f1..6d84a33 100644
--- a/TCC.Lib/Options/CompressOption.cs
+++ b/TCC.Lib/Options/CompressOption.cs
@@ -1,4 +1,5 @@
 using System.Collections.Generic;
+using System.Linq;
 using TCC.Lib.Blocks;
 using TCC.Lib.Database;
 
@@ -27,6 +28,7 @@ public class CompressOption : TccOption
         public string S3Host { get; set; }
         public string S3BucketName { get; set; }
         public string S3Region { get; set; }
+        public IEnumerable<UploadMode> UploadModes { get; set; } = Enumerable.Empty<UploadMode>();
         public UploadMode? UploadMode { get; set; }
     }
 }
\ No newline at end of file
diff --git a/TCC.Lib/Storage/AzureRemoteStorage.cs b/TCC.Lib/Storage/AzureRemoteStorage.cs
index 43287c7..3e61f66 100644
--- a/TCC.Lib/Storage/AzureRemoteStorage.cs
+++ b/TCC.Lib/Storage/AzureRemoteStorage.cs
@@ -24,6 +24,9 @@ public async Task<UploadResponse> UploadAsync(string targetPath, Stream data, Ca
                 ErrorMessage = response.ReasonPhrase,
                 RemoteFilePath = targetPath
             };
-        }
+        }
+
+        public UploadMode GetMode() => UploadMode.AzureSdk;
+
     }
 }
\ No newline at end of file
diff --git a/TCC.Lib/Storage/GoogleRemoteStorage.cs b/TCC.Lib/Storage/GoogleRemoteStorage.cs
index 2b85cdc..c5c2d84 100644
--- a/TCC.Lib/Storage/GoogleRemoteStorage.cs
+++ b/TCC.Lib/Storage/GoogleRemoteStorage.cs
@@ -38,6 +38,8 @@ public async Task<UploadResponse> UploadAsync(string targetPath, Stream data, Ca
                 IsSuccess = true,
                 RemoteFilePath = targetPath
             };
-        }
+        }
+
+        public UploadMode GetMode() => UploadMode.GoogleCloudStorage;
     }
 }
\ No newline at end of file
diff --git a/TCC.Lib/Storage/IRemoteStorage.cs b/TCC.Lib/Storage/IRemoteStorage.cs
index 273348e..71b7765 100644
--- a/TCC.Lib/Storage/IRemoteStorage.cs
+++ b/TCC.Lib/Storage/IRemoteStorage.cs
@@ -8,12 +8,14 @@ namespace TCC.Lib.Storage
     public interface IRemoteStorage
     {
         Task<UploadResponse> UploadAsync(string targetPath, Stream data, CancellationToken token);
-
+
         public async Task<UploadResponse> UploadAsync(FileInfo file, DirectoryInfo rootFolder, CancellationToken token)
         {
             string targetPath = file.GetRelativeTargetPathTo(rootFolder);
             await using FileStream uploadFileStream = File.OpenRead(file.FullName);
             return await UploadAsync(targetPath, uploadFileStream, token);
-        }
+        }
+
+        public UploadMode GetMode();
     }
 }
\ No newline at end of file
diff --git a/TCC.Lib/Storage/NoneRemoteStorage.cs b/TCC.Lib/Storage/NoneRemoteStorage.cs
index eb3e942..bbb3160 100644
--- a/TCC.Lib/Storage/NoneRemoteStorage.cs
+++ b/TCC.Lib/Storage/NoneRemoteStorage.cs
@@ -9,6 +9,8 @@ public class NoneRemoteStorage : IRemoteStorage
         public Task<UploadResponse> UploadAsync(string targetPath, Stream data, CancellationToken token)
         {
             return Task.FromResult(new UploadResponse { IsSuccess = true, RemoteFilePath = targetPath });
-        }
+        }
+
+        public UploadMode GetMode() => UploadMode.None;
     }
 }
diff --git a/TCC.Lib/Storage/RemoteStorageFactory.cs b/TCC.Lib/Storage/RemoteStorageFactory.cs
index f72763e..fe85b35 100644
--- a/TCC.Lib/Storage/RemoteStorageFactory.cs
+++ b/TCC.Lib/Storage/RemoteStorageFactory.cs
@@ -5,7 +5,11 @@
 using Google.Cloud.Storage.V1;
 using Microsoft.Extensions.Logging;
 using System;
+using System.Collections;
+using System.Collections.Generic;
+using System.ComponentModel;
 using System.IO;
+using System.Linq;
 using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
@@ -16,59 +20,69 @@ namespace TCC.Lib.Storage
 {
     public static class RemoteStorageFactory
     {
-        public static async Task<IRemoteStorage> GetRemoteStorageAsync(this CompressOption option, ILogger logger, CancellationToken token)
-        {
-            switch (option.UploadMode)
-            {
-                case UploadMode.AzureSdk:
-                {
-                    if (string.IsNullOrEmpty(option.AzBlobUrl)
-                        || string.IsNullOrEmpty(option.AzBlobContainer)
-                        || string.IsNullOrEmpty(option.AzBlobSaS))
-                    {
-                        logger.LogCritical("Configuration error for azure blob upload");
-                        return new NoneRemoteStorage();
-                    }
-                    var client = new BlobServiceClient(new Uri(option.AzBlobUrl + "/" + option.AzBlobContainer + "?" + option.AzBlobSaS));
-                    BlobContainerClient container = client.GetBlobContainerClient(option.AzBlobContainer);
-                    return new AzureRemoteStorage(container);
-                }
-                case UploadMode.GoogleCloudStorage:
-                {
-                    if (string.IsNullOrEmpty(option.GoogleStorageCredential)
-                        || string.IsNullOrEmpty(option.GoogleStorageBucketName))
-                    {
-                        logger.LogCritical("Configuration error for google storage upload");
-                        return new NoneRemoteStorage();
-                    }
-                    StorageClient storage = await GoogleAuthHelper.GetGoogleStorageClientAsync(option.GoogleStorageCredential, token);
-                    return new GoogleRemoteStorage(storage, option.GoogleStorageBucketName);
-                }
-                case UploadMode.S3:
-                    if (string.IsNullOrEmpty(option.S3AccessKeyId)
-                        || string.IsNullOrEmpty(option.S3Host)
-                        || string.IsNullOrEmpty(option.S3Region)
-                        || string.IsNullOrEmpty(option.S3BucketName)
-                        || string.IsNullOrEmpty(option.S3SecretAcessKey))
-                    {
-                        logger.LogCritical("Configuration error for S3 upload");
-                        return new NoneRemoteStorage();
-                    }
-
-                    var credentials = new BasicAWSCredentials(option.S3AccessKeyId, option.S3SecretAcessKey);
-                    var s3Config = new AmazonS3Config()
-                    {
-                        AuthenticationRegion = option.S3Region,
-                        ServiceURL = option.S3Host,
-                    };
-
-                    return new S3RemoteStorage(new AmazonS3Client(credentials, s3Config), option.S3BucketName);
-                case UploadMode.None:
-                case null:
-                    return new NoneRemoteStorage();
-                default:
-                    throw new ArgumentOutOfRangeException();
-            }
+        public static async Task<IEnumerable<IRemoteStorage>> GetRemoteStoragesAsync(this CompressOption option, ILogger logger, CancellationToken token)
+        {
+            var remoteStorages = new List<IRemoteStorage>();
+
+            option.UploadModes = option.UploadModes.Append(option.UploadMode ?? UploadMode.None).Distinct();
+
+            foreach(var mode in option.UploadModes)
+            {
+                switch (mode)
+                {
+                    case UploadMode.AzureSdk:
+                        {
+                            if (string.IsNullOrEmpty(option.AzBlobUrl)
+                                || string.IsNullOrEmpty(option.AzBlobContainer)
+                                || string.IsNullOrEmpty(option.AzBlobSaS))
+                            {
+                                logger.LogCritical("Configuration error for azure blob upload");
+                                continue;
+                            }
+                            var client = new BlobServiceClient(new Uri(option.AzBlobUrl + "/" + option.AzBlobContainer + "?" + option.AzBlobSaS));
+                            BlobContainerClient container = client.GetBlobContainerClient(option.AzBlobContainer);
+                            remoteStorages.Add(new AzureRemoteStorage(container));
+                            break;
+                        }
+                    case UploadMode.GoogleCloudStorage:
+                        {
+                            if (string.IsNullOrEmpty(option.GoogleStorageCredential)
+                                || string.IsNullOrEmpty(option.GoogleStorageBucketName))
+                            {
+                                logger.LogCritical("Configuration error for google storage upload");
+                                continue;
+                            }
+                            StorageClient storage = await GoogleAuthHelper.GetGoogleStorageClientAsync(option.GoogleStorageCredential, token);
+                            remoteStorages.Add(new GoogleRemoteStorage(storage, option.GoogleStorageBucketName));
+                            break;
+                        }
+                    case UploadMode.S3:
+                        if (string.IsNullOrEmpty(option.S3AccessKeyId)
+                            || string.IsNullOrEmpty(option.S3Host)
+                            || string.IsNullOrEmpty(option.S3Region)
+                            || string.IsNullOrEmpty(option.S3BucketName)
+                            || string.IsNullOrEmpty(option.S3SecretAcessKey))
+                        {
+                            logger.LogCritical("Configuration error for S3 upload");
+
+                        }
+
+                        var credentials = new BasicAWSCredentials(option.S3AccessKeyId, option.S3SecretAcessKey);
+                        var s3Config = new AmazonS3Config()
+                        {
+                            AuthenticationRegion = option.S3Region,
+                            ServiceURL = option.S3Host,
+                        };
+
+                        remoteStorages.Add(new S3RemoteStorage(new AmazonS3Client(credentials, s3Config), option.S3BucketName));
+                        break;
+                    case UploadMode.None:
+                        break;
+                    default:
+                        throw new ArgumentOutOfRangeException();
+                }
+            }
+            return remoteStorages;
         }
     }
 }
\ No newline at end of file
diff --git a/TCC.Lib/Storage/S3RemoteStorage.cs b/TCC.Lib/Storage/S3RemoteStorage.cs
index 2e0533f..b254953 100644
--- a/TCC.Lib/Storage/S3RemoteStorage.cs
+++ b/TCC.Lib/Storage/S3RemoteStorage.cs
@@ -44,5 +44,7 @@ await _s3Client.PutObjectAsync(new Amazon.S3.Model.PutObjectRequest() {
                 RemoteFilePath = targetPath
             };
         }
+
+        public UploadMode GetMode() => UploadMode.S3;
     }
 }
\ No newline at end of file
diff --git a/TCC.Lib/TarCompressCrypt.cs b/TCC.Lib/TarCompressCrypt.cs
index 91f9bd0..b7b004b 100644
--- a/TCC.Lib/TarCompressCrypt.cs
+++ b/TCC.Lib/TarCompressCrypt.cs
@@ -56,7 +56,7 @@ public async Task<OperationSummary> CompressAsync(CompressOption option)
             _logger.LogInformation("Starting compression job");
             var po = ParallelizeOption(option);
 
-            IRemoteStorage uploader = await option.GetRemoteStorageAsync(_logger, _cancellationTokenSource.Token);
+            IEnumerable<IRemoteStorage> uploaders = await option.GetRemoteStoragesAsync(_logger, _cancellationTokenSource.Token);
 
             var operationBlocks = await buffer
                 .AsAsyncStream(_cancellationTokenSource.Token)
@@ -81,90 +81,93 @@ public async Task<OperationSummary> CompressAsync(CompressOption option)
                     return opb;
                 }, po)
                 // Upload loop
-                .ParallelizeStreamAsync((block, token) => UploadBlockInternal(uploader, option, block, token), new ParallelizeOption { FailMode = Fail.Smart, MaxDegreeOfParallelism = option.AzThread ?? 1 })
+                .ParallelizeStreamAsync((block, token) => UploadBlockInternal(uploaders, option, block, token), new ParallelizeOption { FailMode = Fail.Smart, MaxDegreeOfParallelism = option.AzThread ?? 1 })
                 .AsReadOnlyCollectionAsync();
 
             sw.Stop();
             return new OperationSummary(operationBlocks, option.Threads, sw, option.SourceDirOrFile);
         }
 
-        private async Task<OperationCompressionBlock> UploadBlockInternal(IRemoteStorage uploader, CompressOption option, OperationCompressionBlock block, CancellationToken token)
-        {
-            if (uploader is NoneRemoteStorage)
-            {
-                return block;
+        private async Task<OperationCompressionBlock> UploadBlockInternal(IEnumerable<IRemoteStorage> uploaders, CompressOption option, OperationCompressionBlock block, CancellationToken token)
+        {
+            foreach(var uploader in uploaders)
+            {
+                if (uploader is NoneRemoteStorage)
+                {
+                    continue;
+                }
+
+                int count = Interlocked.Increment(ref _uploadCounter);
+                string progress = $"{count}/{_totalCounter}";
+
+                var file = block.CompressionBlock.DestinationArchiveFileInfo;
+                var name = file.Name;
+                RetryContext ctx = null;
+                while (true)
+                {
+                    bool hasError;
+                    try
+                    {
+                        var sw = Stopwatch.StartNew();
+
+                        var result = await uploader.UploadAsync(file, block.CompressionBlock.FolderProvider.RootFolder, token);
+                        hasError = !result.IsSuccess;
+
+                        sw.Stop();
+                        double speed = file.Length / sw.Elapsed.TotalSeconds;
+
+                        block.BlockResult.StepResults.Add(new StepResult
+                        {
+                            Type = StepType.Upload,
+                            UploadMode = uploader.GetMode(),
+                            Errors = result.IsSuccess ? null : result.ErrorMessage,
+                            Infos = result.IsSuccess ? result.ErrorMessage : null,
+                            Duration = sw.Elapsed,
+                            ArchiveFileSize = file.Length,
+                        });
+
+
+                        if (!hasError)
+                        {
+                            _logger.LogInformation($"{progress} Uploaded \"{file.Name}\" in {sw.Elapsed.HumanizedTimeSpan()} at {speed.HumanizedBandwidth()} ");
+                        }
+                        else
+                        {
+                            if (ctx == null && option.RetryPeriodInSeconds.HasValue)
+                            {
+                                ctx = new RetryContext(option.RetryPeriodInSeconds.Value);
+                            }
+                            _logger.LogError($"{progress} Uploaded {file.Name} with errors. {result.ErrorMessage}");
+                        }
+                    }
+                    catch (Exception e)
+                    {
+                        hasError = true;
+                        if (ctx == null && option.RetryPeriodInSeconds.HasValue)
+                        {
+                            ctx = new RetryContext(option.RetryPeriodInSeconds.Value);
+                        }
+                        _logger.LogCritical(e, $"{progress} Error uploading {name}");
+                    }
+
+                    if (hasError)
+                    {
+                        if (ctx != null && await ctx.WaitForNextRetry())
+                        {
+                            _logger.LogWarning($"{progress} Retrying uploading {name}, attempt #{ctx.Retries}");
+                        }
+                        else
+                        {
+                            break;
+                        }
+                    }
+                    else
+                    {
+                        break;
+                    }
+                }
             }
-
-            int count = Interlocked.Increment(ref _uploadCounter);
-            string progress = $"{count}/{_totalCounter}";
-
-            var file = block.CompressionBlock.DestinationArchiveFileInfo;
-            var name = file.Name;
-            RetryContext ctx = null;
-            while (true)
-            {
-                bool hasError;
-                try
-                {
-                    var sw = Stopwatch.StartNew();
-
-                    var result = await uploader.UploadAsync(file, block.CompressionBlock.FolderProvider.RootFolder, token);
-                    hasError = !result.IsSuccess;
-
-                    sw.Stop();
-                    double speed = file.Length / sw.Elapsed.TotalSeconds;
-
-                    block.BlockResult.StepResults.Add(new StepResult
-                    {
-                        Type = StepType.Upload,
-                        Errors = result.IsSuccess ? null : result.ErrorMessage,
-                        Infos = result.IsSuccess ? result.ErrorMessage : null,
-                        Duration = sw.Elapsed,
-                        ArchiveFileSize = file.Length,
-                    });
-
-
-                    if (!hasError)
-                    {
-                        _logger.LogInformation($"{progress} Uploaded \"{file.Name}\" in {sw.Elapsed.HumanizedTimeSpan()} at {speed.HumanizedBandwidth()} ");
-                    }
-                    else
-                    {
-                        if (ctx == null && option.RetryPeriodInSeconds.HasValue)
-                        {
-                            ctx = new RetryContext(option.RetryPeriodInSeconds.Value);
-                        }
-                        _logger.LogError($"{progress} Uploaded {file.Name} with errors. {result.ErrorMessage}");
-                    }
-                }
-                catch (Exception e)
-                {
-                    hasError = true;
-                    if (ctx == null && option.RetryPeriodInSeconds.HasValue)
-                    {
-                        ctx = new RetryContext(option.RetryPeriodInSeconds.Value);
-                    }
-                    _logger.LogCritical(e, $"{progress} Error uploading {name}");
-                }
-
-                if (hasError)
-                {
-                    if (ctx != null && await ctx.WaitForNextRetry())
-                    {
-                        _logger.LogWarning($"{progress} Retrying uploading {name}, attempt #{ctx.Retries}");
-                    }
-                    else
-                    {
-                        break;
-                    }
-                }
-                else
-                {
-                    break;
-                }
-            }
-
-            return block;
+            return block;
         }
 
         private async Task CleanupOldFiles(OperationCompressionBlock opb)
diff --git a/TCC.Tests/Upload/BlobStorageTests.cs b/TCC.Tests/Upload/BlobStorageTests.cs
index 7a71523..1e07d28 100644
--- a/TCC.Tests/Upload/BlobStorageTests.cs
+++ b/TCC.Tests/Upload/BlobStorageTests.cs
@@ -2,6 +2,7 @@
 using Newtonsoft.Json;
 using Newtonsoft.Json.Linq;
 using System;
+using System.Collections.Generic;
 using System.IO;
 using System.Linq;
 using System.Threading;
@@ -35,10 +36,10 @@ public async Task AzureUploadTest()
                 AzBlobContainer = GetEnvVar("AZ_CONTAINER"),
                 AzBlobSaS = GetEnvVar("AZ_SAS_TOKEN")
             };
-            opt.UploadMode = UploadMode.AzureSdk;
-            var uploader = await opt.GetRemoteStorageAsync(NullLogger.Instance, CancellationToken.None);
+            opt.UploadModes = new List<UploadMode>() { UploadMode.AzureSdk };
+            var uploader = await opt.GetRemoteStoragesAsync(NullLogger.Instance, CancellationToken.None);
 
-            var ok = await uploader.UploadAsync(data.Files.First(), new DirectoryInfo(toCompressFolder), CancellationToken.None);
+            var ok = await uploader.First().UploadAsync(data.Files.First(), new DirectoryInfo(toCompressFolder), CancellationToken.None);
 
             Assert.True(ok.IsSuccess);
         }
@@ -55,10 +56,10 @@ public async Task GoogleUploadTest()
                 GoogleStorageCredential = GetEnvVar("GoogleCredential")
             };
 
-            opt.UploadMode = UploadMode.GoogleCloudStorage;
-            var uploader = await opt.GetRemoteStorageAsync(NullLogger.Instance, CancellationToken.None);
+            opt.UploadModes = new List<UploadMode>() { UploadMode.GoogleCloudStorage };
+            var uploader = await opt.GetRemoteStoragesAsync(NullLogger.Instance, CancellationToken.None);
 
-            var ok = await uploader.UploadAsync(data.Files.First(), new DirectoryInfo(toCompressFolder), CancellationToken.None);
+            var ok = await uploader.First().UploadAsync(data.Files.First(), new DirectoryInfo(toCompressFolder), CancellationToken.None);
 
             Assert.True(ok.IsSuccess);
 
diff --git a/TCC/Parser/CompressCommand.cs b/TCC/Parser/CompressCommand.cs
index 9cbcc87..835ba8d 100644
--- a/TCC/Parser/CompressCommand.cs
+++ b/TCC/Parser/CompressCommand.cs
@@ -58,7 +58,8 @@ protected override IEnumerable<Option> CreateOptions()
             yield return new Option<int?>(new[] { "--cleanupTime" }, "Specify the time, in hours, after which the backup files are deleted. If no time is specified, then no backup files are deleted.");
 
             // Remote storage options
-            yield return new Option<UploadMode?>(new[] { "--uploadMode" }, "Upload mode");
+            yield return new Option<UploadMode?>(new[] { "--uploadMode" }, "Upload mode (retro compat with single upload options)");
+            yield return new Option<IEnumerable<UploadMode>>(new[] { "--uploadModes" }, "Upload modes");
             yield return new Option<string>(new[] { "--azBlobUrl" }, "Azure blob storage URL");
             yield return new Option<string>(new[] { "--azBlobContainer" }, "Azure blob storage container id");
             yield return new Option<string>(new[] { "--azBlobSaS" }, "Azure blob storage SaS token");

From f6d2eee579e82c9ba009384c06053904d933295a Mon Sep 17 00:00:00 2001
From: Lilian ARAGO <larago@lucca.fr>
Date: Tue, 14 Feb 2023 17:10:03 +0100
Subject: [PATCH 03/16] Fix progression display

---
 TCC.Lib/TarCompressCrypt.cs          | 16 ++++++++--------
 TCC.Tests/Upload/BlobStorageTests.cs |  2 +-
 2 files changed, 9 insertions(+), 9 deletions(-)

diff --git a/TCC.Lib/TarCompressCrypt.cs b/TCC.Lib/TarCompressCrypt.cs
index b7b004b..adb4794 100644
--- a/TCC.Lib/TarCompressCrypt.cs
+++ b/TCC.Lib/TarCompressCrypt.cs
@@ -90,16 +90,16 @@ public async Task<OperationSummary> CompressAsync(CompressOption option)
 
         private async Task<OperationCompressionBlock> UploadBlockInternal(IEnumerable<IRemoteStorage> uploaders, CompressOption option, OperationCompressionBlock block, CancellationToken token)
         {
-            foreach(var uploader in uploaders)
+            int count = Interlocked.Increment(ref _uploadCounter);
+            string progress = $"{count}/{_totalCounter}";
+
+            foreach (var uploader in uploaders)
             {
                 if (uploader is NoneRemoteStorage)
                 {
                     continue;
                 }
 
-                int count = Interlocked.Increment(ref _uploadCounter);
-                string progress = $"{count}/{_totalCounter}";
-
                 var file = block.CompressionBlock.DestinationArchiveFileInfo;
                 var name = file.Name;
                 RetryContext ctx = null;
@@ -129,7 +129,7 @@ private async Task<OperationCompressionBlock> UploadBlockInternal(IEnumerable<IR
 
                         if (!hasError)
                         {
-                            _logger.LogInformation($"{progress} Uploaded \"{file.Name}\" in {sw.Elapsed.HumanizedTimeSpan()} at {speed.HumanizedBandwidth()} ");
+                            _logger.LogInformation($"[{uploader.GetMode()}] {progress} Uploaded \"{file.Name}\" in {sw.Elapsed.HumanizedTimeSpan()} at {speed.HumanizedBandwidth()} ");
                         }
                         else
                         {
@@ -137,7 +137,7 @@ private async Task<OperationCompressionBlock> UploadBlockInternal(IEnumerable<IR
                             {
                                 ctx = new RetryContext(option.RetryPeriodInSeconds.Value);
                             }
-                            _logger.LogError($"{progress} Uploaded {file.Name} with errors. {result.ErrorMessage}");
+                            _logger.LogError($"[{uploader.GetMode()}] {progress} Uploaded {file.Name} with errors. {result.ErrorMessage}");
                         }
                     }
                     catch (Exception e)
@@ -147,14 +147,14 @@ private async Task<OperationCompressionBlock> UploadBlockInternal(IEnumerable<IR
                         {
                             ctx = new RetryContext(option.RetryPeriodInSeconds.Value);
                         }
-                        _logger.LogCritical(e, $"{progress} Error uploading {name}");
+                        _logger.LogCritical(e, $"[{uploader.GetMode()}] {progress} Error uploading {name}");
                     }
 
                     if (hasError)
                     {
                         if (ctx != null && await ctx.WaitForNextRetry())
                         {
-                            _logger.LogWarning($"{progress} Retrying uploading {name}, attempt #{ctx.Retries}");
+                            _logger.LogWarning($"[{uploader.GetMode()}] {progress} Retrying uploading {name}, attempt #{ctx.Retries}");
                         }
                         else
                         {
diff --git a/TCC.Tests/Upload/BlobStorageTests.cs b/TCC.Tests/Upload/BlobStorageTests.cs
index 1e07d28..00cd531 100644
--- a/TCC.Tests/Upload/BlobStorageTests.cs
+++ b/TCC.Tests/Upload/BlobStorageTests.cs
@@ -63,7 +63,7 @@ public async Task GoogleUploadTest()
 
             Assert.True(ok.IsSuccess);
 
-            var gs = uploader as GoogleRemoteStorage;
+            var gs = uploader.First() as GoogleRemoteStorage;
             
             await gs.Storage.DeleteObjectAsync(gs.BucketName, ok.RemoteFilePath);
         }

From feee3d2f4d5bf45586c9c4040c30de201adda799 Mon Sep 17 00:00:00 2001
From: Lilian ARAGO <larago@lucca.fr>
Date: Thu, 2 Mar 2023 15:56:28 +0100
Subject: [PATCH 04/16] Multipart upload

---
 TCC.Lib/Helpers/StringExtensions.cs     | 18 +++++-
 TCC.Lib/Options/CompressOption.cs       |  2 +
 TCC.Lib/Storage/RemoteStorageFactory.cs |  9 ++-
 TCC.Lib/Storage/S3RemoteStorage.cs      | 84 +++++++++++++++++++++++--
 TCC/Parser/CompressCommand.cs           |  2 +
 5 files changed, 106 insertions(+), 9 deletions(-)

diff --git a/TCC.Lib/Helpers/StringExtensions.cs b/TCC.Lib/Helpers/StringExtensions.cs
index 4f239fa..b246e56 100644
--- a/TCC.Lib/Helpers/StringExtensions.cs
+++ b/TCC.Lib/Helpers/StringExtensions.cs
@@ -1,7 +1,8 @@
 using System;
 using System.Globalization;
 using System.IO;
-
+using System.Linq;
+
 namespace TCC.Lib.Helpers
 {
     public static class StringExtensions
@@ -33,6 +34,21 @@ public static String HumanizeSize(this long size)
             int place = Convert.ToInt32(Math.Floor(Math.Log(bytes, 1024)));
             double num = Math.Round(bytes / Math.Pow(1024, place), 1);
             return (Math.Sign(size) * num).ToString(CultureInfo.InvariantCulture) + " " + suf[place];
+        }
+
+        public static long ParseSize(this string humanizedSize)
+        {
+            string[] suf = { "b", "ko", "mo", "go", "to", "po", "eo" };
+            var size = humanizedSize.Trim().ToLower();
+            var number = string.Join("", size.Where(char.IsDigit));
+            var unit = size.Substring(size.Length - 2);
+            var pow = Array.IndexOf(suf, unit);
+
+            return pow switch
+            {
+                -1 => long.Parse(number),
+                _ => long.Parse(number) * (long)Math.Pow(1024L, pow)
+            };
         }
 
         public static string HumanizedTimeSpan(this TimeSpan t, int parts = 2)
diff --git a/TCC.Lib/Options/CompressOption.cs b/TCC.Lib/Options/CompressOption.cs
index 6d84a33..e003fe5 100644
--- a/TCC.Lib/Options/CompressOption.cs
+++ b/TCC.Lib/Options/CompressOption.cs
@@ -28,6 +28,8 @@ public class CompressOption : TccOption
         public string S3Host { get; set; }
         public string S3BucketName { get; set; }
         public string S3Region { get; set; }
+        public string S3MultipartThreshold { get; set; }
+        public string S3MultipartSize { get; set; }
         public IEnumerable<UploadMode> UploadModes { get; set; } = Enumerable.Empty<UploadMode>();
         public UploadMode? UploadMode { get; set; }
     }
diff --git a/TCC.Lib/Storage/RemoteStorageFactory.cs b/TCC.Lib/Storage/RemoteStorageFactory.cs
index fe85b35..6bd8c3e 100644
--- a/TCC.Lib/Storage/RemoteStorageFactory.cs
+++ b/TCC.Lib/Storage/RemoteStorageFactory.cs
@@ -19,7 +19,8 @@
 namespace TCC.Lib.Storage
 {
     public static class RemoteStorageFactory
-    {
+    {
+        private static long ParseSizeInternal(string size) => string.IsNullOrWhiteSpace(size) ? 0 : size.ParseSize();
         public static async Task<IEnumerable<IRemoteStorage>> GetRemoteStoragesAsync(this CompressOption option, ILogger logger, CancellationToken token)
         {
             var remoteStorages = new List<IRemoteStorage>();
@@ -74,7 +75,11 @@ public static async Task<IEnumerable<IRemoteStorage>> GetRemoteStoragesAsync(thi
                             ServiceURL = option.S3Host,
                         };
 
-                        remoteStorages.Add(new S3RemoteStorage(new AmazonS3Client(credentials, s3Config), option.S3BucketName));
+                        remoteStorages.Add(new S3RemoteStorage(
+                            new AmazonS3Client(credentials, s3Config),
+                            option.S3BucketName,
+                            ParseSizeInternal(option.S3MultipartThreshold),
+                            (int) ParseSizeInternal(option.S3MultipartSize)));
                         break;
                     case UploadMode.None:
                         break;
diff --git a/TCC.Lib/Storage/S3RemoteStorage.cs b/TCC.Lib/Storage/S3RemoteStorage.cs
index b254953..52a645a 100644
--- a/TCC.Lib/Storage/S3RemoteStorage.cs
+++ b/TCC.Lib/Storage/S3RemoteStorage.cs
@@ -1,10 +1,16 @@
 using Amazon;
 using Amazon.Runtime;
 using Amazon.S3;
+using Amazon.S3.Model;
 using System;
+using System.Collections.Generic;
 using System.IO;
+using System.Runtime.CompilerServices;
 using System.Threading;
 using System.Threading.Tasks;
+using TCC.Lib.AsyncStreams;
+using TCC.Lib.Benchmark;
+using TCC.Lib.Helpers;
 
 namespace TCC.Lib.Storage
 {
@@ -12,22 +18,33 @@ public class S3RemoteStorage : IRemoteStorage
     {
         internal string BucketName { get; }
         private readonly AmazonS3Client _s3Client;
+        private readonly long _multipartTreshold;
+        private readonly int _partSize;
 
-        public S3RemoteStorage(AmazonS3Client s3Client, string bucketName)
+        public S3RemoteStorage(AmazonS3Client s3Client, string bucketName, long multipartThreshold = 0, int partSize = 0)
         {
             BucketName = bucketName;
             _s3Client = s3Client;
+            _multipartTreshold = multipartThreshold;
+            _partSize = partSize;
         }
 
         public async Task<UploadResponse> UploadAsync(string targetPath, Stream data, CancellationToken token)
         {
             try
             {
-                await _s3Client.PutObjectAsync(new Amazon.S3.Model.PutObjectRequest() {
-                    BucketName = BucketName,
-                    Key = targetPath,
-                    InputStream = data,
-                }, token);
+                if (_multipartTreshold != 0  && data.Length > _multipartTreshold)
+                {
+                    await UploadStreamToMultiparts(targetPath, data, token);
+                } else
+                {
+                    await _s3Client.PutObjectAsync(new ()
+                    {
+                        BucketName = BucketName,
+                        Key = targetPath,
+                        InputStream = data,
+                    }, token);
+                }
             }
             catch (Exception e)
             {
@@ -45,6 +62,61 @@ await _s3Client.PutObjectAsync(new Amazon.S3.Model.PutObjectRequest() {
             };
         }
 
+        private async Task UploadStreamToMultiparts(string targetPath, Stream data, CancellationToken token)
+        {
+            
+            var multipartUpload = await _s3Client.InitiateMultipartUploadAsync(new ()
+            {
+                BucketName = BucketName,
+                Key = targetPath,
+            }, token);
+            var partsETags = new List<PartETag>();
+            var partNumber = 1;
+
+            Console.WriteLine(data.Length.HumanizeSize());
+
+            await foreach (var chunk in ChunkStreamAsync(data, token))
+            {
+                chunk.Position = 0;
+                var partUpload = await _s3Client.UploadPartAsync(new()
+                {
+                    BucketName = BucketName,
+                    PartNumber = partNumber++,
+                    Key = targetPath,
+                    UploadId = multipartUpload.UploadId,
+                    InputStream = chunk,
+                }, token);
+
+                partsETags.Add(new()
+                {
+                    ETag = partUpload.ETag,
+                    PartNumber = partUpload.PartNumber,
+                });
+            }
+            var response = await _s3Client.CompleteMultipartUploadAsync(new()
+            {
+                BucketName = BucketName,
+                Key = targetPath,
+                UploadId = multipartUpload.UploadId,
+                PartETags = partsETags
+            }, token);
+        }
+        private async IAsyncEnumerable<Stream> ChunkStreamAsync(Stream data, [EnumeratorCancellation] CancellationToken token)
+        {
+            var buffer = new byte[_partSize];
+            int readBytes;
+
+            do
+            {
+                readBytes = await data.ReadAsync(buffer, 0, _partSize, token);
+
+                var notTheSame = new MemoryStream(readBytes);
+                notTheSame.Write(buffer, 0, readBytes);
+                
+                yield return notTheSame;
+            } while (readBytes >= _partSize);
+        }
+
         public UploadMode GetMode() => UploadMode.S3;
     }
 }
\ No newline at end of file
diff --git a/TCC/Parser/CompressCommand.cs b/TCC/Parser/CompressCommand.cs
index 835ba8d..159add9 100644
--- a/TCC/Parser/CompressCommand.cs
+++ b/TCC/Parser/CompressCommand.cs
@@ -71,6 +71,8 @@ protected override IEnumerable<Option> CreateOptions()
             yield return new Option<string>(new[] { "--s3SecretAcessKey" }, "S3 Access Key Secret");
             yield return new Option<string>(new[] { "--s3Host" }, "S3 Host");
             yield return new Option<string>(new[] { "--s3BucketName" }, "S3 destination bucket");
+            yield return new Option<string>(new[] { "--s3MultipartThreshold" }, "Threshold from which sending a file is done with multiparts");
+            yield return new Option<string>(new[] { "--s3MultipartSize" }, "If sending a file in multipart, the multipart size");
         }
         protected override Task RunAsync(ITccController controller, CompressOptionBinding option) => controller.CompressAsync(option);
     }

From 0fbcf58a34c201e02e2d19e4c3096a6210789070 Mon Sep 17 00:00:00 2001
From: Lilian ARAGO <larago@lucca.fr>
Date: Thu, 2 Mar 2023 16:07:47 +0100
Subject: [PATCH 05/16] Fixed sonarcloud code smells

---
 TCC.Lib/Helpers/StringExtensions.cs | 6 +++---
 TCC.Lib/Storage/S3RemoteStorage.cs  | 9 +--------
 2 files changed, 4 insertions(+), 11 deletions(-)

diff --git a/TCC.Lib/Helpers/StringExtensions.cs b/TCC.Lib/Helpers/StringExtensions.cs
index b246e56..98e09be 100644
--- a/TCC.Lib/Helpers/StringExtensions.cs
+++ b/TCC.Lib/Helpers/StringExtensions.cs
@@ -39,15 +39,15 @@ public static String HumanizeSize(this long size)
         public static long ParseSize(this string humanizedSize)
         {
             string[] suf = { "b", "ko", "mo", "go", "to", "po", "eo" };
-            var size = humanizedSize.Trim().ToLower();
+            var size = humanizedSize.Trim().ToLower(CultureInfo.InvariantCulture);
             var number = string.Join("", size.Where(char.IsDigit));
             var unit = size.Substring(size.Length - 2);
             var pow = Array.IndexOf(suf, unit);
 
             return pow switch
             {
-                -1 => long.Parse(number),
-                _ => long.Parse(number) * (long)Math.Pow(1024L, pow)
+                -1 => long.Parse(number, CultureInfo.InvariantCulture),
+                _ => long.Parse(number, CultureInfo.InvariantCulture) * (long)Math.Pow(1024L, pow)
             };
         }
 
diff --git a/TCC.Lib/Storage/S3RemoteStorage.cs b/TCC.Lib/Storage/S3RemoteStorage.cs
index 52a645a..b0dd960 100644
--- a/TCC.Lib/Storage/S3RemoteStorage.cs
+++ b/TCC.Lib/Storage/S3RemoteStorage.cs
@@ -1,6 +1,4 @@
-using Amazon;
-using Amazon.Runtime;
-using Amazon.S3;
+using Amazon.S3;
 using Amazon.S3.Model;
 using System;
 using System.Collections.Generic;
@@ -8,9 +6,6 @@
 using System.Runtime.CompilerServices;
 using System.Threading;
 using System.Threading.Tasks;
-using TCC.Lib.AsyncStreams;
-using TCC.Lib.Benchmark;
-using TCC.Lib.Helpers;
 
 namespace TCC.Lib.Storage
 {
@@ -73,8 +68,6 @@ private async Task UploadStreamToMultiparts(string targetPath, Stream data, Canc
             var partsETags = new List<PartETag>();
             var partNumber = 1;
 
-            Console.WriteLine(data.Length.HumanizeSize());
-
             await foreach (var chunk in ChunkStreamAsync(data, token))
             {
                 chunk.Position = 0;

From 4e5a84da97001faf69bf8b93d59e4d91e3bf8a4b Mon Sep 17 00:00:00 2001
From: Lilian ARAGO <larago@lucca.fr>
Date: Thu, 2 Mar 2023 16:34:51 +0100
Subject: [PATCH 06/16] Fixed some code smells

---
 TCC.Lib/Storage/AzureRemoteStorage.cs   |  2 +-
 TCC.Lib/Storage/GoogleRemoteStorage.cs  |  2 +-
 TCC.Lib/Storage/IRemoteStorage.cs       |  2 +-
 TCC.Lib/Storage/NoneRemoteStorage.cs    |  2 +-
 TCC.Lib/Storage/RemoteStorageFactory.cs |  5 -----
 TCC.Lib/Storage/S3RemoteStorage.cs      | 10 +++++-----
 TCC.Lib/TarCompressCrypt.cs             | 10 +++++-----
 7 files changed, 14 insertions(+), 19 deletions(-)

diff --git a/TCC.Lib/Storage/AzureRemoteStorage.cs b/TCC.Lib/Storage/AzureRemoteStorage.cs
index 3e61f66..943cd45 100644
--- a/TCC.Lib/Storage/AzureRemoteStorage.cs
+++ b/TCC.Lib/Storage/AzureRemoteStorage.cs
@@ -26,7 +26,7 @@ public async Task<UploadResponse> UploadAsync(string targetPath, Stream data, Ca
             };
         }
 
-        public UploadMode GetMode() => UploadMode.AzureSdk;
+        public UploadMode Mode => UploadMode.AzureSdk;
 
     }
 }
\ No newline at end of file
diff --git a/TCC.Lib/Storage/GoogleRemoteStorage.cs b/TCC.Lib/Storage/GoogleRemoteStorage.cs
index c5c2d84..18045eb 100644
--- a/TCC.Lib/Storage/GoogleRemoteStorage.cs
+++ b/TCC.Lib/Storage/GoogleRemoteStorage.cs
@@ -40,6 +40,6 @@ public async Task<UploadResponse> UploadAsync(string targetPath, Stream data, Ca
             };
         }
 
-        public UploadMode GetMode() => UploadMode.GoogleCloudStorage;
+        public UploadMode Mode => UploadMode.GoogleCloudStorage;
     }
 }
\ No newline at end of file
diff --git a/TCC.Lib/Storage/IRemoteStorage.cs b/TCC.Lib/Storage/IRemoteStorage.cs
index 71b7765..7acf3d1 100644
--- a/TCC.Lib/Storage/IRemoteStorage.cs
+++ b/TCC.Lib/Storage/IRemoteStorage.cs
@@ -16,6 +16,6 @@ public async Task<UploadResponse> UploadAsync(FileInfo file, DirectoryInfo rootF
             return await UploadAsync(targetPath, uploadFileStream, token);
         }
 
-        public UploadMode GetMode();
+        public UploadMode Mode { get; }
     }
 }
\ No newline at end of file
diff --git a/TCC.Lib/Storage/NoneRemoteStorage.cs b/TCC.Lib/Storage/NoneRemoteStorage.cs
index bbb3160..6ee437e 100644
--- a/TCC.Lib/Storage/NoneRemoteStorage.cs
+++ b/TCC.Lib/Storage/NoneRemoteStorage.cs
@@ -11,6 +11,6 @@ public Task<UploadResponse> UploadAsync(string targetPath, Stream data, Cancella
             return Task.FromResult(new UploadResponse { IsSuccess = true, RemoteFilePath = targetPath });
         }
 
-        public UploadMode GetMode() => UploadMode.None;
+        public UploadMode Mode => UploadMode.None;
     }
 }
diff --git a/TCC.Lib/Storage/RemoteStorageFactory.cs b/TCC.Lib/Storage/RemoteStorageFactory.cs
index 6bd8c3e..324429e 100644
--- a/TCC.Lib/Storage/RemoteStorageFactory.cs
+++ b/TCC.Lib/Storage/RemoteStorageFactory.cs
@@ -1,16 +1,11 @@
 using Amazon.Runtime;
 using Amazon.S3;
 using Azure.Storage.Blobs;
-using Google.Apis.Auth.OAuth2;
 using Google.Cloud.Storage.V1;
 using Microsoft.Extensions.Logging;
 using System;
-using System.Collections;
 using System.Collections.Generic;
-using System.ComponentModel;
-using System.IO;
 using System.Linq;
-using System.Text;
 using System.Threading;
 using System.Threading.Tasks;
 using TCC.Lib.Helpers;
diff --git a/TCC.Lib/Storage/S3RemoteStorage.cs b/TCC.Lib/Storage/S3RemoteStorage.cs
index b0dd960..b377c51 100644
--- a/TCC.Lib/Storage/S3RemoteStorage.cs
+++ b/TCC.Lib/Storage/S3RemoteStorage.cs
@@ -30,7 +30,7 @@ public async Task<UploadResponse> UploadAsync(string targetPath, Stream data, Ca
             {
                 if (_multipartTreshold != 0  && data.Length > _multipartTreshold)
                 {
-                    await UploadStreamToMultiparts(targetPath, data, token);
+                    await UploadStreamToMultipartsAsync(targetPath, data, token);
                 } else
                 {
                     await _s3Client.PutObjectAsync(new ()
@@ -57,7 +57,7 @@ await _s3Client.PutObjectAsync(new ()
             };
         }
 
-        private async Task UploadStreamToMultiparts(string targetPath, Stream data, CancellationToken token)
+        private async Task UploadStreamToMultipartsAsync(string targetPath, Stream data, CancellationToken token)
         {
             
             var multipartUpload = await _s3Client.InitiateMultipartUploadAsync(new ()
@@ -86,7 +86,7 @@ private async Task UploadStreamToMultiparts(string targetPath, Stream data, Canc
                     PartNumber = partUpload.PartNumber,
                 });
             }
-            var response = await _s3Client.CompleteMultipartUploadAsync(new()
+            await _s3Client.CompleteMultipartUploadAsync(new()
             {
                 BucketName = BucketName,
                 Key = targetPath,
@@ -101,7 +101,7 @@ private async IAsyncEnumerable<Stream> ChunkStreamAsync(Stream data, [Enumerator
 
             do
             {
-                readBytes = await data.ReadAsync(buffer, 0, _partSize, token);
+                readBytes = await data.ReadAsync(buffer.AsMemory(0, _partSize), token);
 
                 var notTheSame = new MemoryStream(readBytes);
                 notTheSame.Write(buffer, 0, readBytes);
@@ -110,6 +110,6 @@ private async IAsyncEnumerable<Stream> ChunkStreamAsync(Stream data, [Enumerator
             } while (readBytes >= _partSize);
         }
 
-        public UploadMode GetMode() => UploadMode.S3;
+        public UploadMode Mode => UploadMode.S3;
     }
 }
\ No newline at end of file
diff --git a/TCC.Lib/TarCompressCrypt.cs b/TCC.Lib/TarCompressCrypt.cs
index adb4794..4b112f3 100644
--- a/TCC.Lib/TarCompressCrypt.cs
+++ b/TCC.Lib/TarCompressCrypt.cs
@@ -119,7 +119,7 @@ private async Task<OperationCompressionBlock> UploadBlockInternal(IEnumerable<IR
                         block.BlockResult.StepResults.Add(new StepResult
                         {
                             Type = StepType.Upload,
-                            UploadMode = uploader.GetMode(),
+                            UploadMode = uploader.Mode,
                             Errors = result.IsSuccess ? null : result.ErrorMessage,
                             Infos = result.IsSuccess ? result.ErrorMessage : null,
                             Duration = sw.Elapsed,
@@ -129,7 +129,7 @@ private async Task<OperationCompressionBlock> UploadBlockInternal(IEnumerable<IR
 
                         if (!hasError)
                         {
-                            _logger.LogInformation($"[{uploader.GetMode()}] {progress} Uploaded \"{file.Name}\" in {sw.Elapsed.HumanizedTimeSpan()} at {speed.HumanizedBandwidth()} ");
+                            _logger.LogInformation($"[{uploader.Mode}] {progress} Uploaded \"{file.Name}\" in {sw.Elapsed.HumanizedTimeSpan()} at {speed.HumanizedBandwidth()} ");
                         }
                         else
                         {
@@ -137,7 +137,7 @@ private async Task<OperationCompressionBlock> UploadBlockInternal(IEnumerable<IR
                             {
                                 ctx = new RetryContext(option.RetryPeriodInSeconds.Value);
                             }
-                            _logger.LogError($"[{uploader.GetMode()}] {progress} Uploaded {file.Name} with errors. {result.ErrorMessage}");
+                            _logger.LogError($"[{uploader.Mode}] {progress} Uploaded {file.Name} with errors. {result.ErrorMessage}");
                         }
                     }
                     catch (Exception e)
@@ -147,14 +147,14 @@ private async Task<OperationCompressionBlock> UploadBlockInternal(IEnumerable<IR
                         {
                             ctx = new RetryContext(option.RetryPeriodInSeconds.Value);
                         }
-                        _logger.LogCritical(e, $"[{uploader.GetMode()}] {progress} Error uploading {name}");
+                        _logger.LogCritical(e, $"[{uploader.Mode}] {progress} Error uploading {name}");
                     }
 
                     if (hasError)
                     {
                         if (ctx != null && await ctx.WaitForNextRetry())
                         {
-                            _logger.LogWarning($"[{uploader.GetMode()}] {progress} Retrying uploading {name}, attempt #{ctx.Retries}");
+                            _logger.LogWarning($"[{uploader.Mode}] {progress} Retrying uploading {name}, attempt #{ctx.Retries}");
                         }
                         else
                         {

From ecc3b685bde407aa1a30399f62473d615fb52973 Mon Sep 17 00:00:00 2001
From: Lilian ARAGO <larago@lucca.fr>
Date: Thu, 2 Mar 2023 16:48:42 +0100
Subject: [PATCH 07/16] Fix code smells

---
 TCC.Lib/Storage/RemoteStorageFactory.cs | 2 +-
 TCC.Lib/TarCompressCrypt.cs             | 8 ++++----
 2 files changed, 5 insertions(+), 5 deletions(-)

diff --git a/TCC.Lib/Storage/RemoteStorageFactory.cs b/TCC.Lib/Storage/RemoteStorageFactory.cs
index 324429e..332a104 100644
--- a/TCC.Lib/Storage/RemoteStorageFactory.cs
+++ b/TCC.Lib/Storage/RemoteStorageFactory.cs
@@ -64,7 +64,7 @@ public static async Task<IEnumerable<IRemoteStorage>> GetRemoteStoragesAsync(thi
                         }
 
                         var credentials = new BasicAWSCredentials(option.S3AccessKeyId, option.S3SecretAcessKey);
-                        var s3Config = new AmazonS3Config()
+                        var s3Config = new AmazonS3Config
                         {
                             AuthenticationRegion = option.S3Region,
                             ServiceURL = option.S3Host,
diff --git a/TCC.Lib/TarCompressCrypt.cs b/TCC.Lib/TarCompressCrypt.cs
index 4b112f3..675b78e 100644
--- a/TCC.Lib/TarCompressCrypt.cs
+++ b/TCC.Lib/TarCompressCrypt.cs
@@ -129,7 +129,7 @@ private async Task<OperationCompressionBlock> UploadBlockInternal(IEnumerable<IR
 
                         if (!hasError)
                         {
-                            _logger.LogInformation($"[{uploader.Mode}] {progress} Uploaded \"{file.Name}\" in {sw.Elapsed.HumanizedTimeSpan()} at {speed.HumanizedBandwidth()} ");
+                            _logger.LogInformation("[{mode}] {progress} Uploaded \"{filename}\" in {duration} at {speed} ", uploader.Mode, progress, file.Name, sw.Elapsed.HumanizedTimeSpan(), speed.HumanizedBandwidth());
                         }
                         else
                         {
@@ -137,7 +137,7 @@ private async Task<OperationCompressionBlock> UploadBlockInternal(IEnumerable<IR
                             {
                                 ctx = new RetryContext(option.RetryPeriodInSeconds.Value);
                             }
-                            _logger.LogError($"[{uploader.Mode}] {progress} Uploaded {file.Name} with errors. {result.ErrorMessage}");
+                            _logger.LogError("[{mode}] {progress} Uploaded {filename} with errors. {errorMessage}", uploader.Mode, progress, file.Name, result.ErrorMessage);
                         }
                     }
                     catch (Exception e)
@@ -147,14 +147,14 @@ private async Task<OperationCompressionBlock> UploadBlockInternal(IEnumerable<IR
                         {
                             ctx = new RetryContext(option.RetryPeriodInSeconds.Value);
                         }
-                        _logger.LogCritical(e, $"[{uploader.Mode}] {progress} Error uploading {name}");
+                        _logger.LogCritical(e, "[{mode}] {progress} Error uploading {name}", uploader.Mode, progress, name);
                     }
 
                     if (hasError)
                     {
                         if (ctx != null && await ctx.WaitForNextRetry())
                         {
-                            _logger.LogWarning($"[{uploader.Mode}] {progress} Retrying uploading {name}, attempt #{ctx.Retries}");
+                            _logger.LogWarning("[{mode}] {progress} Retrying uploading {name}, attempt #{attempt}", uploader.Mode, progress, name, ctx.Retries);
                         }
                         else
                         {

From 848d7bdab17d49e0f21ea8572d2f0eea32f688ae Mon Sep 17 00:00:00 2001
From: Lilian ARAGO <larago@lucca.fr>
Date: Thu, 2 Mar 2023 17:02:04 +0100
Subject: [PATCH 08/16] Fixed code smell

---
 TCC.Lib/TarCompressCrypt.cs | 102 +++++++++++++++++++-----------------
 1 file changed, 53 insertions(+), 49 deletions(-)

diff --git a/TCC.Lib/TarCompressCrypt.cs b/TCC.Lib/TarCompressCrypt.cs
index 675b78e..34bf274 100644
--- a/TCC.Lib/TarCompressCrypt.cs
+++ b/TCC.Lib/TarCompressCrypt.cs
@@ -99,77 +99,81 @@ private async Task<OperationCompressionBlock> UploadBlockInternal(IEnumerable<IR
                 {
                     continue;
                 }
+                await UploadBlockToSingleRemoteStorageAsync(option, block, progress, uploader, token);
+            }
+            return block;
+        }
 
-                var file = block.CompressionBlock.DestinationArchiveFileInfo;
-                var name = file.Name;
-                RetryContext ctx = null;
-                while (true)
+        private async Task UploadBlockToSingleRemoteStorageAsync(CompressOption option, OperationCompressionBlock block, string progress, IRemoteStorage uploader, CancellationToken token)
+        {
+            var file = block.CompressionBlock.DestinationArchiveFileInfo;
+            var name = file.Name;
+            RetryContext ctx = null;
+            while (true)
+            {
+                bool hasError;
+                try
                 {
-                    bool hasError;
-                    try
-                    {
-                        var sw = Stopwatch.StartNew();
+                    var sw = Stopwatch.StartNew();
 
-                        var result = await uploader.UploadAsync(file, block.CompressionBlock.FolderProvider.RootFolder, token);
-                        hasError = !result.IsSuccess;
+                    var result = await uploader.UploadAsync(file, block.CompressionBlock.FolderProvider.RootFolder, token);
+                    hasError = !result.IsSuccess;
 
-                        sw.Stop();
-                        double speed = file.Length / sw.Elapsed.TotalSeconds;
+                    sw.Stop();
+                    double speed = file.Length / sw.Elapsed.TotalSeconds;
 
-                        block.BlockResult.StepResults.Add(new StepResult
-                        {
-                            Type = StepType.Upload,
-                            UploadMode = uploader.Mode,
-                            Errors = result.IsSuccess ? null : result.ErrorMessage,
-                            Infos = result.IsSuccess ? result.ErrorMessage : null,
-                            Duration = sw.Elapsed,
-                            ArchiveFileSize = file.Length,
-                        });
+                    block.BlockResult.StepResults.Add(new StepResult
+                    {
+                        Type = StepType.Upload,
+                        UploadMode = uploader.Mode,
+                        Errors = result.IsSuccess ? null : result.ErrorMessage,
+                        Infos = result.IsSuccess ? result.ErrorMessage : null,
+                        Duration = sw.Elapsed,
+                        ArchiveFileSize = file.Length,
+                    });
 
 
-                        if (!hasError)
-                        {
-                            _logger.LogInformation("[{mode}] {progress} Uploaded \"{filename}\" in {duration} at {speed} ", uploader.Mode, progress, file.Name, sw.Elapsed.HumanizedTimeSpan(), speed.HumanizedBandwidth());
-                        }
-                        else
-                        {
-                            if (ctx == null && option.RetryPeriodInSeconds.HasValue)
-                            {
-                                ctx = new RetryContext(option.RetryPeriodInSeconds.Value);
-                            }
-                            _logger.LogError("[{mode}] {progress} Uploaded {filename} with errors. {errorMessage}", uploader.Mode, progress, file.Name, result.ErrorMessage);
-                        }
+                    if (!hasError)
+                    {
+                        _logger.LogInformation("[{mode}] {progress} Uploaded \"{filename}\" in {duration} at {speed} ", uploader.Mode, progress, file.Name, sw.Elapsed.HumanizedTimeSpan(), speed.HumanizedBandwidth());
                     }
-                    catch (Exception e)
+                    else
                     {
-                        hasError = true;
                         if (ctx == null && option.RetryPeriodInSeconds.HasValue)
                         {
                             ctx = new RetryContext(option.RetryPeriodInSeconds.Value);
                         }
-                        _logger.LogCritical(e, "[{mode}] {progress} Error uploading {name}", uploader.Mode, progress, name);
+                        _logger.LogError("[{mode}] {progress} Uploaded {filename} with errors. {errorMessage}", uploader.Mode, progress, file.Name, result.ErrorMessage);
+                    }
+                }
+                catch (Exception e)
+                {
+                    hasError = true;
+                    if (ctx == null && option.RetryPeriodInSeconds.HasValue)
+                    {
+                        ctx = new RetryContext(option.RetryPeriodInSeconds.Value);
                     }
+                    _logger.LogCritical(e, "[{mode}] {progress} Error uploading {name}", uploader.Mode, progress, name);
+                }
 
-                    if (hasError)
+                if (hasError)
+                {
+                    if (ctx != null && await ctx.WaitForNextRetry())
                     {
-                        if (ctx != null && await ctx.WaitForNextRetry())
-                        {
-                            _logger.LogWarning("[{mode}] {progress} Retrying uploading {name}, attempt #{attempt}", uploader.Mode, progress, name, ctx.Retries);
-                        }
-                        else
-                        {
-                            break;
-                        }
+                        _logger.LogWarning("[{mode}] {progress} Retrying uploading {name}, attempt #{attempt}", uploader.Mode, progress, name, ctx.Retries);
                     }
                     else
                     {
                         break;
                     }
                 }
-            }
-            return block;
-        }
-
+                else
+                {
+                    break;
+                }
+            }
+        }
+
         private async Task CleanupOldFiles(OperationCompressionBlock opb)
         {
             if (opb.CompressionBlock.FullsToDelete != null)

From eb88fb6205d4841480985a77bf6502cc1d791d79 Mon Sep 17 00:00:00 2001
From: Lilian ARAGO <larago@lucca.fr>
Date: Mon, 6 Mar 2023 11:28:05 +0100
Subject: [PATCH 09/16] string.ParseSize return -1 in case of error

---
 TCC.Lib/Helpers/StringExtensions.cs | 1 +
 1 file changed, 1 insertion(+)

diff --git a/TCC.Lib/Helpers/StringExtensions.cs b/TCC.Lib/Helpers/StringExtensions.cs
index 98e09be..24f42c6 100644
--- a/TCC.Lib/Helpers/StringExtensions.cs
+++ b/TCC.Lib/Helpers/StringExtensions.cs
@@ -38,6 +38,7 @@ public static String HumanizeSize(this long size)
 
         public static long ParseSize(this string humanizedSize)
         {
+            if (string.IsNullOrWhiteSpace(humanizedSize)) return -1;
             string[] suf = { "b", "ko", "mo", "go", "to", "po", "eo" };
             var size = humanizedSize.Trim().ToLower(CultureInfo.InvariantCulture);
             var number = string.Join("", size.Where(char.IsDigit));

From 2a3bfb5751bacaaf613e1a5450aba8d4eca6c0e4 Mon Sep 17 00:00:00 2001
From: Lilian ARAGO <larago@lucca.fr>
Date: Mon, 6 Mar 2023 11:28:47 +0100
Subject: [PATCH 10/16] Removed ParseSizeInteral

---
 TCC.Lib/Storage/IRemoteStorage.cs       | 4 ++--
 TCC.Lib/Storage/RemoteStorageFactory.cs | 5 ++---
 2 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/TCC.Lib/Storage/IRemoteStorage.cs b/TCC.Lib/Storage/IRemoteStorage.cs
index 7acf3d1..db402a2 100644
--- a/TCC.Lib/Storage/IRemoteStorage.cs
+++ b/TCC.Lib/Storage/IRemoteStorage.cs
@@ -9,13 +9,13 @@ public interface IRemoteStorage
     {
         Task<UploadResponse> UploadAsync(string targetPath, Stream data, CancellationToken token);
 
-        public async Task<UploadResponse> UploadAsync(FileInfo file, DirectoryInfo rootFolder, CancellationToken token)
+        async Task<UploadResponse> UploadAsync(FileInfo file, DirectoryInfo rootFolder, CancellationToken token)
         {
             string targetPath = file.GetRelativeTargetPathTo(rootFolder);
             await using FileStream uploadFileStream = File.OpenRead(file.FullName);
             return await UploadAsync(targetPath, uploadFileStream, token);
         }
 
-        public UploadMode Mode { get; }
+        UploadMode Mode { get; }
     }
 }
\ No newline at end of file
diff --git a/TCC.Lib/Storage/RemoteStorageFactory.cs b/TCC.Lib/Storage/RemoteStorageFactory.cs
index 332a104..8a6a70b 100644
--- a/TCC.Lib/Storage/RemoteStorageFactory.cs
+++ b/TCC.Lib/Storage/RemoteStorageFactory.cs
@@ -15,7 +15,6 @@ namespace TCC.Lib.Storage
 {
     public static class RemoteStorageFactory
     {
-        private static long ParseSizeInternal(string size) => string.IsNullOrWhiteSpace(size) ? 0 : size.ParseSize();
         public static async Task<IEnumerable<IRemoteStorage>> GetRemoteStoragesAsync(this CompressOption option, ILogger logger, CancellationToken token)
         {
             var remoteStorages = new List<IRemoteStorage>();
@@ -73,8 +72,8 @@ public static async Task<IEnumerable<IRemoteStorage>> GetRemoteStoragesAsync(thi
                         remoteStorages.Add(new S3RemoteStorage(
                             new AmazonS3Client(credentials, s3Config),
                             option.S3BucketName,
-                            ParseSizeInternal(option.S3MultipartThreshold),
-                            (int) ParseSizeInternal(option.S3MultipartSize)));
+                            option.S3MultipartThreshold.ParseSize(),
+                            (int) option.S3MultipartSize.ParseSize()));
                         break;
                     case UploadMode.None:
                         break;

From 14840cc45d51f8bc8617dac4da320fd049169925 Mon Sep 17 00:00:00 2001
From: Lilian ARAGO <larago@lucca.fr>
Date: Mon, 6 Mar 2023 12:05:54 +0100
Subject: [PATCH 11/16] Explicit error messages when nstantiating uploaders

---
 TCC.Lib/Storage/RemoteStorageFactory.cs | 56 ++++++++++++++++++-------
 1 file changed, 42 insertions(+), 14 deletions(-)

diff --git a/TCC.Lib/Storage/RemoteStorageFactory.cs b/TCC.Lib/Storage/RemoteStorageFactory.cs
index 8a6a70b..0bb98ad 100644
--- a/TCC.Lib/Storage/RemoteStorageFactory.cs
+++ b/TCC.Lib/Storage/RemoteStorageFactory.cs
@@ -27,11 +27,19 @@ public static async Task<IEnumerable<IRemoteStorage>> GetRemoteStoragesAsync(thi
                 {
                     case UploadMode.AzureSdk:
                         {
-                            if (string.IsNullOrEmpty(option.AzBlobUrl)
-                                || string.IsNullOrEmpty(option.AzBlobContainer)
-                                || string.IsNullOrEmpty(option.AzBlobSaS))
+                            if (string.IsNullOrEmpty(option.AzBlobUrl))
                             {
-                                logger.LogCritical("Configuration error for azure blob upload");
+                                logger.LogCritical("[AzureBlobStorage] Configuration error: missing/invalid --AzBlobUrl");
+                                continue;
+                            }
+                            if (string.IsNullOrEmpty(option.AzBlobContainer))
+                            {
+                                logger.LogCritical("[AzureBlobStorage] Configuration error: missing/invalid --AzBlobContainer");
+                                continue;
+                            }
+                            if (string.IsNullOrEmpty(option.AzBlobSaS))
+                            {
+                                logger.LogCritical("[AzureBlobStorage] Configuration error: missing/invalid --AzBlobSaS");
                                 continue;
                             }
                             var client = new BlobServiceClient(new Uri(option.AzBlobUrl + "/" + option.AzBlobContainer + "?" + option.AzBlobSaS));
@@ -41,10 +49,14 @@ public static async Task<IEnumerable<IRemoteStorage>> GetRemoteStoragesAsync(thi
                         }
                     case UploadMode.GoogleCloudStorage:
                         {
-                            if (string.IsNullOrEmpty(option.GoogleStorageCredential)
-                                || string.IsNullOrEmpty(option.GoogleStorageBucketName))
+                            if (string.IsNullOrEmpty(option.GoogleStorageCredential))
+                            {
+                                logger.LogCritical("[GoogleStorage] Configuration error: missing/invalid --GoogleStorageCredential");
+                                continue;
+                            }
+                            if (string.IsNullOrEmpty(option.GoogleStorageBucketName))
                             {
-                                logger.LogCritical("Configuration error for google storage upload");
+                                logger.LogCritical("[GoogleStorage] Configuration error: missing/invalid --GoogleStorageBucketName");
                                 continue;
                             }
                             StorageClient storage = await GoogleAuthHelper.GetGoogleStorageClientAsync(option.GoogleStorageCredential, token);
@@ -52,14 +64,30 @@ public static async Task<IEnumerable<IRemoteStorage>> GetRemoteStoragesAsync(thi
                             break;
                         }
                     case UploadMode.S3:
-                        if (string.IsNullOrEmpty(option.S3AccessKeyId)
-                            || string.IsNullOrEmpty(option.S3Host)
-                            || string.IsNullOrEmpty(option.S3Region)
-                            || string.IsNullOrEmpty(option.S3BucketName)
-                            || string.IsNullOrEmpty(option.S3SecretAcessKey))
+                        if (string.IsNullOrEmpty(option.S3AccessKeyId))
                         {
-                            logger.LogCritical("Configuration error for S3 upload");
-
+                            logger.LogCritical("[S3Storage] Configuration error: missing/invalid --S3AccessKeyId");
+                            continue;
+                        }
+                        if (string.IsNullOrEmpty(option.S3Host))
+                        {
+                            logger.LogCritical("[S3Storage] Configuration error: missing/invalid --S3Host");
+                            continue;
+                        }
+                        if (string.IsNullOrEmpty(option.S3Region))
+                        {
+                            logger.LogCritical("[S3Storage] Configuration error: missing/invalid --S3Region");
+                            continue;
+                        }
+                        if (string.IsNullOrEmpty(option.S3BucketName))
+                        {
+                            logger.LogCritical("[S3Storage] Configuration error: missing/invalid --S3BucketName");
+                            continue;
+                        }
+                        if (string.IsNullOrEmpty(option.S3SecretAcessKey))
+                        {
+                            logger.LogCritical("[S3Storage] Configuration error: missing/invalid --S3SecretAcessKey");
+                            continue;
                         }
 
                         var credentials = new BasicAWSCredentials(option.S3AccessKeyId, option.S3SecretAcessKey);

From d5528b5f763040762ab0b860421f268f460a2d1f Mon Sep 17 00:00:00 2001
From: Lilian ARAGO <larago@lucca.fr>
Date: Mon, 6 Mar 2023 14:37:32 +0100
Subject: [PATCH 12/16] Gitignore .idea

---
 .gitignore | 25 +++++++++++++------------
 1 file changed, 13 insertions(+), 12 deletions(-)

diff --git a/.gitignore b/.gitignore
index ab19420..5c3e57e 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,12 +1,13 @@
-.vs/*
-*/obj/*
-*/bin/*
-packages 
-*.user
-/coverage-opencover.xml
-launchSettings.json
-*/logs/*
-*.db
-/TCC.Tests/'coverage.xml'
-/coverage.xml
-/published
+.vs/*
+*/obj/*
+*/bin/*
+packages 
+*.user
+/coverage-opencover.xml
+launchSettings.json
+*/logs/*
+*.db
+/TCC.Tests/'coverage.xml'
+/coverage.xml
+/published
+.idea
\ No newline at end of file

From e38f97bc3b96d557a7857b14c8c6b64611e44f14 Mon Sep 17 00:00:00 2001
From: Lilian ARAGO <larago@lucca.fr>
Date: Mon, 6 Mar 2023 14:38:21 +0100
Subject: [PATCH 13/16] Using AsyncEnumerable for GetRemoteStoragesAsync

---
 TCC.Lib/Storage/RemoteStorageFactory.cs |  45 +-
 TCC.Lib/TCC.Lib.csproj                  |   1 +
 TCC.Lib/TarCompressCrypt.cs             | 740 ++++++++++++------------
 TCC.Tests/Upload/BlobStorageTests.cs    | 222 +++----
 4 files changed, 503 insertions(+), 505 deletions(-)

diff --git a/TCC.Lib/Storage/RemoteStorageFactory.cs b/TCC.Lib/Storage/RemoteStorageFactory.cs
index 0bb98ad..3ed7184 100644
--- a/TCC.Lib/Storage/RemoteStorageFactory.cs
+++ b/TCC.Lib/Storage/RemoteStorageFactory.cs
@@ -1,24 +1,22 @@
-using Amazon.Runtime;
-using Amazon.S3;
-using Azure.Storage.Blobs;
-using Google.Cloud.Storage.V1;
-using Microsoft.Extensions.Logging;
-using System;
+using Amazon.Runtime;
+using Amazon.S3;
+using Azure.Storage.Blobs;
+using Google.Cloud.Storage.V1;
+using Microsoft.Extensions.Logging;
+using System;
 using System.Collections.Generic;
 using System.Linq;
-using System.Threading;
-using System.Threading.Tasks;
-using TCC.Lib.Helpers;
-using TCC.Lib.Options;
-
-namespace TCC.Lib.Storage
-{
-    public static class RemoteStorageFactory
+using System.Runtime.CompilerServices;
+using System.Threading;
+using TCC.Lib.Helpers;
+using TCC.Lib.Options;
+
+namespace TCC.Lib.Storage
+{
+    public static class RemoteStorageFactory
     {
-        public static async Task<IEnumerable<IRemoteStorage>> GetRemoteStoragesAsync(this CompressOption option, ILogger logger, CancellationToken token)
+        public static async IAsyncEnumerable<IRemoteStorage> GetRemoteStoragesAsync(this CompressOption option, ILogger logger, [EnumeratorCancellation] CancellationToken token)
         {
-            var remoteStorages = new List<IRemoteStorage>();
-
             option.UploadModes = option.UploadModes.Append(option.UploadMode ?? UploadMode.None).Distinct();
 
             foreach(var mode in option.UploadModes)
@@ -44,7 +42,7 @@ public static async Task<IEnumerable<IRemoteStorage>> GetRemoteStoragesAsync(thi
                             }
                             var client = new BlobServiceClient(new Uri(option.AzBlobUrl + "/" + option.AzBlobContainer + "?" + option.AzBlobSaS));
                             BlobContainerClient container = client.GetBlobContainerClient(option.AzBlobContainer);
-                            remoteStorages.Add(new AzureRemoteStorage(container));
+                            yield return new AzureRemoteStorage(container);
                             break;
                         }
                     case UploadMode.GoogleCloudStorage:
@@ -60,7 +58,7 @@ public static async Task<IEnumerable<IRemoteStorage>> GetRemoteStoragesAsync(thi
                                 continue;
                             }
                             StorageClient storage = await GoogleAuthHelper.GetGoogleStorageClientAsync(option.GoogleStorageCredential, token);
-                            remoteStorages.Add(new GoogleRemoteStorage(storage, option.GoogleStorageBucketName));
+                            yield return new GoogleRemoteStorage(storage, option.GoogleStorageBucketName);
                             break;
                         }
                     case UploadMode.S3:
@@ -97,11 +95,11 @@ public static async Task<IEnumerable<IRemoteStorage>> GetRemoteStoragesAsync(thi
                             ServiceURL = option.S3Host,
                         };
 
-                        remoteStorages.Add(new S3RemoteStorage(
+                        yield return new S3RemoteStorage(
                             new AmazonS3Client(credentials, s3Config),
                             option.S3BucketName,
                             option.S3MultipartThreshold.ParseSize(),
-                            (int) option.S3MultipartSize.ParseSize()));
+                            (int)option.S3MultipartSize.ParseSize());
                         break;
                     case UploadMode.None:
                         break;
@@ -109,7 +107,6 @@ public static async Task<IEnumerable<IRemoteStorage>> GetRemoteStoragesAsync(thi
                         throw new ArgumentOutOfRangeException();
                 }
             }
-            return remoteStorages;
-        }
-    }
+        }
+    }
 }
\ No newline at end of file
diff --git a/TCC.Lib/TCC.Lib.csproj b/TCC.Lib/TCC.Lib.csproj
index 25043fa..4befbab 100644
--- a/TCC.Lib/TCC.Lib.csproj
+++ b/TCC.Lib/TCC.Lib.csproj
@@ -18,6 +18,7 @@
     <PackageReference Include="Microsoft.EntityFrameworkCore.SqlServer" Version="6.0.7" />
     <PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="6.0.0" />
     <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="6.0.1" />
+    <PackageReference Include="System.Linq.Async" Version="6.0.1" />
     <PackageReference Include="System.Management" Version="6.0.0" />
     <PackageReference Include="System.Threading.Channels" Version="6.0.0" />
     <PackageReference Include="Newtonsoft.Json" Version="13.0.1" />
diff --git a/TCC.Lib/TarCompressCrypt.cs b/TCC.Lib/TarCompressCrypt.cs
index 34bf274..9218a22 100644
--- a/TCC.Lib/TarCompressCrypt.cs
+++ b/TCC.Lib/TarCompressCrypt.cs
@@ -1,94 +1,94 @@
-using Microsoft.Extensions.DependencyInjection;
-using Microsoft.Extensions.Logging;
-using System;
-using System.Collections.Generic;
-using System.Diagnostics;
-using System.IO;
-using System.Linq;
-using System.Threading;
-using System.Threading.Tasks;
-using TCC.Lib.AsyncStreams;
-using TCC.Lib.Blocks;
-using TCC.Lib.Command;
-using TCC.Lib.Database;
-using TCC.Lib.Dependencies;
-using TCC.Lib.Helpers;
-using TCC.Lib.Options;
-using TCC.Lib.PrepareBlocks;
-using TCC.Lib.Storage;
-
-namespace TCC.Lib
-{
-    public class TarCompressCrypt
-    {
-        private readonly CancellationTokenSource _cancellationTokenSource;
-        private readonly ILogger<TarCompressCrypt> _logger;
-        private readonly EncryptionCommands _encryptionCommands;
-        private readonly CompressionCommands _compressionCommands;
-        private readonly IServiceProvider _serviceProvider;
-        private readonly DatabaseHelper _databaseHelper;
-        private int _compressCounter;
-        private int _uploadCounter;
-        private int _totalCounter;
-
-        public TarCompressCrypt(CancellationTokenSource cancellationTokenSource, ILogger<TarCompressCrypt> logger, EncryptionCommands encryptionCommands, CompressionCommands compressionCommands, IServiceProvider serviceProvider, DatabaseHelper databaseHelper)
-        {
-            _cancellationTokenSource = cancellationTokenSource;
-            _logger = logger;
-            _encryptionCommands = encryptionCommands;
-            _compressionCommands = compressionCommands;
-            _serviceProvider = serviceProvider;
-            _databaseHelper = databaseHelper;
-        }
-
-        public async Task<OperationSummary> CompressAsync(CompressOption option)
-        {
-            var sw = Stopwatch.StartNew();
-
-            var compFolder = new CompressionFolderProvider(new DirectoryInfo(option.DestinationDir), option.FolderPerDay);
-
-            IEnumerable<CompressionBlock> blocks = option.GenerateCompressBlocks(compFolder);
-            IPrepareCompressBlocks prepare = new FileSystemPrepareCompressBlocks(compFolder, option.BackupMode, option.CleanupTime);
-            var buffer = prepare.PrepareCompressionBlocksAsync(blocks);
-            _totalCounter = buffer.Count;
-            PrepareBoostRatio(option, buffer);
-            _logger.LogInformation("job prepared in " + sw.Elapsed.HumanizedTimeSpan());
-            _logger.LogInformation("Starting compression job");
-            var po = ParallelizeOption(option);
-
-            IEnumerable<IRemoteStorage> uploaders = await option.GetRemoteStoragesAsync(_logger, _cancellationTokenSource.Token);
-
-            var operationBlocks = await buffer
-                .AsAsyncStream(_cancellationTokenSource.Token)
-                .CountAsync(out var counter)
-                // Prepare encryption
-                .ParallelizeStreamAsync(async (b, token) =>
-                {
-                    b.StartTime = DateTime.UtcNow;
-                    await _encryptionCommands.PrepareEncryptionKey(b, option, token);
-                    return b;
-                }, po)
-                // Core loop 
-                .ParallelizeStreamAsync((block, token) =>
-                {
-                    return CompressionBlockInternal(option, block, token);
-                }, po)
-                // Cleanup loop
-                .ParallelizeStreamAsync(async (opb, token) =>
-                {
-                    await CleanupOldFiles(opb);
-                    await _encryptionCommands.CleanupKey(opb.BlockResult.Block, option, Mode.Compress);
-                    return opb;
-                }, po)
-                // Upload loop
-                .ParallelizeStreamAsync((block, token) => UploadBlockInternal(uploaders, option, block, token), new ParallelizeOption { FailMode = Fail.Smart, MaxDegreeOfParallelism = option.AzThread ?? 1 })
-                .AsReadOnlyCollectionAsync();
-
-            sw.Stop();
-            return new OperationSummary(operationBlocks, option.Threads, sw, option.SourceDirOrFile);
-        }
-
-        private async Task<OperationCompressionBlock> UploadBlockInternal(IEnumerable<IRemoteStorage> uploaders, CompressOption option, OperationCompressionBlock block, CancellationToken token)
+using Microsoft.Extensions.DependencyInjection;
+using Microsoft.Extensions.Logging;
+using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using System.IO;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using TCC.Lib.AsyncStreams;
+using TCC.Lib.Blocks;
+using TCC.Lib.Command;
+using TCC.Lib.Database;
+using TCC.Lib.Dependencies;
+using TCC.Lib.Helpers;
+using TCC.Lib.Options;
+using TCC.Lib.PrepareBlocks;
+using TCC.Lib.Storage;
+
+namespace TCC.Lib
+{
+    public class TarCompressCrypt
+    {
+        private readonly CancellationTokenSource _cancellationTokenSource;
+        private readonly ILogger<TarCompressCrypt> _logger;
+        private readonly EncryptionCommands _encryptionCommands;
+        private readonly CompressionCommands _compressionCommands;
+        private readonly IServiceProvider _serviceProvider;
+        private readonly DatabaseHelper _databaseHelper;
+        private int _compressCounter;
+        private int _uploadCounter;
+        private int _totalCounter;
+
+        public TarCompressCrypt(CancellationTokenSource cancellationTokenSource, ILogger<TarCompressCrypt> logger, EncryptionCommands encryptionCommands, CompressionCommands compressionCommands, IServiceProvider serviceProvider, DatabaseHelper databaseHelper)
+        {
+            _cancellationTokenSource = cancellationTokenSource;
+            _logger = logger;
+            _encryptionCommands = encryptionCommands;
+            _compressionCommands = compressionCommands;
+            _serviceProvider = serviceProvider;
+            _databaseHelper = databaseHelper;
+        }
+
+        public async Task<OperationSummary> CompressAsync(CompressOption option)
+        {
+            var sw = Stopwatch.StartNew();
+
+            var compFolder = new CompressionFolderProvider(new DirectoryInfo(option.DestinationDir), option.FolderPerDay);
+
+            IEnumerable<CompressionBlock> blocks = option.GenerateCompressBlocks(compFolder);
+            IPrepareCompressBlocks prepare = new FileSystemPrepareCompressBlocks(compFolder, option.BackupMode, option.CleanupTime);
+            var buffer = prepare.PrepareCompressionBlocksAsync(blocks);
+            _totalCounter = buffer.Count;
+            PrepareBoostRatio(option, buffer);
+            _logger.LogInformation("job prepared in " + sw.Elapsed.HumanizedTimeSpan());
+            _logger.LogInformation("Starting compression job");
+            var po = ParallelizeOption(option);
+
+            List<IRemoteStorage> uploaders = await option.GetRemoteStoragesAsync(_logger, _cancellationTokenSource.Token).ToListAsync();
+
+            var operationBlocks = await buffer
+                .AsAsyncStream(_cancellationTokenSource.Token)
+                .CountAsync(out var counter)
+                // Prepare encryption
+                .ParallelizeStreamAsync(async (b, token) =>
+                {
+                    b.StartTime = DateTime.UtcNow;
+                    await _encryptionCommands.PrepareEncryptionKey(b, option, token);
+                    return b;
+                }, po)
+                // Core loop 
+                .ParallelizeStreamAsync((block, token) =>
+                {
+                    return CompressionBlockInternal(option, block, token);
+                }, po)
+                // Cleanup loop
+                .ParallelizeStreamAsync(async (opb, token) =>
+                {
+                    await CleanupOldFiles(opb);
+                    await _encryptionCommands.CleanupKey(opb.BlockResult.Block, option, Mode.Compress);
+                    return opb;
+                }, po)
+                // Upload loop
+                .ParallelizeStreamAsync((block, token) => UploadBlockInternal(uploaders, option, block, token), new ParallelizeOption { FailMode = Fail.Smart, MaxDegreeOfParallelism = option.AzThread ?? 1 })
+                .AsReadOnlyCollectionAsync();
+
+            sw.Stop();
+            return new OperationSummary(operationBlocks, option.Threads, sw, option.SourceDirOrFile);
+        }
+
+        private async Task<OperationCompressionBlock> UploadBlockInternal(IEnumerable<IRemoteStorage> uploaders, CompressOption option, OperationCompressionBlock block, CancellationToken token)
         {
             int count = Interlocked.Increment(ref _uploadCounter);
             string progress = $"{count}/{_totalCounter}";
@@ -100,7 +100,7 @@ private async Task<OperationCompressionBlock> UploadBlockInternal(IEnumerable<IR
                     continue;
                 }
                 await UploadBlockToSingleRemoteStorageAsync(option, block, progress, uploader, token);
-            }
+            }
             return block;
         }
 
@@ -174,282 +174,282 @@ private async Task UploadBlockToSingleRemoteStorageAsync(CompressOption option,
             }
         }
 
-        private async Task CleanupOldFiles(OperationCompressionBlock opb)
-        {
-            if (opb.CompressionBlock.FullsToDelete != null)
-            {
-                foreach (var full in opb.CompressionBlock.FullsToDelete)
-                {
-                    try
-                    {
-                        await full.TryDeleteFileWithRetryAsync();
-                    }
-                    catch (Exception e)
-                    {
-                        _logger.LogError(e, "Error while deleting FULL file ");
-                    }
-                }
-            }
-            if (opb.CompressionBlock.DiffsToDelete != null)
-            {
-                foreach (var diff in opb.CompressionBlock.DiffsToDelete)
-                {
-                    try
-                    {
-                        await diff.TryDeleteFileWithRetryAsync();
-                    }
-                    catch (Exception e)
-                    {
-                        _logger.LogError(e, "Error while deleting DIFF file ");
-                    }
-                }
-            }
-        }
-
-        private void PrepareBoostRatio(CompressOption option, IEnumerable<CompressionBlock> buffer)
-        {
-            _logger.LogInformation("Requested order : ");
-            int countFull = 0;
-            int countDiff = 0;
-            foreach (CompressionBlock block in buffer)
-            {
-                if (block.BackupMode == BackupMode.Full)
-                {
-                    countFull++;
-                }
-                else
-                {
-                    countDiff++;
-                }
-            }
-            if (option.Threads > 1 && option.BoostRatio.HasValue)
-            {
-                if (countFull == 0 && countDiff > 0)
-                {
-                    _logger.LogInformation($"100% diff ({countDiff}), running with X{option.BoostRatio.Value} more threads");
-                    // boost mode when 100% of diff, we want to saturate iops : X mode
-                    option.Threads = Math.Min(option.Threads * option.BoostRatio.Value, countDiff);
-                    if (option.AzThread.HasValue)
-                    {
-                        option.AzThread = Math.Min(option.AzThread.Value * option.BoostRatio.Value, countDiff);
-                    }
-                }
-                else if (countFull != 0 && (countDiff / (double)(countFull + countDiff) >= 0.9))
-                {
-                    _logger.LogInformation($"{countFull} full, {countDiff} diffs, running with X{option.BoostRatio.Value} more threads");
-                    // boost mode when 95% of diff, we want to saturate iops
-                    option.Threads = Math.Min(option.Threads * option.BoostRatio.Value, countDiff);
-                    if (option.AzThread.HasValue)
-                    {
-                        option.AzThread = Math.Min(option.AzThread.Value * option.BoostRatio.Value, countDiff);
-                    }
-                }
-                else
-                {
-                    _logger.LogInformation($"No boost mode : {countFull} full, {countDiff} diffs");
-                }
-            }
-        }
-
-        private async Task<OperationCompressionBlock> CompressionBlockInternal(CompressOption option, CompressionBlock block, CancellationToken token)
-        {
-
-            int count = Interlocked.Increment(ref _compressCounter);
-            string progress = $"{count}/{_totalCounter}";
-            RetryContext ctx = null;
-            CommandResult result = null;
-            while (true)
-            {
-                bool hasError = false;
-                try
-                {
-                    string cmd = _compressionCommands.CompressCommand(block, option);
-                    result = await cmd.Run(block.OperationFolder, token);
-                    result.ArchiveFileSize = block.DestinationArchiveFileInfo.Length;
-                    LogCompressionReport(block, result);
-
-                    if (result.HasError)
-                    {
-                        hasError = true;
-                        if (ctx == null && option.RetryPeriodInSeconds.HasValue)
-                        {
-                            ctx = new RetryContext(option.RetryPeriodInSeconds.Value);
-                        }
-                    }
-                    var report = $"{progress} [{block.BackupMode}] : {block.BlockName}";
-                    if (block.BackupMode == BackupMode.Diff)
-                    {
-                        report += $" (from {block.DiffDate})";
-                    }
-                    _logger.LogInformation(report);
-
-                }
-                catch (Exception e)
-                {
-                    hasError = true;
-                    if (ctx == null && option.RetryPeriodInSeconds.HasValue)
-                    {
-                        ctx = new RetryContext(option.RetryPeriodInSeconds.Value);
-                    }
-                    _logger.LogCritical(e, $"{progress} Error compressing {block.Source}");
-                }
-
-                if (hasError)
-                {
-                    if (ctx != null && await ctx.WaitForNextRetry())
-                    {
-                        _logger.LogWarning($"{progress} Retrying compressing {block.Source}, attempt #{ctx.Retries}");
-                        await block.DestinationArchiveFileInfo.TryDeleteFileWithRetryAsync();
-                    }
-                    else
-                    {
-                        await block.DestinationArchiveFileInfo.TryDeleteFileWithRetryAsync();
-                        break;
-                    }
-                }
-                else
-                {
-                    break;
-                }
-            }
-            return new OperationCompressionBlock(block, result);
-        }
-
-        public async Task<OperationSummary> DecompressAsync(DecompressOption option)
-        {
-            var sw = Stopwatch.StartNew();
-            var po = ParallelizeOption(option);
-            var job = await _databaseHelper.InitializeRestoreJobAsync();
-
-            IEnumerable<DecompressionBatch> blocks = option.GenerateDecompressBlocks().ToList();
-            var prepare = new DatabasePreparedDecompressionBlocks(_serviceProvider.GetRequiredService<TccRestoreDbContext>(), _logger);
-            IAsyncEnumerable<DecompressionBatch> ordered = prepare.PrepareDecompressionBlocksAsync(blocks);
-
-            IReadOnlyCollection<OperationDecompressionsBlock> operationBlocks =
-                await ordered
-                .AsAsyncStream(_cancellationTokenSource.Token)
-                .CountAsync(out var counter)
-                // Prepare decryption
-                .ParallelizeStreamAsync(async (b, token) =>
-                {
-                    b.StartTime = DateTime.UtcNow;
-                    await _encryptionCommands.PrepareDecryptionKey(b.BackupFull ?? b.BackupsDiff.FirstOrDefault(), option, token);
-                    return b;
-                }, po)
-                // Core loop 
-                .ParallelizeStreamAsync(async (batch, token) =>
-                {
-                    int count = Interlocked.Increment(ref _compressCounter);
-                    string progress = $"{count}/{_totalCounter}";
-
-                    var blockResults = new List<BlockResult>();
-
-                    if (batch.BackupFull != null)
-                    {
-                        batch.BackupFullCommandResult = await DecompressBlock(option, batch.BackupFull, token);
-                        blockResults.Add(new BlockResult(batch.BackupFull, batch.BackupFullCommandResult, StepType.Decompression));
-                    }
-
-                    if (batch.BackupsDiff != null)
-                    {
-                        batch.BackupDiffCommandResult = new CommandResult[batch.BackupsDiff.Length];
-                        for (int i = 0; i < batch.BackupsDiff.Length; i++)
-                        {
-                            batch.BackupDiffCommandResult[i] = await DecompressBlock(option, batch.BackupsDiff[i], token);
-                            blockResults.Add(new BlockResult(batch.BackupsDiff[i], batch.BackupDiffCommandResult[i], StepType.Decompression));
-                        }
-                    }
-
-                    if (batch.BackupFull != null)
-                    {
-                        var report = $"{progress} [{BackupMode.Full}] : {batch.BackupFull.BlockName} (from {batch.BackupFull.BlockDate})";
-                        _logger.LogInformation(report);
-                    }
-                    if (batch.BackupsDiff != null)
-                    {
-                        foreach (var dec in batch.BackupsDiff)
-                        {
-                            var report = $"{progress} [{BackupMode.Diff}] : {dec.BlockName} (from {dec.BlockDate})";
-                            _logger.LogInformation(report);
-                        }
-                    }
-
-                    return new OperationDecompressionsBlock(blockResults, batch);
-                }, po)
-                // Cleanup loop
-                .ParallelizeStreamAsync(async (odb, token) =>
-                {
-                    foreach (var b in odb.BlockResults)
-                    {
-                        await _encryptionCommands.CleanupKey(b.Block, option, Mode.Compress);
-                    }
-                    return odb;
-                }, po)
-                .AsReadOnlyCollectionAsync();
-
-            sw.Stop();
-            await _databaseHelper.AddRestoreBlockJobAsync(job, operationBlocks);
-            await _databaseHelper.UpdateRestoreJobStatsAsync(sw, job);
-            return new OperationSummary(operationBlocks, option.Threads, sw, option.SourceDirOrFile);
-        }
-
-        private async Task<CommandResult> DecompressBlock(DecompressOption option, DecompressionBlock block, CancellationToken token)
-        {
-            CommandResult result = null;
-            try
-            {
-                string cmd = _compressionCommands.DecompressCommand(block, option);
-                result = await cmd.Run(block.OperationFolder, token);
-                result.ArchiveFileSize = block.SourceArchiveFileInfo.Length;
-
-                LogReport(block, result);
-            }
-            catch (Exception e)
-            {
-                _logger.LogError(e, $"Error decompressing {block.Source}");
-            }
-            return result;
-        }
-
-        private static ParallelizeOption ParallelizeOption(TccOption option)
-        {
-            var po = new ParallelizeOption
-            {
-                FailMode = option.FailFast ? Fail.Fast : Fail.Smart,
-                MaxDegreeOfParallelism = option.Threads
-            };
-            return po;
-        }
-
-        private void LogCompressionReport(CompressionBlock block, CommandResult result)
-        {
-            _logger.LogInformation($"Compressed {block.Source} in {result?.Elapsed.HumanizedTimeSpan()}, {block.DestinationArchiveFileInfo.Length.HumanizeSize()}, {block.DestinationArchiveFileInfo.FullName}");
-
-            if (result?.Infos.Any() ?? false)
-            {
-                _logger.LogWarning($"Compressed {block.Source} with warning : {string.Join(Environment.NewLine, result.Infos)}");
-            }
-
-            if (result?.HasError ?? false)
-            {
-                _logger.LogError($"Compressed {block.Source} with errors : {result.Errors}");
-            }
-        }
-
-
-        private void LogReport(DecompressionBlock block, CommandResult result)
-        {
-            _logger.LogInformation($"Decompressed {block.Source} in {result?.Elapsed.HumanizedTimeSpan()}, {block.SourceArchiveFileInfo.Length.HumanizeSize()}, {block.SourceArchiveFileInfo.FullName}");
-
-            if (result?.Infos.Any() ?? false)
-            {
-                _logger.LogWarning($"Decompressed {block.Source} with warning : {string.Join(Environment.NewLine, result.Infos)}");
-            }
-
-            if (result?.HasError ?? false)
-            {
-                _logger.LogError($"Decompressed {block.Source} with errors : {result.Errors}");
-            }
-        }
-    }
+        private async Task CleanupOldFiles(OperationCompressionBlock opb)
+        {
+            if (opb.CompressionBlock.FullsToDelete != null)
+            {
+                foreach (var full in opb.CompressionBlock.FullsToDelete)
+                {
+                    try
+                    {
+                        await full.TryDeleteFileWithRetryAsync();
+                    }
+                    catch (Exception e)
+                    {
+                        _logger.LogError(e, "Error while deleting FULL file ");
+                    }
+                }
+            }
+            if (opb.CompressionBlock.DiffsToDelete != null)
+            {
+                foreach (var diff in opb.CompressionBlock.DiffsToDelete)
+                {
+                    try
+                    {
+                        await diff.TryDeleteFileWithRetryAsync();
+                    }
+                    catch (Exception e)
+                    {
+                        _logger.LogError(e, "Error while deleting DIFF file ");
+                    }
+                }
+            }
+        }
+
+        private void PrepareBoostRatio(CompressOption option, IEnumerable<CompressionBlock> buffer)
+        {
+            _logger.LogInformation("Requested order : ");
+            int countFull = 0;
+            int countDiff = 0;
+            foreach (CompressionBlock block in buffer)
+            {
+                if (block.BackupMode == BackupMode.Full)
+                {
+                    countFull++;
+                }
+                else
+                {
+                    countDiff++;
+                }
+            }
+            if (option.Threads > 1 && option.BoostRatio.HasValue)
+            {
+                if (countFull == 0 && countDiff > 0)
+                {
+                    _logger.LogInformation($"100% diff ({countDiff}), running with X{option.BoostRatio.Value} more threads");
+                    // boost mode when 100% of diff, we want to saturate iops : X mode
+                    option.Threads = Math.Min(option.Threads * option.BoostRatio.Value, countDiff);
+                    if (option.AzThread.HasValue)
+                    {
+                        option.AzThread = Math.Min(option.AzThread.Value * option.BoostRatio.Value, countDiff);
+                    }
+                }
+                else if (countFull != 0 && (countDiff / (double)(countFull + countDiff) >= 0.9))
+                {
+                    _logger.LogInformation($"{countFull} full, {countDiff} diffs, running with X{option.BoostRatio.Value} more threads");
+                    // boost mode when 95% of diff, we want to saturate iops
+                    option.Threads = Math.Min(option.Threads * option.BoostRatio.Value, countDiff);
+                    if (option.AzThread.HasValue)
+                    {
+                        option.AzThread = Math.Min(option.AzThread.Value * option.BoostRatio.Value, countDiff);
+                    }
+                }
+                else
+                {
+                    _logger.LogInformation($"No boost mode : {countFull} full, {countDiff} diffs");
+                }
+            }
+        }
+
+        private async Task<OperationCompressionBlock> CompressionBlockInternal(CompressOption option, CompressionBlock block, CancellationToken token)
+        {
+
+            int count = Interlocked.Increment(ref _compressCounter);
+            string progress = $"{count}/{_totalCounter}";
+            RetryContext ctx = null;
+            CommandResult result = null;
+            while (true)
+            {
+                bool hasError = false;
+                try
+                {
+                    string cmd = _compressionCommands.CompressCommand(block, option);
+                    result = await cmd.Run(block.OperationFolder, token);
+                    result.ArchiveFileSize = block.DestinationArchiveFileInfo.Length;
+                    LogCompressionReport(block, result);
+
+                    if (result.HasError)
+                    {
+                        hasError = true;
+                        if (ctx == null && option.RetryPeriodInSeconds.HasValue)
+                        {
+                            ctx = new RetryContext(option.RetryPeriodInSeconds.Value);
+                        }
+                    }
+                    var report = $"{progress} [{block.BackupMode}] : {block.BlockName}";
+                    if (block.BackupMode == BackupMode.Diff)
+                    {
+                        report += $" (from {block.DiffDate})";
+                    }
+                    _logger.LogInformation(report);
+
+                }
+                catch (Exception e)
+                {
+                    hasError = true;
+                    if (ctx == null && option.RetryPeriodInSeconds.HasValue)
+                    {
+                        ctx = new RetryContext(option.RetryPeriodInSeconds.Value);
+                    }
+                    _logger.LogCritical(e, $"{progress} Error compressing {block.Source}");
+                }
+
+                if (hasError)
+                {
+                    if (ctx != null && await ctx.WaitForNextRetry())
+                    {
+                        _logger.LogWarning($"{progress} Retrying compressing {block.Source}, attempt #{ctx.Retries}");
+                        await block.DestinationArchiveFileInfo.TryDeleteFileWithRetryAsync();
+                    }
+                    else
+                    {
+                        await block.DestinationArchiveFileInfo.TryDeleteFileWithRetryAsync();
+                        break;
+                    }
+                }
+                else
+                {
+                    break;
+                }
+            }
+            return new OperationCompressionBlock(block, result);
+        }
+
+        public async Task<OperationSummary> DecompressAsync(DecompressOption option)
+        {
+            var sw = Stopwatch.StartNew();
+            var po = ParallelizeOption(option);
+            var job = await _databaseHelper.InitializeRestoreJobAsync();
+
+            IEnumerable<DecompressionBatch> blocks = option.GenerateDecompressBlocks().ToList();
+            var prepare = new DatabasePreparedDecompressionBlocks(_serviceProvider.GetRequiredService<TccRestoreDbContext>(), _logger);
+            IAsyncEnumerable<DecompressionBatch> ordered = prepare.PrepareDecompressionBlocksAsync(blocks);
+
+            IReadOnlyCollection<OperationDecompressionsBlock> operationBlocks =
+                await ordered
+                .AsAsyncStream(_cancellationTokenSource.Token)
+                .CountAsync(out var counter)
+                // Prepare decryption
+                .ParallelizeStreamAsync(async (b, token) =>
+                {
+                    b.StartTime = DateTime.UtcNow;
+                    await _encryptionCommands.PrepareDecryptionKey(b.BackupFull ?? b.BackupsDiff.FirstOrDefault(), option, token);
+                    return b;
+                }, po)
+                // Core loop 
+                .ParallelizeStreamAsync(async (batch, token) =>
+                {
+                    int count = Interlocked.Increment(ref _compressCounter);
+                    string progress = $"{count}/{_totalCounter}";
+
+                    var blockResults = new List<BlockResult>();
+
+                    if (batch.BackupFull != null)
+                    {
+                        batch.BackupFullCommandResult = await DecompressBlock(option, batch.BackupFull, token);
+                        blockResults.Add(new BlockResult(batch.BackupFull, batch.BackupFullCommandResult, StepType.Decompression));
+                    }
+
+                    if (batch.BackupsDiff != null)
+                    {
+                        batch.BackupDiffCommandResult = new CommandResult[batch.BackupsDiff.Length];
+                        for (int i = 0; i < batch.BackupsDiff.Length; i++)
+                        {
+                            batch.BackupDiffCommandResult[i] = await DecompressBlock(option, batch.BackupsDiff[i], token);
+                            blockResults.Add(new BlockResult(batch.BackupsDiff[i], batch.BackupDiffCommandResult[i], StepType.Decompression));
+                        }
+                    }
+
+                    if (batch.BackupFull != null)
+                    {
+                        var report = $"{progress} [{BackupMode.Full}] : {batch.BackupFull.BlockName} (from {batch.BackupFull.BlockDate})";
+                        _logger.LogInformation(report);
+                    }
+                    if (batch.BackupsDiff != null)
+                    {
+                        foreach (var dec in batch.BackupsDiff)
+                        {
+                            var report = $"{progress} [{BackupMode.Diff}] : {dec.BlockName} (from {dec.BlockDate})";
+                            _logger.LogInformation(report);
+                        }
+                    }
+
+                    return new OperationDecompressionsBlock(blockResults, batch);
+                }, po)
+                // Cleanup loop
+                .ParallelizeStreamAsync(async (odb, token) =>
+                {
+                    foreach (var b in odb.BlockResults)
+                    {
+                        await _encryptionCommands.CleanupKey(b.Block, option, Mode.Compress);
+                    }
+                    return odb;
+                }, po)
+                .AsReadOnlyCollectionAsync();
+
+            sw.Stop();
+            await _databaseHelper.AddRestoreBlockJobAsync(job, operationBlocks);
+            await _databaseHelper.UpdateRestoreJobStatsAsync(sw, job);
+            return new OperationSummary(operationBlocks, option.Threads, sw, option.SourceDirOrFile);
+        }
+
+        private async Task<CommandResult> DecompressBlock(DecompressOption option, DecompressionBlock block, CancellationToken token)
+        {
+            CommandResult result = null;
+            try
+            {
+                string cmd = _compressionCommands.DecompressCommand(block, option);
+                result = await cmd.Run(block.OperationFolder, token);
+                result.ArchiveFileSize = block.SourceArchiveFileInfo.Length;
+
+                LogReport(block, result);
+            }
+            catch (Exception e)
+            {
+                _logger.LogError(e, $"Error decompressing {block.Source}");
+            }
+            return result;
+        }
+
+        private static ParallelizeOption ParallelizeOption(TccOption option)
+        {
+            var po = new ParallelizeOption
+            {
+                FailMode = option.FailFast ? Fail.Fast : Fail.Smart,
+                MaxDegreeOfParallelism = option.Threads
+            };
+            return po;
+        }
+
+        private void LogCompressionReport(CompressionBlock block, CommandResult result)
+        {
+            _logger.LogInformation($"Compressed {block.Source} in {result?.Elapsed.HumanizedTimeSpan()}, {block.DestinationArchiveFileInfo.Length.HumanizeSize()}, {block.DestinationArchiveFileInfo.FullName}");
+
+            if (result?.Infos.Any() ?? false)
+            {
+                _logger.LogWarning($"Compressed {block.Source} with warning : {string.Join(Environment.NewLine, result.Infos)}");
+            }
+
+            if (result?.HasError ?? false)
+            {
+                _logger.LogError($"Compressed {block.Source} with errors : {result.Errors}");
+            }
+        }
+
+
+        private void LogReport(DecompressionBlock block, CommandResult result)
+        {
+            _logger.LogInformation($"Decompressed {block.Source} in {result?.Elapsed.HumanizedTimeSpan()}, {block.SourceArchiveFileInfo.Length.HumanizeSize()}, {block.SourceArchiveFileInfo.FullName}");
+
+            if (result?.Infos.Any() ?? false)
+            {
+                _logger.LogWarning($"Decompressed {block.Source} with warning : {string.Join(Environment.NewLine, result.Infos)}");
+            }
+
+            if (result?.HasError ?? false)
+            {
+                _logger.LogError($"Decompressed {block.Source} with errors : {result.Errors}");
+            }
+        }
+    }
 }
\ No newline at end of file
diff --git a/TCC.Tests/Upload/BlobStorageTests.cs b/TCC.Tests/Upload/BlobStorageTests.cs
index 00cd531..830689e 100644
--- a/TCC.Tests/Upload/BlobStorageTests.cs
+++ b/TCC.Tests/Upload/BlobStorageTests.cs
@@ -1,112 +1,112 @@
-using Microsoft.Extensions.Logging.Abstractions;
-using Newtonsoft.Json;
-using Newtonsoft.Json.Linq;
-using System;
+using Microsoft.Extensions.Logging.Abstractions;
+using Newtonsoft.Json;
+using Newtonsoft.Json.Linq;
+using System;
 using System.Collections.Generic;
-using System.IO;
-using System.Linq;
-using System.Threading;
-using System.Threading.Tasks;
-using TCC.Lib;
-using TCC.Lib.Benchmark;
-using TCC.Lib.Options;
-using TCC.Lib.Storage;
-using Xunit;
-
-namespace TCC.Tests.Upload
-{
-    public class UploadToStorageTests : IClassFixture<EnvVarFixture>
-    {
-        private EnvVarFixture _envVarFixture;
-
-        public UploadToStorageTests(EnvVarFixture envVarFixture)
-        {
-            _envVarFixture = envVarFixture;
-        }
-
-        [Fact(Skip = "desactivated accound")]
-        public async Task AzureUploadTest()
-        {
-            string toCompressFolder = TestFileHelper.NewFolder();
-            var data = await TestData.CreateFiles(1, 1024, toCompressFolder);
-
-            var opt = new CompressOption()
-            {
-                AzBlobUrl = GetEnvVar("AZ_URL"),
-                AzBlobContainer = GetEnvVar("AZ_CONTAINER"),
-                AzBlobSaS = GetEnvVar("AZ_SAS_TOKEN")
-            };
-            opt.UploadModes = new List<UploadMode>() { UploadMode.AzureSdk };
-            var uploader = await opt.GetRemoteStoragesAsync(NullLogger.Instance, CancellationToken.None);
-
-            var ok = await uploader.First().UploadAsync(data.Files.First(), new DirectoryInfo(toCompressFolder), CancellationToken.None);
-
-            Assert.True(ok.IsSuccess);
-        }
-
-        [Fact]
-        public async Task GoogleUploadTest()
-        {
-            string toCompressFolder = TestFileHelper.NewFolder();
-            var data = await TestData.CreateFiles(1, 1024, toCompressFolder);
-
-            var opt = new CompressOption()
-            {
-                GoogleStorageBucketName = GetEnvVar("GoogleBucket"),
-                GoogleStorageCredential = GetEnvVar("GoogleCredential")
-            };
-
-            opt.UploadModes = new List<UploadMode>() { UploadMode.GoogleCloudStorage };
-            var uploader = await opt.GetRemoteStoragesAsync(NullLogger.Instance, CancellationToken.None);
-
-            var ok = await uploader.First().UploadAsync(data.Files.First(), new DirectoryInfo(toCompressFolder), CancellationToken.None);
-
-            Assert.True(ok.IsSuccess);
-
-            var gs = uploader.First() as GoogleRemoteStorage;
-            
-            await gs.Storage.DeleteObjectAsync(gs.BucketName, ok.RemoteFilePath);
-        }
-
-        string GetEnvVar(string key)
-        {
-            var s = Environment.GetEnvironmentVariable(key);
-            Assert.True(s != null, key);
-            Assert.True(!string.IsNullOrWhiteSpace(s), key);
-            return s;
-        }
-
-    }
-
-
-    public class EnvVarFixture : IDisposable
-    {
-        public EnvVarFixture()
-        {
-            if (File.Exists("Properties\\launchSettings.json"))
-            {
-                using var file = File.OpenText("Properties\\launchSettings.json");
-                var reader = new JsonTextReader(file);
-                var jObject = JObject.Load(reader);
-
-                var variables = jObject
-                    .GetValue("profiles")
-                    //select a proper profile here
-                    .SelectMany(profiles => profiles.Children())
-                    .SelectMany(profile => profile.Children<JProperty>())
-                    .Where(prop => prop.Name == "environmentVariables")
-                    .SelectMany(prop => prop.Value.Children<JProperty>())
-                    .ToList();
-
-                foreach (var variable in variables)
-                {
-                    Environment.SetEnvironmentVariable(variable.Name, variable.Value.ToString());
-                }
-            }
-        }
-
-        public void Dispose()
-        {
-        }
-    }
-}
+using System.IO;
+using System.Linq;
+using System.Threading;
+using System.Threading.Tasks;
+using TCC.Lib;
+using TCC.Lib.Benchmark;
+using TCC.Lib.Options;
+using TCC.Lib.Storage;
+using Xunit;
+
+namespace TCC.Tests.Upload
+{
+    public class UploadToStorageTests : IClassFixture<EnvVarFixture>
+    {
+        private EnvVarFixture _envVarFixture;
+
+        public UploadToStorageTests(EnvVarFixture envVarFixture)
+        {
+            _envVarFixture = envVarFixture;
+        }
+
+        [Fact(Skip = "desactivated accound")]
+        public async Task AzureUploadTest()
+        {
+            string toCompressFolder = TestFileHelper.NewFolder();
+            var data = await TestData.CreateFiles(1, 1024, toCompressFolder);
+
+            var opt = new CompressOption()
+            {
+                AzBlobUrl = GetEnvVar("AZ_URL"),
+                AzBlobContainer = GetEnvVar("AZ_CONTAINER"),
+                AzBlobSaS = GetEnvVar("AZ_SAS_TOKEN")
+            };
+            opt.UploadModes = new List<UploadMode>() { UploadMode.AzureSdk };
+            var uploader = await opt.GetRemoteStoragesAsync(NullLogger.Instance, CancellationToken.None).ToListAsync();
+
+            var ok = await uploader.First().UploadAsync(data.Files.First(), new DirectoryInfo(toCompressFolder), CancellationToken.None);
+
+            Assert.True(ok.IsSuccess);
+        }
+
+        [Fact]
+        public async Task GoogleUploadTest()
+        {
+            string toCompressFolder = TestFileHelper.NewFolder();
+            var data = await TestData.CreateFiles(1, 1024, toCompressFolder);
+
+            var opt = new CompressOption()
+            {
+                GoogleStorageBucketName = GetEnvVar("GoogleBucket"),
+                GoogleStorageCredential = GetEnvVar("GoogleCredential")
+            };
+
+            opt.UploadModes = new List<UploadMode>() { UploadMode.GoogleCloudStorage };
+            var uploader = await opt.GetRemoteStoragesAsync(NullLogger.Instance, CancellationToken.None).ToListAsync();
+
+            var ok = await uploader.First().UploadAsync(data.Files.First(), new DirectoryInfo(toCompressFolder), CancellationToken.None);
+
+            Assert.True(ok.IsSuccess);
+
+            var gs = uploader.First() as GoogleRemoteStorage;
+            
+            await gs.Storage.DeleteObjectAsync(gs.BucketName, ok.RemoteFilePath);
+        }
+
+        string GetEnvVar(string key)
+        {
+            var s = Environment.GetEnvironmentVariable(key);
+            Assert.True(s != null, key);
+            Assert.True(!string.IsNullOrWhiteSpace(s), key);
+            return s;
+        }
+
+    }
+
+
+    public class EnvVarFixture : IDisposable
+    {
+        public EnvVarFixture()
+        {
+            if (File.Exists("Properties\\launchSettings.json"))
+            {
+                using var file = File.OpenText("Properties\\launchSettings.json");
+                var reader = new JsonTextReader(file);
+                var jObject = JObject.Load(reader);
+
+                var variables = jObject
+                    .GetValue("profiles")
+                    //select a proper profile here
+                    .SelectMany(profiles => profiles.Children())
+                    .SelectMany(profile => profile.Children<JProperty>())
+                    .Where(prop => prop.Name == "environmentVariables")
+                    .SelectMany(prop => prop.Value.Children<JProperty>())
+                    .ToList();
+
+                foreach (var variable in variables)
+                {
+                    Environment.SetEnvironmentVariable(variable.Name, variable.Value.ToString());
+                }
+            }
+        }
+
+        public void Dispose()
+        {
+        }
+    }
+}

From e0345b75e1b17fd46f864e76c5603d11d3b0899a Mon Sep 17 00:00:00 2001
From: Lilian ARAGO <larago@lucca.fr>
Date: Mon, 6 Mar 2023 15:14:10 +0100
Subject: [PATCH 14/16] Clearer code

---
 TCC.Lib/Options/CompressOption.cs       | 59 ++++++++++++-------------
 TCC.Lib/Storage/RemoteStorageFactory.cs |  6 ++-
 2 files changed, 34 insertions(+), 31 deletions(-)

diff --git a/TCC.Lib/Options/CompressOption.cs b/TCC.Lib/Options/CompressOption.cs
index e003fe5..9237e50 100644
--- a/TCC.Lib/Options/CompressOption.cs
+++ b/TCC.Lib/Options/CompressOption.cs
@@ -1,36 +1,35 @@
-using System.Collections.Generic;
-using System.Linq;
-using TCC.Lib.Blocks;
-using TCC.Lib.Database;
-
-namespace TCC.Lib.Options
-{
-    public class CompressOption : TccOption
-    {
-        public BlockMode BlockMode { get; set; }
-        public CompressionAlgo Algo { get; set; }
-        public int CompressionRatio { get; set; }
-        public BackupMode? BackupMode { get; set; }
-        public int? RetryPeriodInSeconds { get; set; }
-        public IEnumerable<string> Filter { get; set; }
-        public IEnumerable<string> Exclude { get; set; }
-        public bool FolderPerDay { get; set; }
-        public int? BoostRatio { get; set; }
-        public int? CleanupTime { get; set; }
-        public string AzBlobUrl { get; set; }
-        public string AzBlobContainer { get; set; }
-        public string AzBlobSaS { get; set; }
-        public int? AzThread { get; set; }
-        public string GoogleStorageBucketName { get; set; }
+using System.Collections.Generic;
+using TCC.Lib.Blocks;
+using TCC.Lib.Database;
+
+namespace TCC.Lib.Options
+{
+    public class CompressOption : TccOption
+    {
+        public BlockMode BlockMode { get; set; }
+        public CompressionAlgo Algo { get; set; }
+        public int CompressionRatio { get; set; }
+        public BackupMode? BackupMode { get; set; }
+        public int? RetryPeriodInSeconds { get; set; }
+        public IEnumerable<string> Filter { get; set; }
+        public IEnumerable<string> Exclude { get; set; }
+        public bool FolderPerDay { get; set; }
+        public int? BoostRatio { get; set; }
+        public int? CleanupTime { get; set; }
+        public string AzBlobUrl { get; set; }
+        public string AzBlobContainer { get; set; }
+        public string AzBlobSaS { get; set; }
+        public int? AzThread { get; set; }
+        public string GoogleStorageBucketName { get; set; }
         public string GoogleStorageCredential { get; set; }
-        public string S3AccessKeyId { get; set; }
-        public string S3SecretAcessKey { get; set; }
-        public string S3Host { get; set; }
+        public string S3AccessKeyId { get; set; }
+        public string S3SecretAcessKey { get; set; }
+        public string S3Host { get; set; }
         public string S3BucketName { get; set; }
         public string S3Region { get; set; }
         public string S3MultipartThreshold { get; set; }
         public string S3MultipartSize { get; set; }
-        public IEnumerable<UploadMode> UploadModes { get; set; } = Enumerable.Empty<UploadMode>();
-        public UploadMode? UploadMode { get; set; }
-    }
+        public List<UploadMode> UploadModes { get; set; } = new ();
+        public UploadMode? UploadMode { get; }
+    }
 }
\ No newline at end of file
diff --git a/TCC.Lib/Storage/RemoteStorageFactory.cs b/TCC.Lib/Storage/RemoteStorageFactory.cs
index 3ed7184..58a1787 100644
--- a/TCC.Lib/Storage/RemoteStorageFactory.cs
+++ b/TCC.Lib/Storage/RemoteStorageFactory.cs
@@ -17,7 +17,11 @@ public static class RemoteStorageFactory
     {
         public static async IAsyncEnumerable<IRemoteStorage> GetRemoteStoragesAsync(this CompressOption option, ILogger logger, [EnumeratorCancellation] CancellationToken token)
         {
-            option.UploadModes = option.UploadModes.Append(option.UploadMode ?? UploadMode.None).Distinct();
+            if (option.UploadMode.HasValue)
+            {
+                option.UploadModes.Add(option.UploadMode.Value);
+            }
+            option.UploadModes = option.UploadModes.Distinct().ToList();
 
             foreach(var mode in option.UploadModes)
             {

From 42b3770668cd3299863fb64e6065975e2debd645 Mon Sep 17 00:00:00 2001
From: Lilian ARAGO <larago@lucca.fr>
Date: Tue, 7 Mar 2023 12:27:02 +0100
Subject: [PATCH 15/16] Implemtented ReadOnlyChunkedStream

---
 TCC.Lib/Options/CompressOption.cs       |  8 +++--
 TCC.Lib/ReadOnlyChunkedStream.cs        | 48 +++++++++++++++++++++++++
 TCC.Lib/Storage/RemoteStorageFactory.cs |  4 +--
 TCC.Lib/Storage/S3RemoteStorage.cs      | 20 ++++++-----
 4 files changed, 66 insertions(+), 14 deletions(-)
 create mode 100644 TCC.Lib/ReadOnlyChunkedStream.cs

diff --git a/TCC.Lib/Options/CompressOption.cs b/TCC.Lib/Options/CompressOption.cs
index 9237e50..6b9590d 100644
--- a/TCC.Lib/Options/CompressOption.cs
+++ b/TCC.Lib/Options/CompressOption.cs
@@ -1,4 +1,6 @@
-using System.Collections.Generic;
+using System;
+using System.Collections.Generic;
+using System.Linq;
 using TCC.Lib.Blocks;
 using TCC.Lib.Database;
 
@@ -29,7 +31,7 @@ public class CompressOption : TccOption
         public string S3Region { get; set; }
         public string S3MultipartThreshold { get; set; }
         public string S3MultipartSize { get; set; }
-        public List<UploadMode> UploadModes { get; set; } = new ();
-        public UploadMode? UploadMode { get; }
+        public IEnumerable<UploadMode> UploadModes { get; set; } = Enumerable.Empty<UploadMode>();
+        public UploadMode? UploadMode { get; set; }
     }
 }
\ No newline at end of file
diff --git a/TCC.Lib/ReadOnlyChunkedStream.cs b/TCC.Lib/ReadOnlyChunkedStream.cs
new file mode 100644
index 0000000..570eab6
--- /dev/null
+++ b/TCC.Lib/ReadOnlyChunkedStream.cs
@@ -0,0 +1,48 @@
+using System;
+using System.IO;
+
+namespace TCC.Lib
+{
+    public class ReadOnlyChunkedStream : Stream
+    {
+        private readonly Stream _inputStream;
+        private readonly int _chunkSize;
+
+        public ReadOnlyChunkedStream(Stream inputStream, int chunkSize)
+        {
+            _inputStream = inputStream;
+            _chunkSize = chunkSize;
+        }
+
+        public override void Flush() => _inputStream.Flush();
+
+        public override int Read(byte[] buffer, int offset, int count)
+        {
+            var maxRead = (int)Math.Min(count, _chunkSize - Position);
+            var nbRead = _inputStream.Read(buffer, offset, maxRead);
+            Position += nbRead;
+            return nbRead;
+        }
+
+        public override long Seek(long offset, SeekOrigin origin)
+        {
+            return _inputStream.Seek(offset, origin);
+        }
+
+        public override void SetLength(long value)
+        {
+            throw new NotImplementedException();
+        }
+
+        public override void Write(byte[] buffer, int offset, int count)
+        {
+            throw new NotImplementedException();
+        }
+
+        public override bool CanRead => _inputStream.CanRead;
+        public override bool CanSeek => _inputStream.CanSeek;
+        public override bool CanWrite => false;
+        public override long Length => Math.Min(_chunkSize, _inputStream.Length - _inputStream.Position);
+        public override long Position { get; set; }
+    }
+}
\ No newline at end of file
diff --git a/TCC.Lib/Storage/RemoteStorageFactory.cs b/TCC.Lib/Storage/RemoteStorageFactory.cs
index 58a1787..5162e1d 100644
--- a/TCC.Lib/Storage/RemoteStorageFactory.cs
+++ b/TCC.Lib/Storage/RemoteStorageFactory.cs
@@ -19,9 +19,9 @@ public static async IAsyncEnumerable<IRemoteStorage> GetRemoteStoragesAsync(this
         {
             if (option.UploadMode.HasValue)
             {
-                option.UploadModes.Add(option.UploadMode.Value);
+                option.UploadModes = option.UploadModes.Append(option.UploadMode.Value);
             }
-            option.UploadModes = option.UploadModes.Distinct().ToList();
+            option.UploadModes = option.UploadModes.Distinct();
 
             foreach(var mode in option.UploadModes)
             {
diff --git a/TCC.Lib/Storage/S3RemoteStorage.cs b/TCC.Lib/Storage/S3RemoteStorage.cs
index b377c51..5908278 100644
--- a/TCC.Lib/Storage/S3RemoteStorage.cs
+++ b/TCC.Lib/Storage/S3RemoteStorage.cs
@@ -28,7 +28,7 @@ public async Task<UploadResponse> UploadAsync(string targetPath, Stream data, Ca
         {
             try
             {
-                if (_multipartTreshold != 0  && data.Length > _multipartTreshold)
+                if (_multipartTreshold > 0 && data.Length > _multipartTreshold)
                 {
                     await UploadStreamToMultipartsAsync(targetPath, data, token);
                 } else
@@ -59,7 +59,6 @@ await _s3Client.PutObjectAsync(new ()
 
         private async Task UploadStreamToMultipartsAsync(string targetPath, Stream data, CancellationToken token)
         {
-            
             var multipartUpload = await _s3Client.InitiateMultipartUploadAsync(new ()
             {
                 BucketName = BucketName,
@@ -68,9 +67,16 @@ private async Task UploadStreamToMultipartsAsync(string targetPath, Stream data,
             var partsETags = new List<PartETag>();
             var partNumber = 1;
 
-            await foreach (var chunk in ChunkStreamAsync(data, token))
+            
+            while (true)
             {
-                chunk.Position = 0;
+                await using var chunk = new ReadOnlyChunkedStream(data, _partSize);
+                
+                if (!chunk.CanRead || chunk.Length == 0)
+                {
+                    break;
+                }
+
                 var partUpload = await _s3Client.UploadPartAsync(new()
                 {
                     BucketName = BucketName,
@@ -80,11 +86,7 @@ private async Task UploadStreamToMultipartsAsync(string targetPath, Stream data,
                     InputStream = chunk,
                 }, token);
 
-                partsETags.Add(new()
-                {
-                    ETag = partUpload.ETag,
-                    PartNumber = partUpload.PartNumber,
-                });
+                partsETags.Add(new() {ETag = partUpload.ETag, PartNumber = partUpload.PartNumber,});
             }
             await _s3Client.CompleteMultipartUploadAsync(new()
             {

From c85ed278ff452b9ac0bc50598e6c4e7f0e9ed4b5 Mon Sep 17 00:00:00 2001
From: Lilian ARAGO <larago@lucca.fr>
Date: Tue, 7 Mar 2023 12:36:57 +0100
Subject: [PATCH 16/16] Fixed code smells

---
 TCC.Lib/Helpers/StringExtensions.cs | 203 ++++++++++++++--------------
 TCC.Lib/Options/CompressOption.cs   |   3 +-
 TCC.Lib/ReadOnlyChunkedStream.cs    |   4 +-
 TCC.Lib/Storage/S3RemoteStorage.cs  |  16 ---
 4 files changed, 105 insertions(+), 121 deletions(-)

diff --git a/TCC.Lib/Helpers/StringExtensions.cs b/TCC.Lib/Helpers/StringExtensions.cs
index 24f42c6..5bd238b 100644
--- a/TCC.Lib/Helpers/StringExtensions.cs
+++ b/TCC.Lib/Helpers/StringExtensions.cs
@@ -1,44 +1,45 @@
-using System;
-using System.Globalization;
-using System.IO;
+using System;
+using System.Globalization;
+using System.IO;
 using System.Linq;
 
-namespace TCC.Lib.Helpers
-{
-    public static class StringExtensions
-    {
-        public static string Escape(this string str)
-        {
-            return '"' + str.Trim('"') + '"';
-        }
-
-        public static string HumanizedBandwidth(this double bandwidth, int decimals = 2)
-        {
-            var ordinals = new[] { "", "K", "M", "G", "T", "P", "E" };
-            var rate = (decimal)bandwidth;
-            var ordinal = 0;
-            while (rate > 1024)
-            {
-                rate /= 1024;
-                ordinal++;
-            }
-            return String.Format("{0:n" + decimals + "} {1}b/s", Math.Round(rate, decimals, MidpointRounding.AwayFromZero), ordinals[ordinal]);
-        }
-
-        public static String HumanizeSize(this long size)
-        {
-            string[] suf = { "B", "Ko", "Mo", "Go", "To", "Po", "Eo" };
-            if (size == 0)
-                return "0" + suf[0];
-            long bytes = Math.Abs(size);
-            int place = Convert.ToInt32(Math.Floor(Math.Log(bytes, 1024)));
-            double num = Math.Round(bytes / Math.Pow(1024, place), 1);
-            return (Math.Sign(size) * num).ToString(CultureInfo.InvariantCulture) + " " + suf[place];
+namespace TCC.Lib.Helpers
+{
+    public static class StringExtensions
+    {
+        public static string Escape(this string str)
+        {
+            return '"' + str.Trim('"') + '"';
+        }
+
+        public static string HumanizedBandwidth(this double bandwidth, int decimals = 2)
+        {
+            var ordinals = new[] { "", "K", "M", "G", "T", "P", "E" };
+            var rate = (decimal)bandwidth;
+            var ordinal = 0;
+            while (rate > 1024)
+            {
+                rate /= 1024;
+                ordinal++;
+            }
+            return String.Format("{0:n" + decimals + "} {1}b/s", Math.Round(rate, decimals, MidpointRounding.AwayFromZero), ordinals[ordinal]);
+        }
+
+        public static String HumanizeSize(this long size)
+        {
+            string[] suf = { "B", "Ko", "Mo", "Go", "To", "Po", "Eo" };
+            if (size == 0)
+                return "0" + suf[0];
+            long bytes = Math.Abs(size);
+            int place = Convert.ToInt32(Math.Floor(Math.Log(bytes, 1024)));
+            double num = Math.Round(bytes / Math.Pow(1024, place), 1);
+            return (Math.Sign(size) * num).ToString(CultureInfo.InvariantCulture) + " " + suf[place];
         }
 
         public static long ParseSize(this string humanizedSize)
         {
-            if (string.IsNullOrWhiteSpace(humanizedSize)) return -1;
+            if (string.IsNullOrWhiteSpace(humanizedSize))
+                return -1;
             string[] suf = { "b", "ko", "mo", "go", "to", "po", "eo" };
             var size = humanizedSize.Trim().ToLower(CultureInfo.InvariantCulture);
             var number = string.Join("", size.Where(char.IsDigit));
@@ -50,70 +51,70 @@ public static long ParseSize(this string humanizedSize)
                 -1 => long.Parse(number, CultureInfo.InvariantCulture),
                 _ => long.Parse(number, CultureInfo.InvariantCulture) * (long)Math.Pow(1024L, pow)
             };
-        }
-
-        public static string HumanizedTimeSpan(this TimeSpan t, int parts = 2)
-        {
-            string result = string.Empty;
-            if (t.TotalDays >= 1 && parts > 0)
-            {
-                result += $"{t.Days}d ";
-                parts--;
-            }
-            if (t.TotalHours >= 1 && parts > 0)
-            {
-                result += $"{t.Hours}h ";
-                parts--;
-            }
-            if (t.TotalMinutes >= 1 && parts > 0)
-            {
-                result += $"{t.Minutes}m ";
-                parts--;
-            }
-            if (t.Seconds >= 1 && parts > 0)
-            {
-                result += $"{t.Seconds}s ";
-                parts--;
-            }
-            if (t.Milliseconds >= 1 && parts > 0)
-            {
-                result += $"{t.Milliseconds}ms";
-            }
-            return result.TrimEnd();
-        }
-
-        public static string Pad(this string source, int length)
-        {
-            if (source == null)
-            {
-                return new string(' ', length);
-            }
-            return source.Length > length ? source.Substring(0, length) : source.PadLeft(length, ' ');
-        }
-
-        public static (string Name, DateTime? Date) ExtractArchiveNameAndDate(this string filePath)
-        {
-            if (filePath == null)
-                throw new ArgumentNullException(nameof(filePath));
-
-            string segment = Path.GetFileNameWithoutExtension(filePath);
-            if (segment.EndsWith(".diff") || segment.EndsWith(".full"))
-            {
-                segment = segment.Substring(0, segment.Length - 5);
-            }
-
-            var lastSegment = segment.LastIndexOf('_');
-            if (lastSegment > 0)
-            {
-                string name = segment.Substring(0, lastSegment);
-                string date = segment.Substring(lastSegment + 1);
-                if (date.TryParseArchiveDateTime(out var dt))
-                {
-                    return (name, dt);
-                }
-                return (name, null);
-            }
-            return (segment, null);
-        }
-    }
+        }
+
+        public static string HumanizedTimeSpan(this TimeSpan t, int parts = 2)
+        {
+            string result = string.Empty;
+            if (t.TotalDays >= 1 && parts > 0)
+            {
+                result += $"{t.Days}d ";
+                parts--;
+            }
+            if (t.TotalHours >= 1 && parts > 0)
+            {
+                result += $"{t.Hours}h ";
+                parts--;
+            }
+            if (t.TotalMinutes >= 1 && parts > 0)
+            {
+                result += $"{t.Minutes}m ";
+                parts--;
+            }
+            if (t.Seconds >= 1 && parts > 0)
+            {
+                result += $"{t.Seconds}s ";
+                parts--;
+            }
+            if (t.Milliseconds >= 1 && parts > 0)
+            {
+                result += $"{t.Milliseconds}ms";
+            }
+            return result.TrimEnd();
+        }
+
+        public static string Pad(this string source, int length)
+        {
+            if (source == null)
+            {
+                return new string(' ', length);
+            }
+            return source.Length > length ? source.Substring(0, length) : source.PadLeft(length, ' ');
+        }
+
+        public static (string Name, DateTime? Date) ExtractArchiveNameAndDate(this string filePath)
+        {
+            if (filePath == null)
+                throw new ArgumentNullException(nameof(filePath));
+
+            string segment = Path.GetFileNameWithoutExtension(filePath);
+            if (segment.EndsWith(".diff") || segment.EndsWith(".full"))
+            {
+                segment = segment.Substring(0, segment.Length - 5);
+            }
+
+            var lastSegment = segment.LastIndexOf('_');
+            if (lastSegment > 0)
+            {
+                string name = segment.Substring(0, lastSegment);
+                string date = segment.Substring(lastSegment + 1);
+                if (date.TryParseArchiveDateTime(out var dt))
+                {
+                    return (name, dt);
+                }
+                return (name, null);
+            }
+            return (segment, null);
+        }
+    }
 }
\ No newline at end of file
diff --git a/TCC.Lib/Options/CompressOption.cs b/TCC.Lib/Options/CompressOption.cs
index 6b9590d..c51c58f 100644
--- a/TCC.Lib/Options/CompressOption.cs
+++ b/TCC.Lib/Options/CompressOption.cs
@@ -1,5 +1,4 @@
-using System;
-using System.Collections.Generic;
+using System.Collections.Generic;
 using System.Linq;
 using TCC.Lib.Blocks;
 using TCC.Lib.Database;
diff --git a/TCC.Lib/ReadOnlyChunkedStream.cs b/TCC.Lib/ReadOnlyChunkedStream.cs
index 570eab6..9adfb4c 100644
--- a/TCC.Lib/ReadOnlyChunkedStream.cs
+++ b/TCC.Lib/ReadOnlyChunkedStream.cs
@@ -31,12 +31,12 @@ public override long Seek(long offset, SeekOrigin origin)
 
         public override void SetLength(long value)
         {
-            throw new NotImplementedException();
+            throw new NotSupportedException();
         }
 
         public override void Write(byte[] buffer, int offset, int count)
         {
-            throw new NotImplementedException();
+            throw new NotSupportedException();
         }
 
         public override bool CanRead => _inputStream.CanRead;
diff --git a/TCC.Lib/Storage/S3RemoteStorage.cs b/TCC.Lib/Storage/S3RemoteStorage.cs
index 5908278..89b2396 100644
--- a/TCC.Lib/Storage/S3RemoteStorage.cs
+++ b/TCC.Lib/Storage/S3RemoteStorage.cs
@@ -96,22 +96,6 @@ await _s3Client.CompleteMultipartUploadAsync(new()
                 PartETags = partsETags
             }, token);
         }
-        private async IAsyncEnumerable<Stream> ChunkStreamAsync(Stream data, [EnumeratorCancellation] CancellationToken token)
-        {
-            var buffer = new byte[_partSize];
-            int readBytes;
-
-            do
-            {
-                readBytes = await data.ReadAsync(buffer.AsMemory(0, _partSize), token);
-
-                var notTheSame = new MemoryStream(readBytes);
-                notTheSame.Write(buffer, 0, readBytes);
-                
-                yield return notTheSame;
-            } while (readBytes >= _partSize);
-        }
-
         public UploadMode Mode => UploadMode.S3;
     }
 }
\ No newline at end of file