S3 storage implementation

.gitignore

@@ -1,12 +1,13 @@
-.vs/*
-*/obj/*
-*/bin/*
-packages 
-*.user
-/coverage-opencover.xml
-launchSettings.json
-*/logs/*
-*.db
-/TCC.Tests/'coverage.xml'
-/coverage.xml
-/published
+.vs/*
+*/obj/*
+*/bin/*
+packages 
+*.user
+/coverage-opencover.xml
+launchSettings.json
+*/logs/*
+*.db
+/TCC.Tests/'coverage.xml'
+/coverage.xml
+/published
+.idea

TCC.Lib/Helpers/StringExtensions.cs

@@ -1,102 +1,120 @@
-using System;
-using System.Globalization;
-using System.IO;
-
-namespace TCC.Lib.Helpers
-{
-    public static class StringExtensions
-    {
-        public static string Escape(this string str)
-        {
-            return '"' + str.Trim('"') + '"';
-        }
-
-        public static string HumanizedBandwidth(this double bandwidth, int decimals = 2)
-        {
-            var ordinals = new[] { "", "K", "M", "G", "T", "P", "E" };
-            var rate = (decimal)bandwidth;
-            var ordinal = 0;
-            while (rate > 1024)
-            {
-                rate /= 1024;
-                ordinal++;
-            }
-            return String.Format("{0:n" + decimals + "} {1}b/s", Math.Round(rate, decimals, MidpointRounding.AwayFromZero), ordinals[ordinal]);
-        }
-
-        public static String HumanizeSize(this long size)
-        {
-            string[] suf = { "B", "Ko", "Mo", "Go", "To", "Po", "Eo" };
-            if (size == 0)
-                return "0" + suf[0];
-            long bytes = Math.Abs(size);
-            int place = Convert.ToInt32(Math.Floor(Math.Log(bytes, 1024)));
-            double num = Math.Round(bytes / Math.Pow(1024, place), 1);
-            return (Math.Sign(size) * num).ToString(CultureInfo.InvariantCulture) + " " + suf[place];
-        }
-
-        public static string HumanizedTimeSpan(this TimeSpan t, int parts = 2)
-        {
-            string result = string.Empty;
-            if (t.TotalDays >= 1 && parts > 0)
-            {
-                result += $"{t.Days}d ";
-                parts--;
-            }
-            if (t.TotalHours >= 1 && parts > 0)
-            {
-                result += $"{t.Hours}h ";
-                parts--;
-            }
-            if (t.TotalMinutes >= 1 && parts > 0)
-            {
-                result += $"{t.Minutes}m ";
-                parts--;
-            }
-            if (t.Seconds >= 1 && parts > 0)
-            {
-                result += $"{t.Seconds}s ";
-                parts--;
-            }
-            if (t.Milliseconds >= 1 && parts > 0)
-            {
-                result += $"{t.Milliseconds}ms";
-            }
-            return result.TrimEnd();
-        }
-
-        public static string Pad(this string source, int length)
-        {
-            if (source == null)
-            {
-                return new string(' ', length);
-            }
-            return source.Length > length ? source.Substring(0, length) : source.PadLeft(length, ' ');
-        }
-
-        public static (string Name, DateTime? Date) ExtractArchiveNameAndDate(this string filePath)
-        {
-            if (filePath == null)
-                throw new ArgumentNullException(nameof(filePath));
-
-            string segment = Path.GetFileNameWithoutExtension(filePath);
-            if (segment.EndsWith(".diff") || segment.EndsWith(".full"))
-            {
-                segment = segment.Substring(0, segment.Length - 5);
-            }
-
-            var lastSegment = segment.LastIndexOf('_');
-            if (lastSegment > 0)
-            {
-                string name = segment.Substring(0, lastSegment);
-                string date = segment.Substring(lastSegment + 1);
-                if (date.TryParseArchiveDateTime(out var dt))
-                {
-                    return (name, dt);
-                }
-                return (name, null);
-            }
-            return (segment, null);
-        }
-    }
+using System;
+using System.Globalization;
+using System.IO;
+using System.Linq;
+
+namespace TCC.Lib.Helpers
+{
+    public static class StringExtensions
+    {
+        public static string Escape(this string str)
+        {
+            return '"' + str.Trim('"') + '"';
+        }
+
+        public static string HumanizedBandwidth(this double bandwidth, int decimals = 2)
+        {
+            var ordinals = new[] { "", "K", "M", "G", "T", "P", "E" };
+            var rate = (decimal)bandwidth;
+            var ordinal = 0;
+            while (rate > 1024)
+            {
+                rate /= 1024;
+                ordinal++;
+            }
+            return String.Format("{0:n" + decimals + "} {1}b/s", Math.Round(rate, decimals, MidpointRounding.AwayFromZero), ordinals[ordinal]);
+        }
+
+        public static String HumanizeSize(this long size)
+        {
+            string[] suf = { "B", "Ko", "Mo", "Go", "To", "Po", "Eo" };
+            if (size == 0)
+                return "0" + suf[0];
+            long bytes = Math.Abs(size);
+            int place = Convert.ToInt32(Math.Floor(Math.Log(bytes, 1024)));
+            double num = Math.Round(bytes / Math.Pow(1024, place), 1);
+            return (Math.Sign(size) * num).ToString(CultureInfo.InvariantCulture) + " " + suf[place];
+        }
+
+        public static long ParseSize(this string humanizedSize)
+        {
+            if (string.IsNullOrWhiteSpace(humanizedSize))
+                return -1;
+            string[] suf = { "b", "ko", "mo", "go", "to", "po", "eo" };
+            var size = humanizedSize.Trim().ToLower(CultureInfo.InvariantCulture);
+            var number = string.Join("", size.Where(char.IsDigit));
+            var unit = size.Substring(size.Length - 2);
+            var pow = Array.IndexOf(suf, unit);
+
+            return pow switch
+            {
+                -1 => long.Parse(number, CultureInfo.InvariantCulture),
+                _ => long.Parse(number, CultureInfo.InvariantCulture) * (long)Math.Pow(1024L, pow)
+            };
+        }
+
+        public static string HumanizedTimeSpan(this TimeSpan t, int parts = 2)
+        {
+            string result = string.Empty;
+            if (t.TotalDays >= 1 && parts > 0)
+            {
+                result += $"{t.Days}d ";
+                parts--;
+            }
+            if (t.TotalHours >= 1 && parts > 0)
+            {
+                result += $"{t.Hours}h ";
+                parts--;
+            }
+            if (t.TotalMinutes >= 1 && parts > 0)
+            {
+                result += $"{t.Minutes}m ";
+                parts--;
+            }
+            if (t.Seconds >= 1 && parts > 0)
+            {
+                result += $"{t.Seconds}s ";
+                parts--;
+            }
+            if (t.Milliseconds >= 1 && parts > 0)
+            {
+                result += $"{t.Milliseconds}ms";
+            }
+            return result.TrimEnd();
+        }
+
+        public static string Pad(this string source, int length)
+        {
+            if (source == null)
+            {
+                return new string(' ', length);
+            }
+            return source.Length > length ? source.Substring(0, length) : source.PadLeft(length, ' ');
+        }
+
+        public static (string Name, DateTime? Date) ExtractArchiveNameAndDate(this string filePath)
+        {
+            if (filePath == null)
+                throw new ArgumentNullException(nameof(filePath));
+
+            string segment = Path.GetFileNameWithoutExtension(filePath);
+            if (segment.EndsWith(".diff") || segment.EndsWith(".full"))
+            {
+                segment = segment.Substring(0, segment.Length - 5);
+            }
+
+            var lastSegment = segment.LastIndexOf('_');
+            if (lastSegment > 0)
+            {
+                string name = segment.Substring(0, lastSegment);
+                string date = segment.Substring(lastSegment + 1);
+                if (date.TryParseArchiveDateTime(out var dt))
+                {
+                    return (name, dt);
+                }
+                return (name, null);
+            }
+            return (segment, null);
+        }
+    }
 }

TCC.Lib/OperationBlock.cs

@@ -236,6 +236,7 @@ public class StepResult
         public string Errors { get; set; }
         public string Warning { get; set; }
         public string Infos { get; set; }
+        public UploadMode? UploadMode { get; set; }
         public bool IsSuccess => !HasError && !HasWarning;
         public bool HasError => !string.IsNullOrWhiteSpace(Errors);
         public bool HasWarning => !string.IsNullOrWhiteSpace(Warning);

TCC.Lib/Options/CompressOption.cs

@@ -1,27 +1,36 @@
-using System.Collections.Generic;
-using TCC.Lib.Blocks;
-using TCC.Lib.Database;
-
-namespace TCC.Lib.Options
-{
-    public class CompressOption : TccOption
-    {
-        public BlockMode BlockMode { get; set; }
-        public CompressionAlgo Algo { get; set; }
-        public int CompressionRatio { get; set; }
-        public BackupMode? BackupMode { get; set; }
-        public int? RetryPeriodInSeconds { get; set; }
-        public IEnumerable<string> Filter { get; set; }
-        public IEnumerable<string> Exclude { get; set; }
-        public bool FolderPerDay { get; set; }
-        public int? BoostRatio { get; set; }
-        public int? CleanupTime { get; set; }
-        public string AzBlobUrl { get; set; }
-        public string AzBlobContainer { get; set; }
-        public string AzBlobSaS { get; set; }
-        public int? AzThread { get; set; }
-        public string GoogleStorageBucketName { get; set; }
-        public string GoogleStorageCredential { get; set; }
-        public UploadMode? UploadMode { get; set; }
-    }
+using System.Collections.Generic;
+using System.Linq;
+using TCC.Lib.Blocks;
+using TCC.Lib.Database;
+
+namespace TCC.Lib.Options
+{
+    public class CompressOption : TccOption
+    {
+        public BlockMode BlockMode { get; set; }
+        public CompressionAlgo Algo { get; set; }
+        public int CompressionRatio { get; set; }
+        public BackupMode? BackupMode { get; set; }
+        public int? RetryPeriodInSeconds { get; set; }
+        public IEnumerable<string> Filter { get; set; }
+        public IEnumerable<string> Exclude { get; set; }
+        public bool FolderPerDay { get; set; }
+        public int? BoostRatio { get; set; }
+        public int? CleanupTime { get; set; }
+        public string AzBlobUrl { get; set; }
+        public string AzBlobContainer { get; set; }
+        public string AzBlobSaS { get; set; }
+        public int? AzThread { get; set; }
+        public string GoogleStorageBucketName { get; set; }
+        public string GoogleStorageCredential { get; set; }
+        public string S3AccessKeyId { get; set; }
+        public string S3SecretAcessKey { get; set; }
+        public string S3Host { get; set; }
+        public string S3BucketName { get; set; }
+        public string S3Region { get; set; }
+        public string S3MultipartThreshold { get; set; }
+        public string S3MultipartSize { get; set; }
+        public IEnumerable<UploadMode> UploadModes { get; set; } = Enumerable.Empty<UploadMode>();
+        public UploadMode? UploadMode { get; set; }
+    }
 }

TCC.Lib/ReadOnlyChunkedStream.cs

@@ -0,0 +1,48 @@
+using System;
+using System.IO;
+
+namespace TCC.Lib
+{
+    public class ReadOnlyChunkedStream : Stream
+    {
+        private readonly Stream _inputStream;
+        private readonly int _chunkSize;
+
+        public ReadOnlyChunkedStream(Stream inputStream, int chunkSize)
+        {
+            _inputStream = inputStream;
+            _chunkSize = chunkSize;
+        }
+
+        public override void Flush() => _inputStream.Flush();
+
+        public override int Read(byte[] buffer, int offset, int count)
+        {
+            var maxRead = (int)Math.Min(count, _chunkSize - Position);
+            var nbRead = _inputStream.Read(buffer, offset, maxRead);
+            Position += nbRead;
+            return nbRead;
+        }
+
+        public override long Seek(long offset, SeekOrigin origin)
+        {
+            return _inputStream.Seek(offset, origin);
+        }
+
+        public override void SetLength(long value)
+        {
+            throw new NotSupportedException();
+        }
+
+        public override void Write(byte[] buffer, int offset, int count)
+        {
+            throw new NotSupportedException();
+        }
+
+        public override bool CanRead => _inputStream.CanRead;
+        public override bool CanSeek => _inputStream.CanSeek;
+        public override bool CanWrite => false;
+        public override long Length => Math.Min(_chunkSize, _inputStream.Length - _inputStream.Position);
+        public override long Position { get; set; }
+    }
+}

TCC.Lib/Storage/AzureRemoteStorage.cs

@@ -24,6 +24,9 @@ public async Task<UploadResponse> UploadAsync(string targetPath, Stream data, Ca
                 ErrorMessage = response.ReasonPhrase,
                 RemoteFilePath = targetPath
             };
-        }
+        }
+
+        public UploadMode Mode => UploadMode.AzureSdk;
+
     }
 }

TCC.Lib/Storage/GoogleRemoteStorage.cs

@@ -38,6 +38,8 @@ public async Task<UploadResponse> UploadAsync(string targetPath, Stream data, Ca
                 IsSuccess = true,
                 RemoteFilePath = targetPath
             };
-        }
+        }
+
+        public UploadMode Mode => UploadMode.GoogleCloudStorage;
     }
 }