Merge multy read 210417

Common/Algorithm.cs

@@ -0,0 +1,48 @@
+using System;
+using System.Text;
+
+namespace Common
+{
+    public static class Algorithm
+    {
+        public static readonly byte[] EolUtf8Bytes = Encoding.UTF8.GetBytes(Environment.NewLine);
+
+        /// <summary>
+        /// Finds subsequence of bytes inside another byte array
+        /// </summary>
+        /// <param name="buffer"></param>
+        /// <param name="searchedBytes"></param>
+        /// <returns>Offset of the subsequence or -1 of the subsequence was not found</returns>
+        public static int FindByteSubsequence(byte[] buffer, byte[] searchedBytes)
+        {
+            for (var i = 0; i < buffer.Length - searchedBytes.Length; ++i)
+            {
+                var match = false;
+                for (var j = 0; j < searchedBytes.Length; ++j)
+                {
+                    if (buffer[i + j] == searchedBytes[j])
+                    {
+                        match = true;
+                    }
+                    else
+                    {
+                        match = false;
+                        break;
+                    }
+                }
+
+                if (match)
+                {
+                    return i;
+                }
+            }
+
+            return -1;
+        }
+
+        public static int FindEolOffset(byte[] buffer)
+        {
+            return FindByteSubsequence(buffer, EolUtf8Bytes);
+        }
+    }
+}

Common/Common.csproj

@@ -41,9 +41,11 @@
     <Reference Include="System.Xml" />
   </ItemGroup>
   <ItemGroup>
+    <Compile Include="Algorithm.cs" />
     <Compile Include="AutoStopwatch.cs" />
     <Compile Include="BytesFormater.cs" />
     <Compile Include="Constants.cs" />
+    <Compile Include="OffsetLength.cs" />
     <Compile Include="Properties\AssemblyInfo.cs" />
   </ItemGroup>
   <Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />

Common/Constants.cs

@@ -2,8 +2,8 @@
 {
     public static class Constants
     {
-        public const uint Mb = 1024 * 1024;
+        public const uint MB = 1024 * 1024;
 
-        public const uint Gb = 1024 * Mb;
+        public const uint GB = 1024 * MB;
     }
 }

Common/OffsetLength.cs

@@ -0,0 +1,9 @@
+namespace Common
+{
+    public class OffsetLength<T>
+    {
+        public T Offset { get; set; }
+
+        public T Length { get; set; }
+    }
+}

DataGenerator/Program.cs

@@ -13,7 +13,7 @@ namespace DataGenerator
 {
     public class Program
     {
-        private const ulong DafaultSize = Constants.Mb;
+        private const ulong DafaultSize = Constants.MB;
 
         private const int OutFileBuffer = 128 * 1024;
         private const int ChunkSize = 128;
@@ -47,11 +47,11 @@ static void Main(string[] args)
                     var lowArg = args[0].ToLowerInvariant();
                     if (lowArg.EndsWith("mb"))
                     {
-                        outSize = (ulong)size * Constants.Mb;
+                        outSize = (ulong)size * Constants.MB;
                     }
                     else if (lowArg.EndsWith("gb"))
                     {
-                        outSize = (ulong)size * Constants.Gb;
+                        outSize = (ulong)size * Constants.GB;
                     }
                     else
                     {

ExternalSort.Tests/Generic.cs

@@ -83,5 +83,18 @@ public void TestBadLinesCopmparer()
             Assert.AreEqual("3. Apple", lines.First());
             Assert.AreEqual("End!", lines.Last());
         }
+
+        [TestMethod]
+        public void TestFindSubByte()
+        {
+            var hello = "Hello";
+            var input = hello + Environment.NewLine + "!";
+
+            var endlineBytes = Encoding.UTF8.GetBytes(Environment.NewLine);
+            var inputBytes = Encoding.UTF8.GetBytes(input);
+
+            var found = Algorithm.FindByteSubsequence(inputBytes, endlineBytes);
+            Assert.AreEqual(hello.Length, found);
+        }
     }
 }

ExternalSort/App.config

@@ -5,5 +5,6 @@
     </startup>
   <appSettings>
     <add key="DeflateTemp" value="true"/>
+    <add key="MaxMemoryUsageBytes" value="1052672"/>
   </appSettings>
 </configuration>

ExternalSort/LineComparer.cs

@@ -7,7 +7,7 @@ namespace ExternalSort
     public class LineComparer : IComparer<string>
     {
         /// <summary>
-        /// This method tries hart to follow the spec. rules but nevertheless it never fails.
+        /// This method tries hard to follow the specs but nevertheless it never fails.
         /// </summary>
         /// <param name="x"></param>
         /// <param name="y"></param>

ExternalSort/Program.cs

@@ -39,14 +39,20 @@ static void Main(string[] args)
 
             var appSettings = ConfigurationManager.AppSettings;
             var deflate = appSettings["DeflateTemp"];
+            var maxMem = appSettings["MaxMemoryUsageBytes"];
+            var st = new Settings
+            {
+                OrdinalStringSortOrder = option.StartsWith("/ord"),
+                DeflateTempFiles = deflate == "true",
+            };
 
-            var ms = new MergeSort(
-                new Settings
-                {
-                    OrdinalStringSortOrder = option.StartsWith("/ord"),
-                    DeflateTempFiles = deflate == "true",
-                });
+            ulong mmem;
+            if (ulong.TryParse(maxMem, out mmem))
+            {
+                st.MaxMemoryUsageBytes = mmem;
+            }
 
+            var ms = new MergeSort(st);
             ms.MergeSortFile(imputFile, outputFile).Wait();
         }
     }

ExternalSort/Settings.cs

@@ -1,13 +1,14 @@
 using System;
+using Common;
 
 namespace ExternalSort
 {
     public class Settings
     {
         public Settings()
         {
-            //512 MB
-            MaxMemoryUsageBytes = 1024 * 1024 * 1024;
+            //2 GB will work for 32 bit systems!
+            MaxMemoryUsageBytes = 2 * Constants.GB;
 
             // Safe value
             MaxQueueRecords = 1000;