Merge pull request #130 from github-for-unity/fixes/break-up-git-add

Fix to handle large git add operations

Author: StanleyGoldman

src/GitHub.Api/Extensions/ListExtensions.cs

@@ -0,0 +1,44 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+
+namespace GitHub.Unity
+{
+    public static class ListExtensions
+    {
+        public static string Join<T>(this IEnumerable<T> list, string separator)
+        {
+            if (list == null)
+                return null;
+            return String.Join(separator, list.Select(x => x?.ToString()).ToArray());
+        }
+        
+        public static IEnumerable<IList<string>> Spool<T>(this IEnumerable<T> items, int spoolLength)
+        {
+            var currentSpoolLength = 0;
+            var currentList= new List<string>();
+
+            foreach (var item in items)
+            {
+                var itemValue = item.ToString();
+                var itemValueLength = itemValue.Length;
+
+                if (currentSpoolLength + itemValueLength > spoolLength)
+                {
+                    yield return currentList;
+
+                    currentSpoolLength = 0;
+                    currentList = new List<string>();
+                }
+
+                currentSpoolLength += itemValueLength;
+                currentList.Add(itemValue);
+            }
+
+            if (currentList.Any())
+            {
+                yield return currentList;
+            }
+        }
+    }
+}

src/GitHub.Api/Extensions/StringExtensions.cs

@@ -151,16 +151,6 @@ public static string LeftBeforeLast(this string s, char search)
         }
     }
 
-    public static class ListExtensions
-    {
-        public static string Join<T>(this IEnumerable<T> list, string separator)
-        {
-            if (list == null)
-                return null;
-            return String.Join(separator, list.Select(x => x?.ToString()).ToArray());
-        }
-    }
-
     public struct StringResult
     {
         public string Chunk;

src/GitHub.Api/Git/GitClient.cs

@@ -62,6 +62,8 @@ ITask<string> Commit(string message, string body,
         ITask<string> Add(IList<string> files,
             IOutputProcessor<string> processor = null);
 
+        ITask<string> AddAll(IOutputProcessor<string> processor = null);
+
         ITask<string> Remove(IList<string> files,
             IOutputProcessor<string> processor = null);
 
@@ -296,13 +298,35 @@ public ITask<string> Commit(string message, string body,
                 .Configure(processManager);
         }
 
+        public ITask<string> AddAll(IOutputProcessor<string> processor = null)
+        {
+            Logger.Trace("Add all files");
+
+            return new GitAddTask(cancellationToken, processor)
+                .Configure(processManager);
+        }
+
         public ITask<string> Add(IList<string> files,
             IOutputProcessor<string> processor = null)
         {
-            Logger.Trace("Add");
+            Logger.Trace("Add Files");
 
-            return new GitAddTask(files, cancellationToken, processor)
-                .Configure(processManager);
+            GitAddTask last = null;
+            foreach (var batch in files.Spool(5000))
+            {
+                var current = new GitAddTask(batch, cancellationToken, processor).Configure(processManager);
+                if (last == null)
+                {
+                    last = current;
+                }
+                else
+                {
+                    last.Then(current);
+                    last = current;
+                }
+            }
+
+            return last;
         }
 
         public ITask<string> Remove(IList<string> files,

src/GitHub.Api/Git/Tasks/GitAddTask.cs

@@ -1,3 +1,4 @@
+using System;
 using System.Collections.Generic;
 using System.Threading;
 
@@ -8,9 +9,8 @@ class GitAddTask : ProcessTask<string>
         private const string TaskName = "git add";
         private readonly string arguments;
 
-        public GitAddTask(IEnumerable<string> files,
-            CancellationToken token, IOutputProcessor<string> processor = null)
-            : base(token, processor ?? new SimpleOutputProcessor())
+        public GitAddTask(IEnumerable<string> files, CancellationToken token, 
+            IOutputProcessor<string> processor = null) : base(token, processor ?? new SimpleOutputProcessor())
         {
             Guard.ArgumentNotNull(files, "files");
             Name = TaskName;
@@ -24,6 +24,12 @@ public GitAddTask(IEnumerable<string> files,
             }
         }
 
+        public GitAddTask(CancellationToken token,
+            IOutputProcessor<string> processor = null) : base(token, processor ?? new SimpleOutputProcessor())
+        {
+            arguments = "add -A";
+        }
+
         public override string ProcessArguments { get { return arguments; } }
         public override TaskAffinity Affinity { get { return TaskAffinity.Exclusive; } }
     }

src/GitHub.Api/GitHub.Api.csproj

@@ -96,6 +96,7 @@
     <Compile Include="Application\ApplicationInfo.cs" />
     <Compile Include="Application\LoginResult.cs" />
     <Compile Include="Application\AppConfiguration.cs" />
+    <Compile Include="Extensions\ListExtensions.cs" />
     <Compile Include="Helpers\AssemblyResources.cs" />
     <Compile Include="Authentication\IKeychain.cs" />
     <Compile Include="Authentication\Keychain.cs" />

src/UnityExtension/Assets/Editor/GitHub.Unity/UI/ChangesView.cs

@@ -189,9 +189,23 @@ private void Commit()
             // Do not allow new commits before we have received one successful update
             busy = true;
 
-            var files = Enumerable.Range(0, tree.Entries.Count).Where(i => tree.CommitTargets[i].All).Select(i => tree.Entries[i].Path).ToArray();
+            var files = Enumerable.Range(0, tree.Entries.Count)
+                .Where(i => tree.CommitTargets[i].All)
+                .Select(i => tree.Entries[i].Path)
+                .ToArray();
 
-            GitClient.Add(files)
+            ITask<string> addTask;
+
+            if (files.Length == tree.Entries.Count)
+            {
+                addTask = GitClient.AddAll();
+            }
+            else
+            {
+                addTask = GitClient.Add(files);
+            }
+
+            addTask
                 .Then(GitClient.Commit(commitMessage, commitBody))
                 .Then(GitClient.Status())
                 .FinallyInUI((b, exception) => 

src/tests/UnitTests/Extensions/ListExtensionTests.cs

@@ -0,0 +1,28 @@
+using System.Linq;
+using FluentAssertions;
+using GitHub.Unity;
+using NUnit.Framework;
+
+namespace UnitTests
+{
+    [TestFixture]
+    public class ListExtensionTests
+    {
+        [Test]
+        public void SpoolTest()
+        {
+            var results = new[] { "qwer", "asdf", "zxcv", "wert", "1234" }.Spool(10).ToArray();
+            results.Length.Should().Be(3);
+            results[0].Count.Should().Be(2);
+            results[0][0].Should().Be("qwer");
+            results[0][1].Should().Be("asdf");
+
+            results[1].Count.Should().Be(2);
+            results[1][0].Should().Be("zxcv");
+            results[1][1].Should().Be("wert");
+
+            results[2].Count.Should().Be(1);
+            results[2][0].Should().Be("1234");
+        }
+    }
+}

src/tests/UnitTests/UnitTests.csproj

@@ -81,6 +81,7 @@
   </ItemGroup>
   <ItemGroup>
     <Compile Include="Authentication\KeychainTests.cs" />
+    <Compile Include="Extensions\ListExtensionTests.cs" />
     <Compile Include="Git\ValidationTests.cs" />
     <Compile Include="Git\GitConfigTests.cs" />
     <Compile Include="IO\BranchListOutputProcessorTests.cs" />