Implementing batch loading

.gitattributes

@@ -1,2 +1,3 @@
 # Auto detect text files and perform LF normalization
 * text=auto
+* text eol=lf

src/SharpIDE.Application/Features/Analysis/ProjectLoader/CustomMsBuildProjectLoader.Worker.cs

@@ -1,13 +1,14 @@
-using System.Collections.Immutable;
-using System.Diagnostics;
-using System.Runtime.InteropServices;
-using System.Text;
 using Microsoft.CodeAnalysis;
 using Microsoft.CodeAnalysis.Diagnostics;
 using Microsoft.CodeAnalysis.Host;
 using Microsoft.CodeAnalysis.MSBuild;
 using Microsoft.CodeAnalysis.Text;
 using Roslyn.Utilities;
+using System.Collections.Immutable;
+using System.Diagnostics;
+using System.Runtime.CompilerServices;
+using System.Runtime.InteropServices;
+using System.Text;
 
 namespace SharpIDE.Application.Features.Analysis.ProjectLoader;
 
@@ -111,46 +112,52 @@ private async Task<TResult> DoOperationAndReportProgressAsync<TResult>(ProjectLo
 			return result;
 		}
 
-		public async Task<(ImmutableArray<ProjectInfo>, Dictionary<ProjectId, ProjectFileInfo>)> LoadAsync(CancellationToken cancellationToken)
+		public async IAsyncEnumerable<(ImmutableArray<ProjectInfo>, Dictionary<ProjectId, ProjectFileInfo>)> LoadInShardsAsync(
+			[EnumeratorCancellation] CancellationToken cancellationToken,
+			int batchSize = 10)
 		{
-			var results = ImmutableArray.CreateBuilder<ProjectInfo>();
 			var processedPaths = new HashSet<string>(PathUtilities.Comparer);
+			var batchResults = ImmutableArray.CreateBuilder<ProjectInfo>();
+			var batchFileInfoMap = new Dictionary<ProjectId, ProjectFileInfo>();
 
+			int count = 0;
 			foreach (var projectPath in _requestedProjectPaths)
 			{
 				cancellationToken.ThrowIfCancellationRequested();
 
 				if (!_pathResolver.TryGetAbsoluteProjectPath(projectPath, _baseDirectory, _requestedProjectOptions.OnPathFailure, out var absoluteProjectPath))
-				{
-					continue; // Failure should already be reported.
-				}
+					continue;
 
 				if (!processedPaths.Add(absoluteProjectPath))
 				{
 					_diagnosticReporter.Report(
 						new WorkspaceDiagnostic(
 							WorkspaceDiagnosticKind.Warning,
 							string.Format(WorkspaceMSBuildResources.Duplicate_project_discovered_and_skipped_0, absoluteProjectPath)));
-
 					continue;
 				}
 
 				var projectFileInfos = await LoadProjectInfosFromPathAsync(absoluteProjectPath, _requestedProjectOptions, cancellationToken).ConfigureAwait(false);
+				batchResults.AddRange(projectFileInfos);
 
-				results.AddRange(projectFileInfos);
-			}
-
-			foreach (var (projectPath, projectInfos) in _pathToDiscoveredProjectInfosMap)
-			{
-				cancellationToken.ThrowIfCancellationRequested();
+				foreach (var info in projectFileInfos)
+				{
+					var projectId = _projectMap.GetOrCreateProjectId(info?.FilePath ?? string.Empty);
+					batchFileInfoMap[projectId] = _projectIdToFileInfoMap[projectId];
+				}
 
-				if (!processedPaths.Contains(projectPath))
+				count++;
+				if (count % batchSize == 0)
 				{
-					results.AddRange(projectInfos);
+					yield return (batchResults.ToImmutableAndClear(), new Dictionary<ProjectId, ProjectFileInfo>(batchFileInfoMap));
+					batchFileInfoMap.Clear();
 				}
 			}
 
-			return (results.ToImmutableAndClear(), _projectIdToFileInfoMap);
+			if (batchResults.Count > 0)
+			{
+				yield return (batchResults.ToImmutableAndClear(), new Dictionary<ProjectId, ProjectFileInfo>(batchFileInfoMap));
+			}
 		}
 
 		private async Task<ImmutableArray<ProjectFileInfo>> LoadProjectFileInfosAsync(string projectPath, DiagnosticReportingOptions reportingOptions, CancellationToken cancellationToken)

src/SharpIDE.Application/Features/Analysis/ProjectLoader/CustomMsBuildProjectLoader.cs

@@ -1,8 +1,7 @@
-using System.Collections.Immutable;
 using Microsoft.Build.Framework;
 using Microsoft.CodeAnalysis;
-using Microsoft.CodeAnalysis.Host;
 using Microsoft.CodeAnalysis.MSBuild;
+using System.Collections.Immutable;
 
 namespace SharpIDE.Application.Features.Analysis.ProjectLoader;
 // I really don't like having to duplicate this, but we need to use IAnalyzerAssemblyLoaderProvider rather than IAnalyzerService,
@@ -52,7 +51,19 @@ public partial class CustomMsBuildProjectLoader(Workspace workspace, ImmutableDi
 			discoveredProjectOptions,
 			this.LoadMetadataForReferencedProjects);
 
-		return await worker.LoadAsync(cancellationToken).ConfigureAwait(false);
+		var allProjectInfos = ImmutableArray.CreateBuilder<ProjectInfo>();
+		var allFileInfoMap = new Dictionary<ProjectId, ProjectFileInfo>();
+
+		await foreach (var (projectInfos, fileInfoMap) in worker.LoadInShardsAsync(cancellationToken).ConfigureAwait(false))
+		{
+			allProjectInfos.AddRange(projectInfos);
+			foreach (var kvp in fileInfoMap)
+			{
+				allFileInfoMap.TryAdd(kvp.Key, kvp.Value);
+			}
+		}
+
+		return (allProjectInfos.ToImmutable(), allFileInfoMap);
 	}
 
 	/// <summary>
@@ -70,10 +81,7 @@ public partial class CustomMsBuildProjectLoader(Workspace workspace, ImmutableDi
 		ILogger? msbuildLogger = null,
 		CancellationToken cancellationToken = default)
 	{
-		if (solutionFilePath == null)
-		{
-			throw new ArgumentNullException(nameof(solutionFilePath));
-		}
+		ArgumentNullException.ThrowIfNull(solutionFilePath);
 
 		var reportingMode = GetReportingModeForUnrecognizedProjects();
 
@@ -109,15 +117,25 @@ public partial class CustomMsBuildProjectLoader(Workspace workspace, ImmutableDi
 			discoveredProjectOptions: reportingOptions,
 			preferMetadataForReferencesOfDiscoveredProjects: false);
 
-		var (projectInfos, projectFileInfos) = await worker.LoadAsync(cancellationToken).ConfigureAwait(false);
+		var allSolutionInfos = ImmutableArray.CreateBuilder<ProjectInfo>();
+		var allFileInfoMap = new Dictionary<ProjectId, ProjectFileInfo>();
+
+		await foreach (var (projectInfos, fileInfoMap) in worker.LoadInShardsAsync(cancellationToken).ConfigureAwait(false))
+		{
+			allSolutionInfos.AddRange(projectInfos);
+			foreach (var kvp in fileInfoMap)
+			{
+				allFileInfoMap.TryAdd(kvp.Key, kvp.Value);
+			}
+		}
 
 		// construct workspace from loaded project infos
 		var solutionInfo = SolutionInfo.Create(
 			SolutionId.CreateNewId(debugName: absoluteSolutionPath),
 			version: default,
 			absoluteSolutionPath,
-			projectInfos);
+			allSolutionInfos);
 
-		return (solutionInfo, projectFileInfos);
+		return (solutionInfo, allFileInfoMap);
 	}
 }