Skip to content

Commit 4c1d795

Browse files
dfedermdfederm
authored
Add ability to skip placing unchanged output files (#111)
1 parent 40eac82 commit 4c1d795

11 files changed

Lines changed: 198 additions & 11 deletions

‎README.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,6 +73,7 @@ These settings are common across all plugins, although different implementations
7373
| `$(MSBuildCacheGlobalPropertiesToIgnore)` | `string[]` | `CurrentSolutionConfigurationContents; ShouldUnsetParentConfigurationAndPlatform; BuildingInsideVisualStudio; BuildingSolutionFile; SolutionDir; SolutionExt; SolutionFileName; SolutionName; SolutionPath; _MSDeployUserAgent`, as well as all proeprties related to plugin settings | The list of global properties to exclude from consideration by the cache |
7474
| `$(MSBuildCacheGetResultsForUnqueriedDependencies)` | `bool` | false | Whether to try and query the cache for dependencies if they have not previously been requested. This option can help in cases where the build isn't done in graph order, or if some projects are skipped. |
7575
| `$(MSBuildCacheTargetsToIgnore)` | `string[]` | `GetTargetFrameworks;GetNativeManifest;GetCopyToOutputDirectoryItems;GetTargetFrameworksWithPlatformForSingleTargetFramework` | The list of targets to ignore when determining if a build request matches a cache entry. This is intended for "information gathering" targets which do not have side-effect. eg. a build with `/t:Build` and `/t:Build;GetTargetFrameworks` should be considered to have equivalent results. Note: This only works "one-way" in that the build request is allowed to have missing targets, while the cache entry is not. This is to avoid a situation where a build request recieves a cache hit with missing target results, where a cache hit with extra target results is acceptable. |
76+
| `$(MSBuildCacheSkipUnchangedOutputFiles)` | `bool` | false | Whether to avoid writing output files on cache hit if the file is unchanged, which can improve performance for incremental builds. A file is considered unchanged if it exists, the previously placed file and file to be placed have the same hash, and the the previously placed file and current file on disk have the same timestamp and file size. |
7677

7778
When configuring settings which are list types, you should always append to the existing value to avoid overriding the defaults:
7879

‎src/AzureBlobStorage/MSBuildCacheAzureBlobStoragePlugin.cs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -108,7 +108,8 @@ protected override async Task<ICacheClient> CreateCacheClientAsync(PluginLoggerB
108108
GetFileRealizationMode,
109109
Settings.MaxConcurrentCacheContentOperations,
110110
Settings.AsyncCachePublishing,
111-
Settings.AsyncCacheMaterialization);
111+
Settings.AsyncCacheMaterialization,
112+
Settings.SkipUnchangedOutputFiles);
112113
}
113114

114115
private IAzureStorageCredentials CreateAzureStorageCredentials(Context context, AzureBlobStoragePluginSettings settings, CancellationToken cancellationToken)

‎src/AzurePipelines/MSBuildCacheAzurePipelinesPlugin.cs‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,8 @@ protected override async Task<ICacheClient> CreateCacheClientAsync(PluginLoggerB
6161
Settings.MaxConcurrentCacheContentOperations,
6262
Settings.RemoteCacheIsReadOnly,
6363
Settings.AsyncCachePublishing,
64-
Settings.AsyncCacheMaterialization);
64+
Settings.AsyncCacheMaterialization,
65+
Settings.SkipUnchangedOutputFiles);
6566
}
6667

6768
private static async Task<ICacheSession> StartCacheSessionAsync(Context context, LocalCache cache, string name)

‎src/AzurePipelines/PipelineCachingCacheClient.cs‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -105,8 +105,9 @@ public PipelineCachingCacheClient(
105105
int maxConcurrentCacheContentOperations,
106106
bool remoteCacheIsReadOnly,
107107
bool enableAsyncPublishing,
108-
bool enableAsyncMaterialization)
109-
: base(rootContext, fingerprintFactory, hasher, repoRoot, nugetPackageRoot, getFileRealizationMode, localCache, localCAS, maxConcurrentCacheContentOperations, enableAsyncPublishing, enableAsyncMaterialization)
108+
bool enableAsyncMaterialization,
109+
bool skipUnchangedOutputFiles)
110+
: base(rootContext, fingerprintFactory, hasher, repoRoot, nugetPackageRoot, getFileRealizationMode, localCache, localCAS, maxConcurrentCacheContentOperations, enableAsyncPublishing, enableAsyncMaterialization, skipUnchangedOutputFiles)
110111
{
111112
_remoteCacheIsReadOnly = remoteCacheIsReadOnly;
112113
_universe = $"pccc-{(int)hasher.Info.HashType}-{InternalSeed}-" + (string.IsNullOrEmpty(universe) ? "DEFAULT" : universe);

‎src/Common/Caching/CacheClient.cs‎

Lines changed: 35 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@ public abstract class CacheClient : ICacheClient
4343
private readonly ICache _localCache;
4444
private readonly string _nugetPackageRoot;
4545
private readonly bool _canCloneInNugetCachePath;
46+
private readonly LocalCacheStateManager? _localCacheStateManager;
4647

4748
protected CacheClient(
4849
Context rootContext,
@@ -55,7 +56,8 @@ protected CacheClient(
5556
IContentSession localCas,
5657
int maxConcurrentCacheContentOperations,
5758
bool enableAsyncPublishing,
58-
bool enableAsyncMaterialization)
59+
bool enableAsyncMaterialization,
60+
bool skipUnchangedOutputFiles)
5961
{
6062
RootContext = rootContext;
6163
_fingerprintFactory = fingerprintFactory;
@@ -89,6 +91,11 @@ protected CacheClient(
8991
}
9092

9193
_canCloneInNugetCachePath = _copyOnWriteFilesystem.CopyOnWriteLinkSupportedInDirectoryTree(_nugetPackageRoot);
94+
95+
if (skipUnchangedOutputFiles)
96+
{
97+
_localCacheStateManager = new LocalCacheStateManager(repoRoot);
98+
}
9299
}
93100

94101
protected Tracer Tracer { get; } = new Tracer(nameof(CacheClient));
@@ -341,6 +348,11 @@ await AddNodeAsync(
341348
(nodeBuildResultHash, nodeBuildResultBytes),
342349
pathSetBytes,
343350
cancellationToken);
351+
352+
if (_localCacheStateManager is not null)
353+
{
354+
await _localCacheStateManager.WriteStateFileAsync(nodeContext, nodeBuildResult);
355+
}
344356
}
345357

346358
public async Task<(PathSet?, NodeBuildResult?)> GetNodeAsync(
@@ -447,8 +459,22 @@ async Task PlaceFilesAsync(CancellationToken ct)
447459
{
448460
List<Task> tasks = new(nodeBuildResult.PackageFilesToCopy.Count + 1);
449461

450-
Dictionary<string, ContentHash> outputsToPlace = new(nodeBuildResult.Outputs.Count - nodeBuildResult.PackageFilesToCopy.Count);
451-
foreach (KeyValuePair<string, ContentHash> kvp in nodeBuildResult.Outputs)
462+
IEnumerable<KeyValuePair<string, ContentHash>> outputs;
463+
int outputsToPlaceSizeEstimate;
464+
if (_localCacheStateManager is not null)
465+
{
466+
List<KeyValuePair<string, ContentHash>> outOfDateFiles = await _localCacheStateManager.GetOutOfDateFilesAsync(context, nodeContext, nodeBuildResult);
467+
outputs = outOfDateFiles;
468+
outputsToPlaceSizeEstimate = outOfDateFiles.Count;
469+
}
470+
else
471+
{
472+
outputs = nodeBuildResult.Outputs;
473+
outputsToPlaceSizeEstimate = nodeBuildResult.Outputs.Count - nodeBuildResult.PackageFilesToCopy.Count;
474+
}
475+
476+
Dictionary<string, ContentHash> outputsToPlace = new(outputsToPlaceSizeEstimate);
477+
foreach (KeyValuePair<string, ContentHash> kvp in outputs)
452478
{
453479
string destinationAbsolutePath = Path.Combine(RepoRoot, kvp.Key);
454480
if (nodeBuildResult.PackageFilesToCopy.TryGetValue(kvp.Key, out string? packageFile))
@@ -465,8 +491,13 @@ async Task PlaceFilesAsync(CancellationToken ct)
465491
Task placeFilesTask = cacheEntry.PlaceFilesAsync(context, outputsToPlace, ct);
466492
tasks.Add(placeFilesTask);
467493

494+
if (_localCacheStateManager is not null)
495+
{
496+
await _localCacheStateManager.WriteStateFileAsync(nodeContext, nodeBuildResult);
497+
}
498+
468499
await Task.WhenAll(tasks);
469-
};
500+
}
470501

471502
if (_enableAsyncMaterialization)
472503
{

‎src/Common/Caching/CasCacheClient.cs‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -51,8 +51,9 @@ public CasCacheClient(
5151
Func<string, FileRealizationMode> getFileRealizationMode,
5252
int maxConcurrentCacheContentOperations,
5353
bool enableAsyncPublishing,
54-
bool enableAsyncMaterialization)
55-
: base(rootContext, fingerprintFactory, hasher, repoRoot, nugetPackageRoot, getFileRealizationMode, localCache, localCacheSession, maxConcurrentCacheContentOperations, enableAsyncPublishing, enableAsyncMaterialization)
54+
bool enableAsyncMaterialization,
55+
bool skipUnchangedOutputFiles)
56+
: base(rootContext, fingerprintFactory, hasher, repoRoot, nugetPackageRoot, getFileRealizationMode, localCache, localCacheSession, maxConcurrentCacheContentOperations, enableAsyncPublishing, enableAsyncMaterialization, skipUnchangedOutputFiles)
5657
{
5758
ICacheSession cacheSession;
5859
if (remoteCache == null)

‎src/Common/Caching/LocalCacheStateManager.cs‎

Lines changed: 144 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,144 @@
1+
// Copyright (c) Microsoft. All rights reserved.
2+
// Licensed under the MIT license. See LICENSE file in the project root for full license information.
3+
4+
using System;
5+
using System.Collections.Generic;
6+
using System.IO;
7+
using System.Text.Json;
8+
using System.Threading.Tasks;
9+
using BuildXL.Cache.ContentStore.Hashing;
10+
using BuildXL.Cache.ContentStore.Interfaces.Tracing;
11+
using BuildXL.Cache.ContentStore.Tracing;
12+
13+
namespace Microsoft.MSBuildCache.Caching;
14+
15+
internal sealed record LocalCacheStateEntry(string Hash, long LastWriteTime, long FileSize);
16+
17+
internal sealed record LocalCacheStateFile(Dictionary<string, LocalCacheStateEntry> Files);
18+
19+
internal sealed class LocalCacheStateManager
20+
{
21+
private const string CacheStateDirName = ".msbuildcache";
22+
23+
private readonly Tracer _tracer = new(nameof(LocalCacheStateManager));
24+
private readonly string _repoRoot;
25+
private readonly string _cacheStateDir;
26+
27+
public LocalCacheStateManager(string repoRoot)
28+
{
29+
_repoRoot = repoRoot;
30+
_cacheStateDir = Path.Combine(repoRoot, CacheStateDirName);
31+
}
32+
33+
internal async Task WriteStateFileAsync(
34+
NodeContext nodeContext,
35+
NodeBuildResult nodeBuildResult)
36+
{
37+
Dictionary<string, LocalCacheStateEntry> files = new(StringComparer.OrdinalIgnoreCase);
38+
foreach (KeyValuePair<string, ContentHash> kvp in nodeBuildResult.Outputs)
39+
{
40+
string relativeFilePath = kvp.Key;
41+
ContentHash contentHash = kvp.Value;
42+
FileInfo fileInfo = new(Path.Combine(_repoRoot, relativeFilePath));
43+
files[relativeFilePath] = new LocalCacheStateEntry(contentHash.ToShortString(), fileInfo.LastWriteTimeUtc.Ticks, fileInfo.Length);
44+
}
45+
46+
Directory.CreateDirectory(_cacheStateDir);
47+
48+
string stateFilePath = Path.Combine(_cacheStateDir, nodeContext.Id + ".json");
49+
LocalCacheStateFile stateFile = new(files);
50+
51+
using FileStream fileStream = File.Create(stateFilePath);
52+
await JsonSerializer.SerializeAsync(fileStream, stateFile, SourceGenerationContext.Default.LocalCacheStateFile);
53+
}
54+
55+
internal async Task<List<KeyValuePair<string, ContentHash>>> GetOutOfDateFilesAsync(
56+
Context context,
57+
NodeContext nodeContext,
58+
NodeBuildResult nodeBuildResult)
59+
{
60+
string stateFilePath = Path.Combine(_repoRoot, CacheStateDirName, nodeContext.Id + ".json");
61+
62+
LocalCacheStateFile? depFile = null;
63+
if (File.Exists(stateFilePath))
64+
{
65+
try
66+
{
67+
using FileStream fileStream = File.OpenRead(stateFilePath);
68+
depFile = await JsonSerializer.DeserializeAsync(fileStream, SourceGenerationContext.Default.LocalCacheStateFile);
69+
}
70+
catch (JsonException ex)
71+
{
72+
_tracer.Debug(context, $"Error reading local cache state for node {nodeContext.Id}. {ex.Message}");
73+
74+
File.Delete(stateFilePath);
75+
depFile = null;
76+
}
77+
}
78+
else
79+
{
80+
_tracer.Debug(context, $"Local cache state for build target {nodeContext.Id} did not exist.");
81+
}
82+
83+
if (depFile == null)
84+
{
85+
_tracer.Debug(context, "Considering all output files out of date.");
86+
}
87+
88+
List<KeyValuePair<string, ContentHash>> outOfDateFiles = new(nodeBuildResult.Outputs.Count);
89+
90+
foreach (KeyValuePair<string, ContentHash> kvp in nodeBuildResult.Outputs)
91+
{
92+
string relativeFilePath = kvp.Key;
93+
ContentHash contentHash = kvp.Value;
94+
if (!IsFileUpToDate(context, depFile, relativeFilePath, contentHash))
95+
{
96+
outOfDateFiles.Add(kvp);
97+
}
98+
}
99+
100+
return outOfDateFiles;
101+
}
102+
103+
private bool IsFileUpToDate(Context context, LocalCacheStateFile? depFile, string relativeFilePath, ContentHash expectedHash)
104+
{
105+
if (depFile == null)
106+
{
107+
return false;
108+
}
109+
110+
if (!depFile.Files.TryGetValue(relativeFilePath, out LocalCacheStateEntry? cachedInfo))
111+
{
112+
_tracer.Debug(context, $"File {relativeFilePath} was out of date. It was missing from the state file.");
113+
return false;
114+
}
115+
116+
if (!expectedHash.ToShortString().Equals(cachedInfo.Hash, StringComparison.OrdinalIgnoreCase))
117+
{
118+
_tracer.Debug(context, $"File {relativeFilePath} was out of date. The hash did not match.");
119+
return false;
120+
}
121+
122+
FileInfo fileInfo = new(Path.Combine(_repoRoot, relativeFilePath));
123+
if (!fileInfo.Exists)
124+
{
125+
_tracer.Debug(context, $"File {relativeFilePath} was out of date. The file does not exist.");
126+
return false;
127+
}
128+
129+
if (cachedInfo.LastWriteTime != fileInfo.LastWriteTimeUtc.Ticks)
130+
{
131+
_tracer.Debug(context, $"File {relativeFilePath} was out of date. The timestamp did not match.");
132+
return false;
133+
}
134+
135+
if (cachedInfo.FileSize != fileInfo.Length)
136+
{
137+
_tracer.Debug(context, $"File {relativeFilePath} was out of date. The file size did not match.");
138+
return false;
139+
}
140+
141+
_tracer.Debug(context, $"Skipping unchanged file: {relativeFilePath}");
142+
return true;
143+
}
144+
}

‎src/Common/PluginSettings.cs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -107,6 +107,8 @@ public string LocalCacheRootPath
107107

108108
public bool IgnoreDotNetSdkPatchVersion { get; init; }
109109

110+
public bool SkipUnchangedOutputFiles { get; init; }
111+
110112
public static T Create<T>(
111113
IReadOnlyDictionary<string, string> settings,
112114
PluginLoggerBase logger,

‎src/Common/SourceGenerationContext.cs‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,13 +4,15 @@
44
using System.Collections.Generic;
55
using System.Text.Json.Serialization;
66
using BuildXL.Cache.ContentStore.Hashing;
7+
using Microsoft.MSBuildCache.Caching;
78
using Microsoft.MSBuildCache.Fingerprinting;
89

910
namespace Microsoft.MSBuildCache;
1011

1112
[JsonSourceGenerationOptions(WriteIndented = true, Converters = [typeof(ContentHashJsonConverter), typeof(SortedDictionaryConverter)])]
1213
[JsonSerializable(typeof(NodeBuildResult))]
1314
[JsonSerializable(typeof(PathSet))]
15+
[JsonSerializable(typeof(LocalCacheStateFile))]
1416
[JsonSerializable(typeof(IDictionary<string, ContentHash>))]
1517
internal partial class SourceGenerationContext : JsonSerializerContext
1618
{

‎src/Common/build/Microsoft.MSBuildCache.Common.targets‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
<MSBuildCacheGlobalPropertiesToIgnore>$(MSBuildCacheGlobalPropertiesToIgnore);MSBuildCacheGetResultsForUnqueriedDependencies</MSBuildCacheGlobalPropertiesToIgnore>
2424
<MSBuildCacheGlobalPropertiesToIgnore>$(MSBuildCacheGlobalPropertiesToIgnore);MSBuildCacheTargetsToIgnore</MSBuildCacheGlobalPropertiesToIgnore>
2525
<MSBuildCacheGlobalPropertiesToIgnore>$(MSBuildCacheGlobalPropertiesToIgnore);MSBuildCacheIgnoreDotNetSdkPatchVersion</MSBuildCacheGlobalPropertiesToIgnore>
26+
<MSBuildCacheGlobalPropertiesToIgnore>$(MSBuildCacheGlobalPropertiesToIgnore);MSBuildCacheSkipUnchangedOutputFiles</MSBuildCacheGlobalPropertiesToIgnore>
2627
</PropertyGroup>
2728

2829
<ItemGroup Condition="'$(MSBuildCacheEnabled)' != 'false'">
@@ -45,6 +46,7 @@
4546
<GetResultsForUnqueriedDependencies>$(MSBuildCacheGetResultsForUnqueriedDependencies)</GetResultsForUnqueriedDependencies>
4647
<TargetsToIgnore>$(MSBuildCacheTargetsToIgnore)</TargetsToIgnore>
4748
<IgnoreDotNetSdkPatchVersion>$(MSBuildCacheIgnoreDotNetSdkPatchVersion)</IgnoreDotNetSdkPatchVersion>
49+
<SkipUnchangedOutputFiles>$(MSBuildCacheSkipUnchangedOutputFiles)</SkipUnchangedOutputFiles>
4850
</ProjectCachePlugin>
4951
</ItemGroup>
5052

0 commit comments

Comments
 (0)