Skip to content

Commit cf67805

Browse files
radekdoulikCopilot
radekdoulik
and
Copilot
committed
[NO-REVIEW] Batch WASM CoreCLR library test suites on Helix
Reduce Helix queue pressure by grouping ~172 individual WASM CoreCLR library test work items into ~23 batched work items (87% reduction). Changes: - Add eng/testing/WasmBatchRunner.sh: batch runner that extracts and runs multiple test suites sequentially within a single work item, with per-suite result isolation - Add greedy bin-packing inline MSBuild task (_GroupWorkItems) that distributes test archives into balanced batches by file size - Add _AddBatchedWorkItemsForLibraryTests target gated on WasmBatchLibraryTests property (defaults true for CoreCLR+Chrome) - Sample apps excluded from batching, kept as individual work items - Can be disabled with /p:WasmBatchLibraryTests=false Expected impact: - 172 → ~23 Helix work items (87% queue pressure reduction) - ~6% machine time savings (~26 minutes) - Longest batch ~18 minutes (well-balanced bin-packing) Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>
1 parent e053f2e commit cf67805

2 files changed

Lines changed: 283 additions & 1 deletion

File tree

‎eng/testing/WasmBatchRunner.sh‎

Lines changed: 89 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,89 @@
1+
#!/usr/bin/env bash
2+
3+
EXECUTION_DIR=$(dirname "$0")
4+
5+
if [[ -z "$HELIX_WORKITEM_UPLOAD_ROOT" ]]; then
6+
ORIGINAL_UPLOAD_ROOT="$PWD/test-results"
7+
else
8+
ORIGINAL_UPLOAD_ROOT="$HELIX_WORKITEM_UPLOAD_ROOT"
9+
fi
10+
11+
BATCH_DIR="$PWD"
12+
SUITE_COUNT=0
13+
FAIL_COUNT=0
14+
SUITE_NAMES=()
15+
SUITE_EXIT_CODES=()
16+
SUITE_DURATIONS=()
17+
18+
echo "=== WasmBatchRunner ==="
19+
echo "BATCH_DIR=$BATCH_DIR"
20+
echo "ORIGINAL_UPLOAD_ROOT=$ORIGINAL_UPLOAD_ROOT"
21+
22+
for zipFile in "$BATCH_DIR"/*.zip; do
23+
if [[ ! -f "$zipFile" ]]; then
24+
echo "No .zip files found in $BATCH_DIR"
25+
exit 1
26+
fi
27+
28+
suiteName=$(basename "$zipFile" .zip)
29+
suiteDir="$BATCH_DIR/$suiteName"
30+
31+
echo ""
32+
echo "========================= BEGIN $suiteName ============================="
33+
34+
mkdir -p "$suiteDir"
35+
unzip -q -o "$zipFile" -d "$suiteDir"
36+
37+
export HELIX_WORKITEM_UPLOAD_ROOT="$ORIGINAL_UPLOAD_ROOT/$suiteName"
38+
mkdir -p "$HELIX_WORKITEM_UPLOAD_ROOT"
39+
40+
pushd "$suiteDir" >/dev/null
41+
42+
chmod +x RunTests.sh
43+
44+
startTime=$(date +%s)
45+
./RunTests.sh "$@"
46+
suiteExitCode=$?
47+
endTime=$(date +%s)
48+
49+
popd >/dev/null
50+
51+
duration=$((endTime - startTime))
52+
53+
SUITE_NAMES+=("$suiteName")
54+
SUITE_EXIT_CODES+=("$suiteExitCode")
55+
SUITE_DURATIONS+=("$duration")
56+
SUITE_COUNT=$((SUITE_COUNT + 1))
57+
58+
if [[ $suiteExitCode -ne 0 ]]; then
59+
FAIL_COUNT=$((FAIL_COUNT + 1))
60+
echo "----- FAIL $suiteName — exit code $suiteExitCode — ${duration}s -----"
61+
else
62+
echo "----- PASS $suiteName — ${duration}s -----"
63+
fi
64+
65+
echo "========================= END $suiteName ==============================="
66+
done
67+
68+
echo ""
69+
echo "=== Batch Summary ==="
70+
printf "%-60s %-6s %s\n" "Suite" "Status" "Duration"
71+
printf "%-60s %-6s %s\n" "-----" "------" "--------"
72+
73+
for i in "${!SUITE_NAMES[@]}"; do
74+
if [[ ${SUITE_EXIT_CODES[$i]} -eq 0 ]]; then
75+
status="PASS"
76+
else
77+
status="FAIL"
78+
fi
79+
printf "%-60s %-6s %ss\n" "${SUITE_NAMES[$i]}" "$status" "${SUITE_DURATIONS[$i]}"
80+
done
81+
82+
echo ""
83+
echo "Total: $SUITE_COUNT | Passed: $((SUITE_COUNT - FAIL_COUNT)) | Failed: $FAIL_COUNT"
84+
85+
if [[ $FAIL_COUNT -ne 0 ]]; then
86+
exit 1
87+
fi
88+
89+
exit 0

‎src/libraries/sendtohelix-browser.targets‎

Lines changed: 194 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,10 +41,15 @@
4141
'$(Scenario)' == 'WasmTestOnChrome' or
4242
'$(Scenario)' == 'WasmTestOnFirefox'">true</IsRunningLibraryTests>
4343

44+
<WasmBatchLibraryTests Condition="'$(WasmBatchLibraryTests)' == '' and '$(RuntimeFlavor)' == 'CoreCLR' and '$(Scenario)' == 'WasmTestOnChrome'">true</WasmBatchLibraryTests>
45+
<WasmBatchLibraryTests Condition="'$(WasmBatchLibraryTests)' == ''">false</WasmBatchLibraryTests>
46+
<_WasmBatchLargeThreshold Condition="'$(_WasmBatchLargeThreshold)' == ''">52428800</_WasmBatchLargeThreshold>
47+
4448
<HelixExtensionTargets />
4549
<PrepareForBuildHelixWorkItems_WasmDependsOn>
4650
PrepareHelixCorrelationPayload_Wasm;
4751
_AddWorkItemsForLibraryTests;
52+
_AddBatchedWorkItemsForLibraryTests;
4853
_AddWorkItemsForBuildWasmApps
4954
</PrepareForBuildHelixWorkItems_WasmDependsOn>
5055

@@ -172,6 +177,135 @@
172177

173178
<Import Project="$(RepositoryEngineeringDir)testing\wasm-provisioning.targets" />
174179

180+
<UsingTask TaskName="_GroupWorkItems" TaskFactory="RoslynCodeTaskFactory"
181+
AssemblyFile="$(MSBuildToolsPath)\Microsoft.Build.Tasks.Core.dll">
182+
<ParameterGroup>
183+
<Items ParameterType="Microsoft.Build.Framework.ITaskItem[]" Required="true" />
184+
<BatchSize ParameterType="System.Int32" Required="false" />
185+
<LargeThreshold ParameterType="System.Int64" Required="false" />
186+
<GroupedItems ParameterType="Microsoft.Build.Framework.ITaskItem[]" Output="true" />
187+
</ParameterGroup>
188+
<Task>
189+
<Using Namespace="System" />
190+
<Using Namespace="System.Collections.Generic" />
191+
<Using Namespace="System.IO" />
192+
<Using Namespace="System.Linq" />
193+
<Code Type="Fragment" Language="cs">
194+
<![CDATA[
195+
if (BatchSize <= 0) BatchSize = 10;
196+
if (LargeThreshold <= 0) LargeThreshold = 52428800L; // 50MB
197+
198+
var itemsWithSize = new List<(ITaskItem item, long size)>();
199+
foreach (var item in Items)
200+
{
201+
long size = 0;
202+
if (File.Exists(item.ItemSpec))
203+
{
204+
size = new FileInfo(item.ItemSpec).Length;
205+
}
206+
itemsWithSize.Add((item, size));
207+
}
208+
209+
// Sort largest first for greedy bin-packing
210+
itemsWithSize.Sort((a, b) => b.size.CompareTo(a.size));
211+
212+
var result = new List<ITaskItem>();
213+
int negativeBatchId = -1;
214+
215+
// Separate large items (each gets its own batch)
216+
var smallItems = new List<(ITaskItem item, long size)>();
217+
foreach (var entry in itemsWithSize)
218+
{
219+
if (entry.size > LargeThreshold)
220+
{
221+
var newItem = new TaskItem(entry.item);
222+
newItem.SetMetadata("BatchId", negativeBatchId.ToString());
223+
negativeBatchId--;
224+
result.Add(newItem);
225+
}
226+
else
227+
{
228+
smallItems.Add(entry);
229+
}
230+
}
231+
232+
// Greedy bin-packing for small items
233+
if (smallItems.Count > 0)
234+
{
235+
int numBatches = Math.Min(BatchSize, smallItems.Count);
236+
var batchSizes = new long[numBatches];
237+
var batchAssignments = new List<ITaskItem>[numBatches];
238+
for (int i = 0; i < numBatches; i++)
239+
batchAssignments[i] = new List<ITaskItem>();
240+
241+
foreach (var entry in smallItems)
242+
{
243+
// Find batch with smallest total size
244+
int minIdx = 0;
245+
for (int i = 1; i < numBatches; i++)
246+
{
247+
if (batchSizes[i] < batchSizes[minIdx])
248+
minIdx = i;
249+
}
250+
batchSizes[minIdx] += entry.size;
251+
var newItem = new TaskItem(entry.item);
252+
newItem.SetMetadata("BatchId", minIdx.ToString());
253+
batchAssignments[minIdx].Add(newItem);
254+
}
255+
256+
for (int i = 0; i < numBatches; i++)
257+
result.AddRange(batchAssignments[i]);
258+
}
259+
260+
GroupedItems = result.ToArray();
261+
]]>
262+
</Code>
263+
</Task>
264+
</UsingTask>
265+
266+
<UsingTask TaskName="_ComputeBatchTimeout" TaskFactory="RoslynCodeTaskFactory"
267+
AssemblyFile="$(MSBuildToolsPath)\Microsoft.Build.Tasks.Core.dll">
268+
<ParameterGroup>
269+
<GroupedItems ParameterType="Microsoft.Build.Framework.ITaskItem[]" Required="true" />
270+
<BatchIds ParameterType="Microsoft.Build.Framework.ITaskItem[]" Required="true" />
271+
<ItemPrefix ParameterType="System.String" Required="true" />
272+
<BatchOutputDir ParameterType="System.String" Required="true" />
273+
<TimedItems ParameterType="Microsoft.Build.Framework.ITaskItem[]" Output="true" />
274+
</ParameterGroup>
275+
<Task>
276+
<Using Namespace="System" />
277+
<Using Namespace="System.Collections.Generic" />
278+
<Code Type="Fragment" Language="cs">
279+
<![CDATA[
280+
var counts = new Dictionary<string, int>();
281+
foreach (var item in GroupedItems)
282+
{
283+
string bid = item.GetMetadata("BatchId");
284+
if (!counts.ContainsKey(bid)) counts[bid] = 0;
285+
counts[bid]++;
286+
}
287+
288+
var result = new List<ITaskItem>();
289+
foreach (var batchId in BatchIds)
290+
{
291+
string bid = batchId.ItemSpec;
292+
int count = counts.ContainsKey(bid) ? counts[bid] : 1;
293+
int totalMinutes = Math.Max(10, count * 2);
294+
var ts = TimeSpan.FromMinutes(totalMinutes);
295+
296+
var helixItem = new TaskItem(ItemPrefix + "Batch-" + bid);
297+
helixItem.SetMetadata("PayloadDirectory", BatchOutputDir + "batch-" + bid + "/");
298+
helixItem.SetMetadata("Command", "chmod +x WasmBatchRunner.sh && ./WasmBatchRunner.sh");
299+
helixItem.SetMetadata("Timeout", ts.ToString(@"hh\:mm\:ss"));
300+
result.Add(helixItem);
301+
}
302+
303+
TimedItems = result.ToArray();
304+
]]>
305+
</Code>
306+
</Task>
307+
</UsingTask>
308+
175309
<Target Name="PrepareHelixCorrelationPayload_Wasm">
176310
<Error Condition="'$(Scenario)' != 'WasmTestOnV8' and
177311
'$(Scenario)' != 'WasmTestOnChrome' and
@@ -228,7 +362,7 @@
228362
<Target Name="PrepareForBuildHelixWorkItems_Wasm"
229363
DependsOnTargets="$(PrepareForBuildHelixWorkItems_WasmDependsOn);$(HelixExtensionTargets)" />
230364

231-
<Target Name="_AddWorkItemsForLibraryTests" Condition="'$(IsRunningLibraryTests)' == 'true'">
365+
<Target Name="_AddWorkItemsForLibraryTests" Condition="'$(IsRunningLibraryTests)' == 'true' and '$(WasmBatchLibraryTests)' != 'true'">
232366
<ItemGroup Label="Add samples">
233367
<_WasmWorkItem Include="$(TestArchiveRoot)browseronly/**/*.zip" Condition="'$(Scenario)' == 'WasmTestOnChrome' or '$(Scenario)' == 'WasmTestOnFirefox'" />
234368
<_WasmWorkItem Include="$(TestArchiveRoot)chromeonly/**/*.zip" Condition="'$(Scenario)' == 'WasmTestOnChrome'" />
@@ -273,4 +407,63 @@
273407

274408
</ItemGroup>
275409
</Target>
410+
411+
<Target Name="_AddBatchedWorkItemsForLibraryTests"
412+
Condition="'$(IsRunningLibraryTests)' == 'true' and '$(WasmBatchLibraryTests)' == 'true'">
413+
414+
<!-- Collect all test ZIPs the same way the non-batched path does -->
415+
<ItemGroup>
416+
<_WasmBatchWorkItem Include="$(TestArchiveRoot)browseronly/**/*.zip" Condition="'$(Scenario)' == 'WasmTestOnChrome' or '$(Scenario)' == 'WasmTestOnFirefox'" />
417+
<_WasmBatchWorkItem Include="$(TestArchiveRoot)chromeonly/**/*.zip" Condition="'$(Scenario)' == 'WasmTestOnChrome'" />
418+
</ItemGroup>
419+
420+
<!-- Sample apps have different runners (no RunTests.sh), keep them as individual work items -->
421+
<ItemGroup>
422+
<_WasmBatchSampleZip Condition="'$(Scenario)' == 'WasmTestOnV8'" Include="$(TestArchiveRoot)runonly/**/*.Console.V8.*.Sample.zip" />
423+
<_WasmBatchSampleZip Condition="'$(Scenario)' == 'WasmTestOnChrome'" Include="$(TestArchiveRoot)runonly/**/*.Browser.*.Sample.zip" />
424+
425+
<HelixWorkItem Include="@(_WasmBatchSampleZip -> '$(WorkItemPrefix)%(FileName)')">
426+
<PayloadArchive>%(Identity)</PayloadArchive>
427+
<Command>$(HelixCommand)</Command>
428+
<Timeout>$(_workItemTimeout)</Timeout>
429+
</HelixWorkItem>
430+
</ItemGroup>
431+
432+
<ItemGroup>
433+
<_WasmBatchDefaultItems Include="$(WorkItemArchiveWildCard)" Exclude="$(HelixCorrelationPayload)" />
434+
</ItemGroup>
435+
436+
<!-- Batch only test suites (not sample apps) -->
437+
<ItemGroup>
438+
<_WasmBatchAllItems Include="@(_WasmBatchWorkItem)" />
439+
<_WasmBatchAllItems Include="@(_WasmBatchDefaultItems)" />
440+
</ItemGroup>
441+
442+
<!-- Assign batch IDs via greedy bin-packing -->
443+
<_GroupWorkItems Items="@(_WasmBatchAllItems)" BatchSize="20" LargeThreshold="$(_WasmBatchLargeThreshold)">
444+
<Output TaskParameter="GroupedItems" ItemName="_WasmGroupedItem" />
445+
</_GroupWorkItems>
446+
447+
<!-- Determine unique batch IDs -->
448+
<ItemGroup>
449+
<_WasmBatchId Include="@(_WasmGroupedItem -> '%(BatchId)')" />
450+
<_WasmUniqueBatchId Include="@(_WasmBatchId->Distinct())" />
451+
</ItemGroup>
452+
453+
<!-- Stage each batch: copy ZIPs and the runner script into a per-batch directory -->
454+
<MakeDir Directories="$(IntermediateOutputPath)helix-batches/batch-%(_WasmUniqueBatchId.Identity)/" />
455+
<Copy SourceFiles="@(_WasmGroupedItem)" DestinationFolder="$(IntermediateOutputPath)helix-batches/batch-%(BatchId)/" />
456+
<Copy SourceFiles="$(RepositoryEngineeringDir)testing/WasmBatchRunner.sh"
457+
DestinationFolder="$(IntermediateOutputPath)helix-batches/batch-%(_WasmUniqueBatchId.Identity)/" />
458+
459+
<!-- Compute per-batch timeout: 2 minutes per suite, minimum 10 minutes -->
460+
<_ComputeBatchTimeout GroupedItems="@(_WasmGroupedItem)" BatchIds="@(_WasmUniqueBatchId)"
461+
ItemPrefix="$(WorkItemPrefix)" BatchOutputDir="$(IntermediateOutputPath)helix-batches/">
462+
<Output TaskParameter="TimedItems" ItemName="_WasmTimedBatchItem" />
463+
</_ComputeBatchTimeout>
464+
465+
<ItemGroup>
466+
<HelixWorkItem Include="@(_WasmTimedBatchItem)" />
467+
</ItemGroup>
468+
</Target>
276469
</Project>

0 commit comments

Comments
 (0)