diff --git a/eng/testing/WasmBatchRunner.sh b/eng/testing/WasmBatchRunner.sh new file mode 100755 index 00000000000000..22a5c20b689f99 --- /dev/null +++ b/eng/testing/WasmBatchRunner.sh @@ -0,0 +1,103 @@ +#!/usr/bin/env bash + +if [[ -z "$HELIX_WORKITEM_UPLOAD_ROOT" ]]; then + ORIGINAL_UPLOAD_ROOT="$PWD/test-results" +else + ORIGINAL_UPLOAD_ROOT="$HELIX_WORKITEM_UPLOAD_ROOT" +fi + +BATCH_DIR="$PWD" +SUITE_COUNT=0 +FAIL_COUNT=0 +SUITE_NAMES=() +SUITE_EXIT_CODES=() +SUITE_DURATIONS=() + +echo "=== WasmBatchRunner ===" +echo "BATCH_DIR=$BATCH_DIR" +echo "ORIGINAL_UPLOAD_ROOT=$ORIGINAL_UPLOAD_ROOT" + +for zipFile in "$BATCH_DIR"/*.zip; do + if [[ ! -f "$zipFile" ]]; then + echo "No .zip files found in $BATCH_DIR" + exit 1 + fi + + suiteName=$(basename "$zipFile" .zip) + suiteDir="$BATCH_DIR/$suiteName" + + echo "" + echo "========================= BEGIN $suiteName =============================" + + mkdir -p "$suiteDir" + unzip -q -o "$zipFile" -d "$suiteDir" + unzipExitCode=$? + if [[ $unzipExitCode -ne 0 ]]; then + echo "ERROR: Failed to extract $zipFile (exit code: $unzipExitCode)" + FAIL_COUNT=$((FAIL_COUNT + 1)) + SUITE_NAMES+=("$suiteName") + SUITE_EXIT_CODES+=("$unzipExitCode") + SUITE_DURATIONS+=("0") + SUITE_COUNT=$((SUITE_COUNT + 1)) + rm -rf "$suiteDir" + continue + fi + + export HELIX_WORKITEM_UPLOAD_ROOT="$ORIGINAL_UPLOAD_ROOT/$suiteName" + mkdir -p "$HELIX_WORKITEM_UPLOAD_ROOT" + + pushd "$suiteDir" >/dev/null + + chmod +x RunTests.sh + + startTime=$(date +%s) + ./RunTests.sh "$@" + suiteExitCode=$? + endTime=$(date +%s) + + popd >/dev/null + + rm -rf "$suiteDir" + + duration=$((endTime - startTime)) + + SUITE_NAMES+=("$suiteName") + SUITE_EXIT_CODES+=("$suiteExitCode") + SUITE_DURATIONS+=("$duration") + SUITE_COUNT=$((SUITE_COUNT + 1)) + + if [[ $suiteExitCode -ne 0 ]]; then + FAIL_COUNT=$((FAIL_COUNT + 1)) + echo "----- FAIL $suiteName — exit code $suiteExitCode — ${duration}s -----" + else + echo "----- PASS $suiteName — ${duration}s -----" + fi + + echo "========================= END $suiteName ===============================" +done + +# Restore so Helix post-commands write artifacts to the expected root +export HELIX_WORKITEM_UPLOAD_ROOT="$ORIGINAL_UPLOAD_ROOT" + +echo "" +echo "=== Batch Summary ===" +printf "%-60s %-6s %s\n" "Suite" "Status" "Duration" +printf "%-60s %-6s %s\n" "-----" "------" "--------" + +for i in "${!SUITE_NAMES[@]}"; do + if [[ ${SUITE_EXIT_CODES[$i]} -eq 0 ]]; then + status="PASS" + else + status="FAIL" + fi + printf "%-60s %-6s %ss\n" "${SUITE_NAMES[$i]}" "$status" "${SUITE_DURATIONS[$i]}" +done + +echo "" +echo "Total: $SUITE_COUNT | Passed: $((SUITE_COUNT - FAIL_COUNT)) | Failed: $FAIL_COUNT" + +if [[ $FAIL_COUNT -ne 0 ]]; then + exit 1 +fi + +exit 0 diff --git a/src/libraries/sendtohelix-browser.targets b/src/libraries/sendtohelix-browser.targets index dc2fc9f44c8c89..f93858ba92b47c 100644 --- a/src/libraries/sendtohelix-browser.targets +++ b/src/libraries/sendtohelix-browser.targets @@ -41,10 +41,15 @@ '$(Scenario)' == 'WasmTestOnChrome' or '$(Scenario)' == 'WasmTestOnFirefox'">true + true + false + <_WasmBatchLargeThreshold Condition="'$(_WasmBatchLargeThreshold)' == ''">52428800 + PrepareHelixCorrelationPayload_Wasm; _AddWorkItemsForLibraryTests; + _AddBatchedWorkItemsForLibraryTests; _AddWorkItemsForBuildWasmApps @@ -172,6 +177,135 @@ + + + + + + + + + + + + +(); +foreach (var item in Items) +{ + long size = 0; + if (File.Exists(item.ItemSpec)) + { + size = new FileInfo(item.ItemSpec).Length; + } + itemsWithSize.Add((item, size)); +} + +// Sort largest first for greedy bin-packing +itemsWithSize.Sort((a, b) => b.size.CompareTo(a.size)); + +var result = new List(); +int negativeBatchId = -1; + +// Separate large items (each gets its own batch) +var smallItems = new List<(ITaskItem item, long size)>(); +foreach (var entry in itemsWithSize) +{ + if (entry.size > LargeThreshold) + { + var newItem = new TaskItem(entry.item); + newItem.SetMetadata("BatchId", negativeBatchId.ToString()); + negativeBatchId--; + result.Add(newItem); + } + else + { + smallItems.Add(entry); + } +} + +// Greedy bin-packing for small items +if (smallItems.Count > 0) +{ + int numBatches = Math.Min(BatchSize, smallItems.Count); + var batchSizes = new long[numBatches]; + var batchAssignments = new List[numBatches]; + for (int i = 0; i < numBatches; i++) + batchAssignments[i] = new List(); + + foreach (var entry in smallItems) + { + // Find batch with smallest total size + int minIdx = 0; + for (int i = 1; i < numBatches; i++) + { + if (batchSizes[i] < batchSizes[minIdx]) + minIdx = i; + } + batchSizes[minIdx] += entry.size; + var newItem = new TaskItem(entry.item); + newItem.SetMetadata("BatchId", minIdx.ToString()); + batchAssignments[minIdx].Add(newItem); + } + + for (int i = 0; i < numBatches; i++) + result.AddRange(batchAssignments[i]); +} + +GroupedItems = result.ToArray(); +]]> + + + + + + + + + + + + + + + + +(); +foreach (var item in GroupedItems) +{ + string bid = item.GetMetadata("BatchId"); + if (!counts.ContainsKey(bid)) counts[bid] = 0; + counts[bid]++; +} + +var result = new List(); +foreach (var batchId in BatchIds) +{ + string bid = batchId.ItemSpec; + int count = counts.ContainsKey(bid) ? counts[bid] : 1; + // 20 minutes per suite to account for WASM startup overhead + test execution; + // minimum 30 minutes to handle the heaviest individual suites (e.g. Cryptography ~17m) + int totalMinutes = Math.Max(30, count * 20); + var ts = TimeSpan.FromMinutes(totalMinutes); + + var helixItem = new TaskItem(ItemPrefix + "Batch-" + bid); + helixItem.SetMetadata("BatchDir", BatchOutputDir + "batch-" + bid + "/"); + helixItem.SetMetadata("Timeout", ts.ToString(@"hh\:mm\:ss")); + result.Add(helixItem); +} + +TimedItems = result.ToArray(); +]]> + + + + - + <_WasmWorkItem Include="$(TestArchiveRoot)browseronly/**/*.zip" Condition="'$(Scenario)' == 'WasmTestOnChrome' or '$(Scenario)' == 'WasmTestOnFirefox'" /> <_WasmWorkItem Include="$(TestArchiveRoot)chromeonly/**/*.zip" Condition="'$(Scenario)' == 'WasmTestOnChrome'" /> @@ -273,4 +407,78 @@ + + + + + + <_WasmBatchWorkItem Include="$(TestArchiveRoot)browseronly/**/*.zip" /> + <_WasmBatchWorkItem Include="$(TestArchiveRoot)chromeonly/**/*.zip" /> + + + + + <_WasmBatchSampleZip Include="$(TestArchiveRoot)runonly/**/*.Browser.*.Sample.zip" /> + + + %(Identity) + $(HelixCommand) + $(_workItemTimeout) + + + + + <_WasmBatchDefaultItems Include="$(WorkItemArchiveWildCard)" Exclude="$(HelixCorrelationPayload)" /> + + + + + <_WasmBatchAllItems Include="@(_WasmBatchWorkItem)" /> + <_WasmBatchAllItems Include="@(_WasmBatchDefaultItems)" /> + + + + <_GroupWorkItems Items="@(_WasmBatchAllItems)" BatchSize="20" LargeThreshold="$(_WasmBatchLargeThreshold)"> + + + + + + <_WasmBatchId Include="@(_WasmGroupedItem -> '%(BatchId)')" /> + <_WasmUniqueBatchId Include="@(_WasmBatchId->Distinct())" /> + + + + + + + + + + <_ComputeBatchTimeout GroupedItems="@(_WasmGroupedItem)" BatchIds="@(_WasmUniqueBatchId)" + ItemPrefix="$(WorkItemPrefix)" BatchOutputDir="$(IntermediateOutputPath)helix-batches/"> + + + + + + + + + + <_WasmBatchHelixCommand>$(HelixCommand.Replace('./RunTests.sh', 'chmod +x WasmBatchRunner.sh && ./WasmBatchRunner.sh')) + + + + + $(IntermediateOutputPath)helix-batches/%(Identity).zip + $(_WasmBatchHelixCommand) + + +