diff --git a/test.wprp b/.github/scripts/cpu.wprp similarity index 95% rename from test.wprp rename to .github/scripts/cpu.wprp index b925fe75..9a0feec8 100644 --- a/test.wprp +++ b/.github/scripts/cpu.wprp @@ -1,12 +1,12 @@ - - + + - - + + @@ -27,12 +27,12 @@ - + @@ -64,7 +64,9 @@ + @@ -93,14 +95,14 @@ - + - + @@ -129,4 +131,4 @@ - \ No newline at end of file + diff --git a/.github/scripts/performance_utlilites.psm1 b/.github/scripts/performance_utlilites.psm1 new file mode 100644 index 00000000..4e1ef034 --- /dev/null +++ b/.github/scripts/performance_utlilites.psm1 @@ -0,0 +1,535 @@ +# WPR CPU profiling helpers +$script:WprProfiles = @{} + +function Start-WprCpuProfile { + param([Parameter(Mandatory=$true)][string]$Which) + + if (-not $CpuProfile) { return } + + $Workspace = $env:GITHUB_WORKSPACE + $etlDir = Join-Path $Workspace 'ETL' + if (-not (Test-Path $etlDir)) { New-Item -ItemType Directory -Path $etlDir | Out-Null } + + $WprProfile = Join-Path $etlDir 'cpu.wprp' + + $outFile = Join-Path $etlDir ("cpu_profile-$Which.etl") + if (Test-Path $outFile) { Remove-Item $outFile -Force -ErrorAction SilentlyContinue } + + Write-Host "Starting WPR CPU profiling -> $outFile" + try { + # Check if WPR is already running to avoid the "profiles are already running" error + $status = $null + try { + $status = & wpr -status 2>&1 + } catch { + $status = $_.ToString() + } + + if ($status -and $status -match 'profile(s)?\s+are\s+already\s+running|Profiles are already running|The profiles are already running') { + Write-Host "WPR already running. Cancelling any existing profiles so we can start a fresh one..." + try { + & wpr -cancel 2>&1 | Out-Null + Start-Sleep -Seconds 1 + } + catch { + Write-Host "Failed to cancel existing WPR session: $($_.Exception.Message). Proceeding to start a new profile anyway." + } + } + + try { + & wpr -start $WprProfile -filemode | Out-Null + } + catch { + Write-Host "wpr -start with custom profile failed: $($_.Exception.Message). Falling back to built-in CPU profile." + try { & wpr -start CPU -filemode | Out-Null } catch { Write-Host "Fallback CPU start also failed: $($_.Exception.Message)" } + } + $script:WprProfiles[$Which] = $outFile + } + catch { + Write-Host "Failed to start WPR: $($_.Exception.Message)" + } +} + +function Stop-WprCpuProfile { + param([Parameter(Mandatory=$true)][string]$Which) + + if (-not $CpuProfile) { return } + + if (-not $script:WprProfiles.ContainsKey($Which)) { + Write-Host "No WPR profile active for '$Which'" + return + } + + $outFile = $script:WprProfiles[$Which] + Write-Host "Stopping WPR CPU profiling, saving to $outFile" + try { + # Attempt to stop WPR and save to the given file. If no profile is running, log and continue. + try { + & wpr -stop $outFile | Out-Null + } + catch { + Write-Host "wpr -stop failed: $($_.Exception.Message). Attempting to query status..." + try { + $s = & wpr -status 2>&1 + Write-Host "WPR status: $s" + } catch { } + } + $script:WprProfiles.Remove($Which) | Out-Null + } + catch { + Write-Host "Failed to stop WPR: $($_.Exception.Message)" + } +} + +function Receive-JobOrThrow { + param([Parameter(Mandatory)] $Job) + + Wait-Job $Job | Out-Null + + # Drain output (keep so we can inspect again if needed) + $null = Receive-Job $Job -Keep + + $errs = @() + foreach ($cj in $Job.ChildJobs) { + if ($cj.Error -and $cj.Error.Count -gt 0) { + $errs += $cj.Error + } + if ($cj.JobStateInfo.State -eq 'Failed' -and $cj.JobStateInfo.Reason) { + $errs += $cj.JobStateInfo.Reason + } + } + + if ($errs.Count -gt 0) { + foreach ($er in $errs) { + if ($er -is [System.Management.Automation.ErrorRecord]) { + $er | Write-DetailedError + } + else { + Write-Host $er + } + } + throw "One or more remote errors occurred (job id: $($Job.Id))." + } + + if ($Job.State -eq 'Failed') { + throw "Remote job failed (job id: $($Job.Id)): $($Job.JobStateInfo.Reason)" + } +} + +function Create-Session { + param( + [Parameter(Mandatory=$true)][string]$PeerName, + [string]$RemotePSConfiguration = 'PowerShell.7' + ) + + $script:RemotePSConfiguration = $RemotePSConfiguration + $script:RemoteDir = 'C:\_work' + + $Username = (Get-ItemProperty 'HKLM:\Software\Microsoft\Windows NT\CurrentVersion\Winlogon').DefaultUserName + $Password = (Get-ItemProperty 'HKLM:\Software\Microsoft\Windows NT\CurrentVersion\Winlogon').DefaultPassword | ConvertTo-SecureString -AsPlainText -Force + $Creds = New-Object System.Management.Automation.PSCredential ($Username, $Password) + + try { + Write-Host "Creating PSSession to $PeerName using configuration '$RemotePSConfiguration'..." + $s = New-PSSession -ComputerName $PeerName -Credential $Creds -ConfigurationName $RemotePSConfiguration -ErrorAction Stop + Write-Host "Session created using configuration '$RemotePSConfiguration'." + } + catch { + Write-Host "Failed to create session using configuration '$RemotePSConfiguration': $($_.Exception.Message)" + Write-Host "Attempting fallback: creating session without ConfigurationName..." + try { + $s = New-PSSession -ComputerName $PeerName -Credential $Creds -ErrorAction Stop + Write-Host "Session created using default configuration." + } + catch { + Write-Host "Fallback session creation failed: $($_.Exception.Message)" + throw "Failed to create remote session to $PeerName" + } + } + + $script:Session = $s + return $s +} + +function Save-And-Disable-Firewalls { + param([Parameter(Mandatory=$true)]$Session) + + # Coerce possible multi-output (array) from Create-Session into the actual PSSession object. + if ($Session -is [System.Array]) { + $found = $Session | Where-Object { $_ -is [System.Management.Automation.Runspaces.PSSession] } + if ($found -and $found.Count -gt 0) { $Session = $found[0] } + else { $Session = $Session[0] } + } + + if (-not ($Session -is [System.Management.Automation.Runspaces.PSSession])) { + throw "Save-And-Disable-Firewalls requires a PSSession object. Got: $($Session.GetType().FullName) - $Session" + } + + Write-Host "Saving and disabling local firewall profiles..." + $script:localFwState = Get-NetFirewallProfile -Profile Domain, Public, Private | Select-Object Name, Enabled + Set-NetFirewallProfile -Profile Domain, Public, Private -Enabled False + + Write-Host "Disabling firewall on remote machine..." + Invoke-Command -Session $Session -ScriptBlock { + param() + $fw = Get-NetFirewallProfile -Profile Domain, Public, Private | Select-Object Name, Enabled + Set-Variable -Name __SavedFirewallState -Value $fw -Scope Global -Force + Set-NetFirewallProfile -Profile Domain, Public, Private -Enabled False + } -ErrorAction Stop +} + + +function CaptureCpuUsagePerformanceMonitorAsJob { + param( + [Parameter(Mandatory=$true)][string]$DurationSeconds + ) + # Ensure we pass a numeric duration into the job and use that value inside + $intDuration = [int]::Parse($DurationSeconds) + + $cpuMonitorJob = Start-Job -ScriptBlock { + param($duration) + + $counter = '\Processor(_Total)\% Processor Time' + $d = [int]$duration + + try { + $samples = Get-Counter -Counter $counter -SampleInterval 1 -MaxSamples $d -ErrorAction Stop + $values = $samples.CounterSamples | ForEach-Object { [double]$_.CookedValue } + } + catch { + $values = @(0) + } + + + $average = ($values | Measure-Object -Average).Average + + # Emit a raw numeric value so the caller can parse it reliably + Write-Output $average + } -ArgumentList $intDuration + + return $cpuMonitorJob +} + +function CaptureIndividualCpuUsagePerformanceMonitorAsJob { + param( + [Parameter(Mandatory=$true)][string]$DurationSeconds + ) + + # Ensure we pass a numeric duration into the job and use that value inside + $intDuration = [int]::Parse($DurationSeconds) + + $cpuMonitorJob = Start-Job -ScriptBlock { + param($duration) + + $counter = '\Processor(*)\% Processor Time' + $d = [int]$duration + + try { + $samples = Get-Counter -Counter $counter -SampleInterval 1 -MaxSamples $d -ErrorAction Stop + # Group samples by instance (processor index) and compute average per instance + $grouped = $samples.CounterSamples | Group-Object -Property InstanceName + $results = @() + foreach ($g in $grouped) { + $vals = $g.Group | ForEach-Object { [double]$_.CookedValue } + $avg = ($vals | Measure-Object -Average).Average + $results += [PSCustomObject]@{ Processor = $g.Name; Average = $avg } + } + # Sort by processor name to have consistent ordering (e.g., _Total last or first) + $sorted = $results | Sort-Object @{Expression={$_.Processor -replace '^CPU',''}},Processor + # Emit numeric array (only per-CPU numeric averages, excluding the _Total instance) + $numeric = $sorted | Where-Object { $_.Processor -ne '_Total' } | ForEach-Object { [double]$_.Average } + } + catch { + $numeric = @(0) + } + + # Emit the numeric array so the caller receives per-CPU averages + Write-Output $numeric + } -ArgumentList $intDuration + + return $cpuMonitorJob +} + +function Restore-FirewallAndCleanup { + param([object]$Session) + + try { + if ($null -ne $Session) { + try { + Write-Host "Restoring firewall state on remote machine..." + Invoke-Command -Session $Session -ScriptBlock { + if (Get-Variable -Name __SavedFirewallState -Scope Global -ErrorAction SilentlyContinue) { + $saved = Get-Variable -Name __SavedFirewallState -Scope Global -ValueOnly + foreach ($p in $saved) { + Set-NetFirewallProfile -Profile $p.Name -Enabled $p.Enabled + } + Remove-Variable -Name __SavedFirewallState -Scope Global -ErrorAction SilentlyContinue + } + else { + Set-NetFirewallProfile -Profile Domain, Public, Private -Enabled True + } + } -ErrorAction SilentlyContinue + } + catch { + $_ | Write-DetailedError + } + + try { + Remove-PSSession $Session -ErrorAction SilentlyContinue + } + catch { + $_ | Write-DetailedError + } + } + + Write-Host "Restoring local firewall state..." + if ($localFwState) { + foreach ($p in $localFwState) { + Set-NetFirewallProfile -Profile $p.Name -Enabled $p.Enabled + } + } + else { + Set-NetFirewallProfile -Profile Domain, Public, Private -Enabled True + } + } + catch { + $_ | Write-DetailedError + } +} + +function Set-RssSettings { + param( + [Parameter(Mandatory=$true)][string]$AdapterName, + [Parameter(Mandatory=$false)][int]$CpuCount + ) + + Write-Host "Applying RSS settings to adapter '$AdapterName'..." + + function Get-RssCapabilities { + param([string]$AdapterName) + + # Prefer the convenient cmdlet if available + if (Get-Command -Name Get-NetAdapterRssCapabilities -ErrorAction SilentlyContinue) { + try { + $res = Get-NetAdapterRssCapabilities -Name $AdapterName -ErrorAction SilentlyContinue + if ($res) { + return [PSCustomObject]@{ + MaxProcessorNumber = [int]$res.MaxProcessorNumber + MaxProcessorGroup = [int]$res.MaxProcessorGroup + } + } + return $null + } catch { + return $null + } + } + + # Fallback: query the CIM class directly + try { + $filter = "Name='$AdapterName'" + $obj = Get-CimInstance -Namespace root/StandardCimv2 -ClassName MSFT_NetAdapterRssSettingData -Filter $filter -ErrorAction SilentlyContinue + if ($obj) { + return [PSCustomObject]@{ + MaxProcessorNumber = [int]$obj.MaxProcessorNumber + MaxProcessorGroup = [int]$obj.MaxProcessorGroup + } + } + } catch { + # ignore + } + return $null + } + + # Use the adapter name provided by the caller and validate it exists and is operational + try { + $adapter = Get-NetAdapter -Name $AdapterName -ErrorAction SilentlyContinue + } catch { + $adapter = $null + } + if (-not $adapter) { + Write-Host "Adapter '$AdapterName' not found. Returning." + return + } + if ($adapter.Status -ne 'Up') { + Write-Host "Adapter '$AdapterName' is not operational (Status: $($adapter.Status)). Returning." + return + } + + $ReachableNIC = $adapter.Name + Write-Host "Configuring RSS on adapter: $ReachableNIC" + + # Check RSS capabilities (cmdlet or CIM fallback) + $capCheck = Get-RssCapabilities -AdapterName $ReachableNIC + if (-not $capCheck) { + Write-Host "Adapter '$ReachableNIC' does not expose RSS capabilities or does not support RSS. Skipping RSS configuration." + return + } + + # Enable RSS if the cmdlet exists; otherwise inform and continue to capability-only flow + if (Get-Command -Name Enable-NetAdapterRss -ErrorAction SilentlyContinue) { + try { + Enable-NetAdapterRss -Name $ReachableNIC -ErrorAction Stop + } catch { + Write-Host "Failed to enable RSS on '$ReachableNIC': $($_.Exception.Message)" + return + } + } else { + Write-Host "Enable-NetAdapterRss cmdlet not present; skipping enable step." + } + + # Get RSS capabilities to determine CPU range + # Re-read capabilities (cmdlet or CIM fallback) + $cap = Get-RssCapabilities -AdapterName $ReachableNIC + if (-not $cap) { + Write-Host "No RSS capability information returned. Returning." + return + } + + $maxCPU = $cap.MaxProcessorNumber + $maxGroup = $cap.MaxProcessorGroup + + if (-not ($maxCPU -is [int]) -or -not ($maxGroup -is [int])) { + Write-Host "Unexpected RSS capability values. Returning." + return + } + + if ($maxGroup -lt 1) { + Write-Host "No processor groups reported. Assuming group 0." + $useGroup = 0 + } else { + $useGroup = 0 + } + if ($maxCPU -lt 0) { + Write-Host "Invalid MaxProcessorNumber ($maxCPU). Returning." + return + } + + if (Get-Command -Name Set-NetAdapterRss -ErrorAction SilentlyContinue) { + # Determine how many CPUs to set. Use provided CpuCount if valid, otherwise the adapter max. + if ($CpuCount) { + if ($CpuCount -lt 1) { + Write-Host "Provided CpuCount ($CpuCount) is invalid; must be >= 1. Returning." + return + } + if ($CpuCount -gt ($maxCPU + 1)) { + Write-Host "Provided CpuCount ($CpuCount) exceeds adapter MaxProcessorNumber ($maxCPU + 1). Clamping to max." + $useMax = $maxCPU + } else { + # Convert CpuCount (count) to MaxProcessorNumber (index) + $useMax = [int]($CpuCount - 1) + } + } else { + $useMax = $maxCPU + } + + $CpuCount = $useMax + 1 + + Write-Host "Setting RSS to use CPUs 0..$useMax in group $useGroup" + try { + Set-NetAdapterRss -Name $ReachableNIC -BaseProcessorGroup $useGroup -MaxProcessorNumber $useMax -MaxProcessors $CpuCount -Profile NUMAStatic -ErrorAction Stop + } catch { + Write-Host "Failed to set RSS on '$ReachableNIC': $($_.Exception.Message)" + return + } + + # Disable then re-enable the adapter to ensure settings apply + if ((Get-Command -Name Disable-NetAdapter -ErrorAction SilentlyContinue) -and (Get-Command -Name Enable-NetAdapter -ErrorAction SilentlyContinue)) { + try { + Write-Host "Disabling adapter '$ReachableNIC' to apply settings..." + Disable-NetAdapter -Name $ReachableNIC -Confirm:$false -ErrorAction Stop + Start-Sleep -Seconds 2 + Write-Host "Re-enabling adapter '$ReachableNIC'..." + Enable-NetAdapter -Name $ReachableNIC -Confirm:$false -ErrorAction Stop + Start-Sleep -Seconds 2 + } catch { + Write-Host "Warning: failed to toggle adapter '$ReachableNIC': $($_.Exception.Message)" + } + } else { + Write-Host "Disable/Enable adapter cmdlets not present; skipping adapter toggle." + } + + if (Get-Command -Name Get-NetAdapterRss -ErrorAction SilentlyContinue) { + Write-Host "Updated RSS settings for '$ReachableNIC':" + Get-NetAdapterRss -Name $ReachableNIC + } else { + Write-Host "Set successful (no Get-NetAdapterRss cmdlet present to display settings)." + } + } else { + Write-Host "Set-NetAdapterRss cmdlet not present; cannot modify RSS settings. Displaying reported capabilities instead:" + Write-Host "MaxProcessorNumber: $maxCPU, MaxProcessorGroup: $maxGroup" + } +} + +function CapturePerformanceMonitorAsJob { + param( + [Parameter(Mandatory=$true)][string]$DurationSeconds, + [Parameter(Mandatory=$false)][string[]]$Counters = @('\Processor Information(*)\% Processor Time') + ) + + # Ensure numeric duration + $intDuration = [int]::Parse($DurationSeconds) + + $perfJob = Start-Job -ScriptBlock { + param($duration, $counters) + + $d = [int]$duration + if (-not $counters -or $counters.Count -eq 0) { + $counters = @('\Processor Information(*)\% Processor Time') + } + + # Sample once per second for the requested duration and accumulate per-counter values. + $store = @{} + + for ($i = 0; $i -lt $d; $i++) { + $samples = $null + try { + # Try to collect all counters in a single quick sample (returns immediately) + $samples = Get-Counter -Counter $counters -MaxSamples 1 -ErrorAction Stop + } + catch { + # If that fails, collect available counters individually (quick single-sample calls) + $samples = New-Object System.Collections.Generic.List[object] + foreach ($c in $counters) { + try { + $s = Get-Counter -Counter $c -MaxSamples 1 -ErrorAction Stop + if ($s.CounterSamples) { $s.CounterSamples | ForEach-Object { [void]$samples.Add($_) } } + } + catch { + # skip bad counter + continue + } + } + } + + if ($samples -ne $null) { + $csamples = $samples.CounterSamples + if (-not $csamples -and ($samples -is [System.Collections.IEnumerable])) { $csamples = $samples } + foreach ($cs in $csamples) { + $path = $cs.Path + $inst = $cs.InstanceName + if ([string]::IsNullOrEmpty($inst) -or $inst -eq '_Total') { continue } + if ($cs.Status -ne 'Success') { continue } + $key = "$path`|$inst" + if (-not $store.ContainsKey($key)) { $store[$key] = New-Object System.Collections.ArrayList } + [void]$store[$key].Add([double]$cs.CookedValue) + } + } + + Start-Sleep -Seconds 1 + } + + $results = @() + foreach ($k in $store.Keys) { + $parts = $k -split '\|',2 + $path = $parts[0] + $inst = $parts[1] + $avg = ($store[$k] | Measure-Object -Average).Average + $results += [PSCustomObject]@{ Counter = $path; Instance = $inst; Average = $avg } + } + + # Emit structured results: an array of PSObjects with Counter, Instance, Average + $results + } -ArgumentList $intDuration, $Counters + + return $perfJob +} \ No newline at end of file diff --git a/.github/scripts/run-echo-test-linux.ps1 b/.github/scripts/run-echo-test-linux.ps1 new file mode 100644 index 00000000..328d0c0d --- /dev/null +++ b/.github/scripts/run-echo-test-linux.ps1 @@ -0,0 +1,99 @@ +param( + [string]$PeerName = "netperf-peer", + [string]$SenderOptions, + [string]$ReceiverOptions, + [string]$Duration = "60", + [string]$RemoteServerPath = "/tmp/echo_server", + [string]$RemoteServerLogPath = "/tmp/server.log" +) + +$ErrorActionPreference = 'Stop' + +function Ensure-Executable($path) { + if (-not (Test-Path $path)) { throw "Missing binary: $path" } + try { chmod +x $path } catch { Write-Host "chmod failed (may already be executable): $path" } +} + +# Resolve local echo binaries in current working directory +$cwd = Get-Location +$serverPath = Join-Path $cwd 'echo_server' +$clientPath = Join-Path $cwd 'echo_client' + +Ensure-Executable $serverPath +Ensure-Executable $clientPath + +# Establish SSH PowerShell remoting to Linux peer +Write-Host "Creating PowerShell SSH session to peer: $PeerName" +$session = $null +try { + $session = New-PSSession -HostName $PeerName +} catch { + throw "Failed to create SSH PSSession to $PeerName. Ensure SSH keys/config are set. Error: $_" +} + +# Copy server binary to peer and ensure executable +Write-Host "Copying server binary to peer at $RemoteServerPath" +Copy-Item -Path $serverPath -Destination $RemoteServerPath -ToSession $session +Invoke-Command -Session $session -ScriptBlock { chmod +x $using:RemoteServerPath } + +# Start server in background on peer, capture PID +Write-Host "Starting server on peer" +$startServer = @" + bash -lc "nohup $using:RemoteServerPath $using:ReceiverOptions > $using:RemoteServerLogPath 2>&1 & echo \$!" +"@ +$serverPid = Invoke-Command -Session $session -ScriptBlock ([ScriptBlock]::Create($startServer)) +Write-Host "Server PID on peer: $serverPid" +Start-Sleep -Seconds 2 + +# Run client locally and capture output +Write-Host "Running echo client locally" +$clientCmd = @" +bash -lc '$clientPath $SenderOptions --duration $Duration' +"@ +$clientOutput = & pwsh -NoProfile -Command $clientCmd 2>&1 +$clientExit = $LASTEXITCODE + +# Save client output +"$clientOutput" | Out-File -FilePath "echo_client_output.txt" -Encoding utf8 +Write-Host "Client exit: $clientExit" + +# Attempt to parse simple metrics from client output +$sent = 0 +$received = 0 +try { + $sentMatch = ($clientOutput | Select-String -Pattern "Packets sent: (\\d+)") + $recvMatch = ($clientOutput | Select-String -Pattern "Packets received: (\\d+)") + if ($sentMatch) { $sent = [int]($sentMatch.Matches[0].Groups[1].Value) } + if ($recvMatch) { $received = [int]($recvMatch.Matches[0].Groups[1].Value) } +} catch { } + +# Write a minimal CSV summary +$csvLines = @() +$csvLines += "Test,Sent,Received,Duration" +$csvLines += "LinuxEcho,$sent,$received,$Duration" +$csvLines | Out-File -FilePath "echo_summary.csv" -Encoding utf8 + +# Stop server on peer and collect logs +Write-Host "Stopping server on peer" +try { + Invoke-Command -Session $session -ScriptBlock { kill -TERM $using:serverPid } | Out-Null +} catch { + Write-Host "Kill failed: $_" +} +Start-Sleep -Seconds 1 + +Write-Host "Fetching server log from peer" +try { + Copy-Item -FromSession $session -Path $RemoteServerLogPath -Destination 'server.log' +} catch { + Write-Host "Failed to copy server.log: $_" +} + +# Close session +if ($session) { Remove-PSSession $session } + +# Return non-zero if client failed +if ($clientExit -ne 0) { + Write-Host "Client reported non-zero exit: $clientExit" + exit $clientExit +} diff --git a/.github/scripts/run-echo-test.ps1 b/.github/scripts/run-echo-test.ps1 new file mode 100644 index 00000000..4c27b595 --- /dev/null +++ b/.github/scripts/run-echo-test.ps1 @@ -0,0 +1,834 @@ +param( + [switch]$CpuProfile, + [string]$PeerName, + [string]$SenderOptions, + [string]$ReceiverOptions, + [string]$Duration = "60" +) + +Set-StrictMode -Version Latest + +# Write out the parameters for logging +Write-Host "Parameters:" +Write-Host " CpuProfile: $CpuProfile" +Write-Host " PeerName: $PeerName" +Write-Host " SenderOptions: $SenderOptions" +Write-Host " ReceiverOptions: $ReceiverOptions" + +# Add --server to the sender/client options if not already present +if ($SenderOptions -notmatch '--server') { + $SenderOptions += " --server $PeerName" +} + +# Add duration option if specified and not already present to both sender and receiver +if ($Duration -and $Duration -gt 0 -and $SenderOptions -notmatch '--duration') { + $SenderOptions += " --duration $Duration" +} + +if ($Duration -and $Duration -gt 0 -and $ReceiverOptions -notmatch '--duration') { + $ReceiverOptions += " --duration $Duration" +} + +# Make errors terminate so catch can handle them +$ErrorActionPreference = 'Stop' +$Session = $null +$exitCode = 0 + +# Ensure local firewall state variable exists so cleanup never errors +$localFwState = $null + +# Helper to parse quoted command-line option strings into an array +function Convert-ArgStringToArray($s) { + if ([string]::IsNullOrEmpty($s)) { return @() } + # Pattern allows quoted strings with backslash-escaped characters, or unquoted tokens + # Matches either: "( (?: \\. | [^"\\] )* )" or [^"\s]+ + $pattern = '("((?:\\.|[^"\\])*)"|[^"\s]+)' + $regexMatches = [regex]::Matches($s, $pattern) + $out = @() + foreach ($m in $regexMatches) { + if ($m.Groups[2].Success) { + # Quoted token; Group 2 contains inner text with possible escapes + $val = $m.Groups[2].Value + # Unescape backslash-escaped sequences commonly used in CLI args + $val = $val -replace '\\', '\' + $val = $val -replace '\"', '"' + } + else { + # Unquoted token in Group 1 + $val = $m.Groups[1].Value + } + $out += $val.Trim() + } + return $out +} + +# Normalize tokens: prefix '-' only for standalone tokens that don't look like values +function Normalize-Args { + param([Parameter(Mandatory=$true)][object[]]$Tokens) + if ($null -eq $Tokens) { return @() } + $out = @() + for ($i = 0; $i -lt $Tokens.Count; $i++) { + $t = $Tokens[$i] + if ([string]::IsNullOrEmpty($t)) { continue } + + # Keep tokens that already start with '-' or contain '=' as-is + if ($t -like '-*' -or $t -match '=') { + $out += $t + continue + } + + # If previous token exists and starts with '-', treat this token as that option's value + if ($i -gt 0 -and ($Tokens[$i-1] -is [string]) -and ($Tokens[$i-1] -like '-*')) { + $out += $t + continue + } + + # Otherwise prefix a single '-' + $out += ('-' + $t) + } + return $out +} + +# Rename a local file if it exists; ignore if not present +function Rename-LocalIfExists { + param( + [Parameter(Mandatory=$true)][string]$Path, + [Parameter(Mandatory=$true)][string]$NewName + ) + try { + if (Test-Path $Path) { + # If the desired new name already exists, remove it first so Rename-Item succeeds + if (Test-Path $NewName) { + try { Remove-Item -Path $NewName -Force -ErrorAction Stop } catch { Write-Host "Warning: failed to remove existing '$NewName': $($_.Exception.Message)" } + } + Rename-Item -Path $Path -NewName $NewName -ErrorAction Stop + } + } + catch { + Write-Host "Failed to rename $Path -> $NewName $($_.Exception.Message)" + } +} + +# Print detailed information for an ErrorRecord or Exception. Supports pipeline input. +function Write-DetailedError { + param( + [Parameter(ValueFromPipeline=$true, ValueFromPipelineByPropertyName=$true)] $InputObject + ) + + process { + $er = $InputObject + if ($null -eq $er) { return } + if ($er -is [System.Management.Automation.ErrorRecord]) { + Write-Host "ERROR: $($er.Exception.Message)" + if ($er.Exception.StackTrace) { Write-Host "StackTrace: $($er.Exception.StackTrace)" } + if ($er.InvocationInfo) { Write-Host "Invocation: $($er.InvocationInfo.PositionMessage)" } + Write-Host "ErrorRecord: $er" + } + elseif ($er -is [System.Exception]) { + Write-Host "EXCEPTION: $($er.Message)" + if ($er.StackTrace) { Write-Host "StackTrace: $($er.StackTrace)" } + } + else { + Write-Host $er + } + } +} + +# WPR CPU profiling helpers +$script:WprProfiles = @{} + +function Start-WprCpuProfile { + param([Parameter(Mandatory=$true)][string]$Which) + + if (-not $CpuProfile) { return } + + $Workspace = $env:GITHUB_WORKSPACE + $etlDir = Join-Path $Workspace 'ETL' + if (-not (Test-Path $etlDir)) { New-Item -ItemType Directory -Path $etlDir | Out-Null } + + $WprProfile = Join-Path $etlDir 'cpu.wprp' + + $outFile = Join-Path $etlDir ("cpu_profile-$Which.etl") + if (Test-Path $outFile) { Remove-Item $outFile -Force -ErrorAction SilentlyContinue } + + Write-Host "Starting WPR CPU profiling -> $outFile" + try { + # Check if WPR is already running to avoid the "profiles are already running" error + $status = $null + try { + $status = & wpr -status 2>&1 + } catch { + $status = $_.ToString() + } + + if ($status -and $status -match 'profile(s)?\s+are\s+already\s+running|Profiles are already running|The profiles are already running') { + Write-Host "WPR already running. Cancelling any existing profiles so we can start a fresh one..." + try { + & wpr -cancel 2>&1 | Out-Null + Start-Sleep -Seconds 1 + } + catch { + Write-Host "Failed to cancel existing WPR session: $($_.Exception.Message). Proceeding to start a new profile anyway." + } + } + + try { + & wpr -start $WprProfile -filemode | Out-Null + } + catch { + Write-Host "wpr -start with custom profile failed: $($_.Exception.Message). Falling back to built-in CPU profile." + try { & wpr -start CPU -filemode | Out-Null } catch { Write-Host "Fallback CPU start also failed: $($_.Exception.Message)" } + } + $script:WprProfiles[$Which] = $outFile + } + catch { + Write-Host "Failed to start WPR: $($_.Exception.Message)" + } +} + +function Stop-WprCpuProfile { + param([Parameter(Mandatory=$true)][string]$Which) + + if (-not $CpuProfile) { return } + + if (-not $script:WprProfiles.ContainsKey($Which)) { + Write-Host "No WPR profile active for '$Which'" + return + } + + $outFile = $script:WprProfiles[$Which] + Write-Host "Stopping WPR CPU profiling, saving to $outFile" + try { + # Attempt to stop WPR and save to the given file. If no profile is running, log and continue. + try { + & wpr -stop $outFile | Out-Null + } + catch { + Write-Host "wpr -stop failed: $($_.Exception.Message). Attempting to query status..." + try { + $s = & wpr -status 2>&1 + Write-Host "WPR status: $s" + } catch { } + } + $script:WprProfiles.Remove($Which) | Out-Null + } + catch { + Write-Host "Failed to stop WPR: $($_.Exception.Message)" + } +} + + +# ========================= +# Remote job helpers +# ========================= +function Invoke-EchoInSession { + param($Session, $RemoteDir, $Name, $Options) + + $Job = Invoke-Command -Session $Session -ScriptBlock { + param($RemoteDir, $Name, $Options, $WaitSeconds) + + Set-Location (Join-Path $RemoteDir 'echo') + + $Tool = Join-Path $RemoteDir ("echo\$Name.exe") + Write-Host "[Remote] Running: $Tool" + if ($Options -is [System.Array]) { + Write-Host "[Remote] Arguments (array):" + foreach ($arg in $Options) { Write-Host " $arg" } + $argList = $Options + } + else { + Write-Host "[Remote] Arguments (string):" + Write-Host " $Options" + $argList = @() + if (-not [string]::IsNullOrEmpty($Options)) { $argList = @($Options) } + } + + try { + # Invoke the tool directly. When a timeout is requested, run the invocation + # inside a PowerShell background job so we can enforce a timeout and cancel + # the job (and any matching process) if it doesn't finish in time. + if ($WaitSeconds -and $WaitSeconds -gt 0) { + Write-Host "[Remote] Starting tool as background job for timeout control..." + $jobScript = { + param($ToolPath, $ArgList) + if ($ArgList -is [System.Array]) { + & $ToolPath @ArgList + } + elseif (-not [string]::IsNullOrEmpty($ArgList)) { + & $ToolPath $ArgList + } + else { + & $ToolPath + } + return $LASTEXITCODE + } + + $j = Start-Job -ScriptBlock $jobScript -ArgumentList $Tool, $argList -ErrorAction Stop + Write-Host "[Remote] Started job Id=$($j.Id)" + + # Wait-Job uses seconds for timeout + $completed = $j | Wait-Job + $output = Receive-Job $j -Keep + # The job returns the tool's exit code as the last object + $rc = $output | Where-Object { ($_ -is [int]) -or ($_ -match '^[0-9]+$') } | Select-Object -Last 1 + if ($rc -eq $null) { $rc = 0 } + Write-Host "[Remote] Process (job) exited with code $rc" + if ($rc -ne 0) { throw "Remote $Tool.exe exited with code $rc" } + } + else { + Write-Host "[Remote] Running tool in foreground (no timeout)..." + if ($argList -is [System.Array]) { & $Tool @argList } elseif (-not [string]::IsNullOrEmpty($argList)) { & $Tool $argList } else { & $Tool } + Write-Host "[Remote] Process exited with code $LASTEXITCODE" + if ($LASTEXITCODE -ne 0) { throw "Remote $Tool.exe exited with code $LASTEXITCODE" } + } + } + catch { + throw "Failed to launch or monitor process $Tool $($_.Exception.Message)" + } + } -ArgumentList $RemoteDir, $Name, $Options -AsJob -ErrorAction Stop + + return $Job +} + +function Receive-JobOrThrow { + param([Parameter(Mandatory)] $Job) + + Wait-Job $Job | Out-Null + + # Drain output (keep so we can inspect again if needed) + $null = Receive-Job $Job -Keep + + $errs = @() + foreach ($cj in $Job.ChildJobs) { + if ($cj.Error -and $cj.Error.Count -gt 0) { + $errs += $cj.Error + } + if ($cj.JobStateInfo.State -eq 'Failed' -and $cj.JobStateInfo.Reason) { + $errs += $cj.JobStateInfo.Reason + } + } + + if ($errs.Count -gt 0) { + foreach ($er in $errs) { + if ($er -is [System.Management.Automation.ErrorRecord]) { + $er | Write-DetailedError + } + else { + Write-Host $er + } + } + throw "One or more remote errors occurred (job id: $($Job.Id))." + } + + if ($Job.State -eq 'Failed') { + throw "Remote job failed (job id: $($Job.Id)): $($Job.JobStateInfo.Reason)" + } +} + +# ------------------------- +# Refactored workflow functions +# ------------------------- + +function Create-Session { + param( + [Parameter(Mandatory=$true)][string]$PeerName, + [string]$RemotePSConfiguration = 'PowerShell.7' + ) + + $script:RemotePSConfiguration = $RemotePSConfiguration + $script:RemoteDir = 'C:\_work' + + $Username = (Get-ItemProperty 'HKLM:\Software\Microsoft\Windows NT\CurrentVersion\Winlogon').DefaultUserName + $Password = (Get-ItemProperty 'HKLM:\Software\Microsoft\Windows NT\CurrentVersion\Winlogon').DefaultPassword | ConvertTo-SecureString -AsPlainText -Force + $Creds = New-Object System.Management.Automation.PSCredential ($Username, $Password) + + try { + Write-Host "Creating PSSession to $PeerName using configuration '$RemotePSConfiguration'..." + $s = New-PSSession -ComputerName $PeerName -Credential $Creds -ConfigurationName $RemotePSConfiguration -ErrorAction Stop + Write-Host "Session created using configuration '$RemotePSConfiguration'." + } + catch { + Write-Host "Failed to create session using configuration '$RemotePSConfiguration': $($_.Exception.Message)" + Write-Host "Attempting fallback: creating session without ConfigurationName..." + try { + $s = New-PSSession -ComputerName $PeerName -Credential $Creds -ErrorAction Stop + Write-Host "Session created using default configuration." + } + catch { + Write-Host "Fallback session creation failed: $($_.Exception.Message)" + throw "Failed to create remote session to $PeerName" + } + } + + $script:Session = $s + return $s +} + +function Save-And-Disable-Firewalls { + param([Parameter(Mandatory=$true)]$Session) + + # Coerce possible multi-output (array) from Create-Session into the actual PSSession object. + if ($Session -is [System.Array]) { + $found = $Session | Where-Object { $_ -is [System.Management.Automation.Runspaces.PSSession] } + if ($found -and $found.Count -gt 0) { $Session = $found[0] } + else { $Session = $Session[0] } + } + + if (-not ($Session -is [System.Management.Automation.Runspaces.PSSession])) { + throw "Save-And-Disable-Firewalls requires a PSSession object. Got: $($Session.GetType().FullName) - $Session" + } + + Write-Host "Saving and disabling local firewall profiles..." + $script:localFwState = Get-NetFirewallProfile -Profile Domain, Public, Private | Select-Object Name, Enabled + Set-NetFirewallProfile -Profile Domain, Public, Private -Enabled False + + Write-Host "Disabling firewall on remote machine..." + Invoke-Command -Session $Session -ScriptBlock { + param() + $fw = Get-NetFirewallProfile -Profile Domain, Public, Private | Select-Object Name, Enabled + Set-Variable -Name __SavedFirewallState -Value $fw -Scope Global -Force + Set-NetFirewallProfile -Profile Domain, Public, Private -Enabled False + } -ErrorAction Stop +} + +function Copy-EchoToRemote { + param([Parameter(Mandatory=$true)]$Session) + # Ensure the remote base directory and the 'echo' subdirectory both exist, + # then copy the *contents* of the local directory into the remote folder. + Invoke-Command -Session $Session -ScriptBlock { + param($base, $sub) + if (-not (Test-Path $base)) { New-Item -ItemType Directory -Path $base | Out-Null } + $full = Join-Path $base $sub + if (-not (Test-Path $full)) { New-Item -ItemType Directory -Path $full | Out-Null } + } -ArgumentList $script:RemoteDir, 'echo' -ErrorAction Stop + + $localPath = (Resolve-Path .).Path + Copy-Item -ToSession $Session -Path (Join-Path $localPath '*') -Destination "$script:RemoteDir\echo" -Recurse -Force +} + +# Robust remote file fetch: try Copy-Item -FromSession, fall back to Invoke-Command/Get-Content +function Fetch-RemoteFile { + param( + [Parameter(Mandatory=$true)]$Session, + [Parameter(Mandatory=$true)][string]$RemotePath, + [Parameter(Mandatory=$true)][string]$LocalDestination + ) + + Write-Host "Fetching remote file '$RemotePath' to local '$LocalDestination'..." + + try { + Copy-Item -FromSession $Session -Path $RemotePath -Destination $LocalDestination -ErrorAction Stop + Write-Host "Successfully fetched remote file '$RemotePath' to '$LocalDestination' via Copy-Item -FromSession." + return $true + } + catch { + Write-Host "Copy-Item -FromSession failed for '$RemotePath': $($_.Exception.Message). Attempting Invoke-Command fallback..." + try { + $content = Invoke-Command -Session $Session -ScriptBlock { param($p) Get-Content -Path $p -Raw -ErrorAction Stop } -ArgumentList $RemotePath -ErrorAction Stop + if ($null -ne $content) { + $content | Out-File -FilePath $LocalDestination -Encoding utf8 -Force + return $true + } + else { + Write-Host "Invoke-Command returned no content for '$RemotePath'" + return $false + } + } + catch { + Write-Host "Failed to fetch remote file '$RemotePath' via Invoke-Command: $($_.Exception.Message)" + return $false + } + } +} + +function Run-SendTest { + param( + [Parameter(Mandatory=$true)][string]$PeerName, + [Parameter(Mandatory=$true)]$Session, + [Parameter(Mandatory=$true)][string]$SenderOptions, + [Parameter(Mandatory=$true)][string]$ReceiverOptions + ) + + $serverArgs = Convert-ArgStringToArray $ReceiverOptions + # Normalize server args + $serverArgs = Normalize-Args -Tokens $serverArgs + Write-Host "[Local->Remote] Invoking remote job with arguments:" + if ($serverArgs -is [System.Array]) { foreach ($arg in $serverArgs) { Write-Host " $arg" } } else { Write-Host " $serverArgs" } + $Job = Invoke-EchoInSession -Session $Session -RemoteDir $script:RemoteDir -Name "echo_server" -Options $serverArgs + + $clientArgs = Convert-ArgStringToArray $SenderOptions + $clientArgs = Normalize-Args -Tokens $clientArgs + + $clientArgs += @('--stats-file', 'echo_client_stats.json') + + Write-Host "[Local] Running: .\echo_client.exe" + Write-Host "[Local] Arguments:" + foreach ($a in $clientArgs) { Write-Host " $a" } + Start-WprCpuProfile -Which 'send' + & .\echo_client.exe @clientArgs + $script:localExit = $LASTEXITCODE + Stop-WprCpuProfile -Which 'send' + + Receive-JobOrThrow -Job $Job +} + +function Run-RecvTest { + param( + [Parameter(Mandatory=$true)][string]$PeerName, + [Parameter(Mandatory=$true)]$Session, + [Parameter(Mandatory=$true)][string]$SenderOptions, + [Parameter(Mandatory=$true)][string]$ReceiverOptions + ) + + $serverArgs = Convert-ArgStringToArray $SenderOptions + $serverArgs = Normalize-Args -Tokens $serverArgs + Write-Host "[Local->Remote] Invoking remote job with arguments:" + if ($serverArgs -is [System.Array]) { foreach ($arg in $serverArgs) { Write-Host " $arg" } } else { Write-Host " $serverArgs" } + $Job = Invoke-EchoInSession -Session $Session -RemoteDir $script:RemoteDir -Name "echo_client" -Options $serverArgs + + $clientArgs = Convert-ArgStringToArray $ReceiverOptions + $clientArgs = Normalize-Args -Tokens $clientArgs + + Write-Host "[Local] Running: .\echo_server.exe" + Write-Host "[Local] Arguments:" + foreach ($a in $clientArgs) { Write-Host " $a" } + Start-WprCpuProfile -Which 'recv' + & .\echo_server.exe @clientArgs + $script:localExit = $LASTEXITCODE + Stop-WprCpuProfile -Which 'recv' + + Receive-JobOrThrow -Job $Job +} + +function CaptureIndividualCpuUsagePerformanceMonitorAsJob { + param( + [Parameter(Mandatory=$true)][string]$DurationSeconds + ) + + # Ensure we pass a numeric duration into the job and use that value inside + $intDuration = [int]::Parse($DurationSeconds) + + $cpuMonitorJob = Start-Job -ScriptBlock { + param($duration) + + # Use the Processor Information counter which contains CPU instances across all groups + # (e.g., "0,0", "0,1", "1,0" etc.) so we capture CPUs from every group, not just group 0. + $counter = '\Processor Information(*)\% Processor Time' + $d = [int]$duration + + try { + $samples = Get-Counter -Counter $counter -SampleInterval 1 -MaxSamples $d -ErrorAction Stop + # Group samples by instance (processor information name) and compute average per instance. + # InstanceName for Processor Information uses formats like "0,0" (group,index) or descriptive names. + $grouped = $samples.CounterSamples | Group-Object -Property InstanceName + $results = @() + foreach ($g in $grouped) { + $instName = $g.Name + # Normalize instance names: skip the _Total instance and any empty names + if ([string]::IsNullOrEmpty($instName) -or $instName -eq '_Total') { continue } + $vals = $g.Group | ForEach-Object { [double]$_.CookedValue } + $avg = ($vals | Measure-Object -Average).Average + $results += [PSCustomObject]@{ Processor = $instName; Average = $avg } + } + # Sort by numeric ordering where possible, otherwise by name for consistent output + $sorted = $results | Sort-Object @{Expression={ + $n = $_.Processor -replace '[^0-9,]','' + # If the processor string contains a comma (group,index), split and compute a sortable key + if ($n -match ',') { $parts = $n -split ','; return ([int]$parts[0]*1000 + [int]$parts[1]) } + if ($n -match '^[0-9]+$') { return [int]$n } + return $_.Processor + }},Processor + # Emit numeric array of per-CPU numeric averages + $numeric = $sorted | ForEach-Object { [double]$_.Average } + } + catch { + $numeric = @(0) + } + + # Emit the numeric array so the caller receives per-CPU averages + Write-Output $numeric + } -ArgumentList $intDuration + + return $cpuMonitorJob +} + + +function CapturePerformanceMonitorAsJob { + param( + [Parameter(Mandatory=$true)][string]$DurationSeconds, + [Parameter(Mandatory=$false)][string[]]$Counters = @('\Processor Information(*)\% Processor Time') + ) + + # Ensure numeric duration + $intDuration = [int]::Parse($DurationSeconds) + + $perfJob = Start-Job -ScriptBlock { + param($duration, $counters) + + $d = [int]$duration + if (-not $counters -or $counters.Count -eq 0) { + $counters = @('\Processor Information(*)\% Processor Time') + } + + # Sample once per second for the requested duration and accumulate per-counter values. + $store = @{} + + for ($i = 0; $i -lt $d; $i++) { + $samples = $null + try { + # Try to collect all counters in a single quick sample (returns immediately) + $samples = Get-Counter -Counter $counters -MaxSamples 1 -ErrorAction Stop + } + catch { + # If that fails, collect available counters individually (quick single-sample calls) + $samples = New-Object System.Collections.Generic.List[object] + foreach ($c in $counters) { + try { + $s = Get-Counter -Counter $c -MaxSamples 1 -ErrorAction Stop + if ($s.CounterSamples) { $s.CounterSamples | ForEach-Object { [void]$samples.Add($_) } } + } + catch { + # skip bad counter + continue + } + } + } + + if ($samples -ne $null) { + $csamples = $samples.CounterSamples + if (-not $csamples -and ($samples -is [System.Collections.IEnumerable])) { $csamples = $samples } + foreach ($cs in $csamples) { + $path = $cs.Path + $inst = $cs.InstanceName + if ([string]::IsNullOrEmpty($inst) -or $inst -eq '_Total') { continue } + if ($cs.Status -ne 'Success') { continue } + $key = "$path`|$inst" + if (-not $store.ContainsKey($key)) { $store[$key] = New-Object System.Collections.ArrayList } + [void]$store[$key].Add([double]$cs.CookedValue) + } + } + + Start-Sleep -Seconds 1 + } + + $results = @() + foreach ($k in $store.Keys) { + $parts = $k -split '\|',2 + $path = $parts[0] + $inst = $parts[1] + $avg = ($store[$k] | Measure-Object -Average).Average + $results += [PSCustomObject]@{ Counter = $path; Instance = $inst; Average = $avg } + } + + # Emit structured results: an array of PSObjects with Counter, Instance, Average + $results + } -ArgumentList $intDuration, $Counters + + return $perfJob +} + + +function Restore-FirewallAndCleanup { + param([object]$Session) + + try { + if ($null -ne $Session) { + try { + Write-Host "Restoring firewall state on remote machine..." + Invoke-Command -Session $Session -ScriptBlock { + if (Get-Variable -Name __SavedFirewallState -Scope Global -ErrorAction SilentlyContinue) { + $saved = Get-Variable -Name __SavedFirewallState -Scope Global -ValueOnly + foreach ($p in $saved) { + Set-NetFirewallProfile -Profile $p.Name -Enabled $p.Enabled + } + Remove-Variable -Name __SavedFirewallState -Scope Global -ErrorAction SilentlyContinue + } + else { + Set-NetFirewallProfile -Profile Domain, Public, Private -Enabled True + } + } -ErrorAction SilentlyContinue + } + catch { + $_ | Write-DetailedError + } + + try { + Remove-PSSession $Session -ErrorAction SilentlyContinue + } + catch { + $_ | Write-DetailedError + } + } + + Write-Host "Restoring local firewall state..." + if ($localFwState) { + foreach ($p in $localFwState) { + Set-NetFirewallProfile -Profile $p.Name -Enabled $p.Enabled + } + } + else { + Set-NetFirewallProfile -Profile Domain, Public, Private -Enabled True + } + } + catch { + $_ | Write-DetailedError + } +} + +$PerformanceCounters = +@( + '\UDPv4\Datagrams Received Errors', + '\UDPv6\Datagrams Received Errors', + + '\Network Interface(*)\Packets Received Errors', + '\Network Interface(*)\Packets Received Discarded', + '\Network Interface(*)\Packets Outbound Discarded', + + '\IPv4\Datagrams Received Discarded', + '\IPv4\Datagrams Received Header Errors', + '\IPv4\Datagrams Received Address Errors', + + '\IPv6\Datagrams Received Discarded', + '\IPv6\Datagrams Received Header Errors', + '\IPv6\Datagrams Received Address Errors', + + '\WFPv4\Packets Discarded/sec', + '\WFPv6\Packets Discarded/sec', + '\Processor Information(*)\% Processor Time' +) + +# ========================= +# Main workflow +# ========================= +try { + + # Print the current working directory + $cwd = (Get-Location).Path + Write-Host "Current working directory: $cwd" + + Get-NetAdapterRss + + Write-Host "\nStarting echo tests to peer '$PeerName' with duration $Duration seconds..." + + # Create remote session + $Session = Create-Session -PeerName $PeerName -RemotePSConfiguration 'PowerShell.7' + + # Save and disable firewalls + Save-And-Disable-Firewalls -Session $Session + + # Copy tool to remote + Copy-EchoToRemote -Session $Session + + # Launch per-CPU usage monitor as a background job (returns array of per-CPU averages) + $cpuMonitorJob = CaptureIndividualCpuUsagePerformanceMonitorAsJob $Duration + $perfCounterJob = CapturePerformanceMonitorAsJob -DurationSeconds $Duration -Counters $PerformanceCounters + + # Run tests + Run-SendTest -PeerName $PeerName -Session $Session -SenderOptions $SenderOptions -ReceiverOptions $ReceiverOptions + + # Recover CPU usage data (monitor returns per-CPU averages). Print per-CPU values. + $cpuUsagePerCpu = Receive-Job -Job $cpuMonitorJob -Wait -AutoRemoveJob + if ($cpuUsagePerCpu -is [System.Array]) { + $i = 0 + foreach ($val in $cpuUsagePerCpu) { + $i++ + Write-Host "CPU$i $([math]::Round([double]$val, 2)) %" + } + # Compute and print overall average across all CPUs + $overall = (($cpuUsagePerCpu | Measure-Object -Average).Average) + Write-Host "Overall average CPU Usage: $([math]::Round($overall, 2)) %" + } + else { + Write-Host "CPU1 $([math]::Round([double]$cpuUsagePerCpu, 2)) %" + } + + # Write the performance counter results as a JSON file + $perfResults = Receive-Job -Job $perfCounterJob -Wait -AutoRemoveJob + $perfJsonPath = Join-Path $cwd 'echo_client_perf_counters.json' + $perfResults | ConvertTo-Json | Out-File -FilePath $perfJsonPath -Encoding utf8 -Force + + # Launch another per-CPU usage monitor for the recv test + $cpuMonitorJob = CaptureIndividualCpuUsagePerformanceMonitorAsJob $Duration + $perfCounterJob = CapturePerformanceMonitorAsJob -DurationSeconds $Duration -Counters $PerformanceCounters + + Run-RecvTest -PeerName $PeerName -Session $Session -SenderOptions $SenderOptions -ReceiverOptions $ReceiverOptions + + # Recover CPU usage data (monitor returns per-CPU averages). Print per-CPU values. + $cpuUsagePerCpu = Receive-Job -Job $cpuMonitorJob -Wait -AutoRemoveJob + if ($cpuUsagePerCpu -is [System.Array]) { + $i = 0 + foreach ($val in $cpuUsagePerCpu) { + $i++ + Write-Host "CPU$i $([math]::Round([double]$val, 2)) %" + } + # Compute and print overall average across all CPUs + $overall = (($cpuUsagePerCpu | Measure-Object -Average).Average) + Write-Host "Overall average CPU Usage: $([math]::Round($overall, 2)) %" + } + else { + Write-Host "CPU1 $([math]::Round([double]$cpuUsagePerCpu, 2)) %" + } + + # Write the performance counter results as a JSON file + $perfResults = Receive-Job -Job $perfCounterJob -Wait -AutoRemoveJob + $perfJsonPath = Join-Path $cwd 'echo_server_perf_counters.json' + $perfResults | ConvertTo-Json | Out-File -FilePath $perfJsonPath -Encoding utf8 -Force + + # List json files in cwd + Write-Host "JSON files in $cwd" + Get-ChildItem -Path $cwd -Filter *.json | ForEach-Object { Write-Host " $($_.FullName)" } + + # Print each JSON file's contents + Get-ChildItem -Path $cwd -Filter *.json | ForEach-Object { + Write-Host "Contents of $($_.FullName) - " + Get-Content -Path $_.FullName | ForEach-Object { Write-Host " $_" } + } + + # Copy the stats file to the parent folder for GitHub Actions artifact upload + Copy-Item -Path *.json -Destination $cwd\.. -Force + + Write-Host "echo tests completed successfully." +} +catch { + # $_ is an ErrorRecord; print everything useful + Write-Host "echo tests failed." + Write-Host $_ + $exitCode = 2 +} +finally { + # Use refactored cleanup function + Restore-FirewallAndCleanup -Session $Session + Write-Host "Exiting with code $exitCode" + exit $exitCode +} + +<# + Troubleshooting notes for PowerShell Remoting errors: + + - Add remote host to TrustedHosts (run as Administrator on the client): + ```powershell + Set-Item WSMan:\localhost\Client\TrustedHosts -Value "alanjo-test-2" -Force + # or allow all (less secure): + Set-Item WSMan:\localhost\Client\TrustedHosts -Value "*" -Force + ``` + + - Enable WinRM on the remote machine (run as Administrator on the remote): + ```powershell + Enable-PSRemoting -Force + # Ensure the WinRM service is running: + Set-Service WinRM -StartupType Automatic + Start-Service WinRM + ``` + + - If using PowerShell 7 session configuration, register it on the remote (run on remote machine in PowerShell 7): + ```powershell + # Register a PowerShell 7 endpoint named 'PowerShell.7' + Register-PSSessionConfiguration -Name PowerShell.7 -RunAsCredential (Get-Credential) -Force + # Or use pwsh's implicit registration helper if available: + # pwsh -NoProfile -Command "Register-PSSessionConfiguration -Name PowerShell.7 -Force" + ``` + + - For HTTPS transport, create an HTTPS listener and configure an SSL cert for WinRM on the remote. See `about_Remote_Troubleshooting`. + + Note: Adding hosts to TrustedHosts weakens authentication; prefer HTTPS or domain-joined Kerberos where possible. + #> diff --git a/.github/workflows/ebpf.yml b/.github/workflows/ebpf.yml index 285bc3a9..df4222aa 100644 --- a/.github/workflows/ebpf.yml +++ b/.github/workflows/ebpf.yml @@ -226,7 +226,7 @@ jobs: with: name: "cts-traffic Release" path: ${{ github.workspace }}\cts-traffic - + - name: Start TCPIP tracing - Baseline if: ${{ github.event.inputs.tcp_ip_tracing }} run: | @@ -235,6 +235,7 @@ jobs: Invoke-WebRequest -uri "https://raw.githubusercontent.com/microsoft/netperf/main/.github/workflows/tcpip.wprp" -OutFile "tcpip.wprp" wpr -start tcpip.wprp -filemode + # Run CTS traffic without XDP installed to establish a baseline. - name: Run CTS cts-traffic baseline working-directory: ${{ github.workspace }}\cts-traffic diff --git a/.github/workflows/network-traffic-performance-linux.yml b/.github/workflows/network-traffic-performance-linux.yml new file mode 100644 index 00000000..5a1bfb3f --- /dev/null +++ b/.github/workflows/network-traffic-performance-linux.yml @@ -0,0 +1,142 @@ +name: network-traffic-performance-linux + +on: + workflow_dispatch: + inputs: + ref: + description: 'Test Branch or Commit' + required: false + default: 'main' + type: string + sender_options: + description: 'Command-line options for the sender (quoted string)' + required: true + type: string + receiver_options: + description: 'Command-line options for the receiver (quoted string)' + required: true + type: string + test_tool_ref: + description: 'Branch or ref to build the test tool from' + required: false + default: 'main' + type: string + duration: + description: 'Duration of the test in seconds' + required: false + default: '60' + type: string + +permissions: write-all + +jobs: + build_echo_test_linux: + name: Build echo server/client (Linux) + uses: Alan-Jowett/LinuxUDPShardedEcho/.github/workflows/reusable-build.yml@main + with: + artifact_name: echo_test + repository: 'Alan-Jowett/LinuxUDPShardedEcho' + config: 'RelWithDebInfo' + ref: ${{ inputs.test_tool_ref }} + + test_echo_server_linux: + name: Network Traffic Test (echo, Linux) + needs: [build_echo_test_linux] + strategy: + fail-fast: false + matrix: + vec: + - env: lab + os: "ubuntu-24.04" + arch: x64 + runs-on: + - self-hosted + - ${{ matrix.vec.env }} + - Linux + - ${{ matrix.vec.arch }} + - os-${{ matrix.vec.os }} + + steps: + - name: Checkout repository + uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 + + - name: Ensure PowerShell 7 is installed (Linux) + shell: bash + run: | + if ! command -v pwsh >/dev/null 2>&1; then + sudo bash ./install-pwsh.sh + fi + + - name: Setup workspace (Linux) + shell: pwsh + run: | + Get-ChildItem | % { Remove-Item -Recurse $_ -Force -ErrorAction SilentlyContinue } + if (Test-Path "$env:GITHUB_WORKSPACE/echo") { Remove-Item -Recurse -Force "$env:GITHUB_WORKSPACE/echo" } + if (Test-Path "$env:GITHUB_WORKSPACE/ETL") { Remove-Item -Recurse -Force "$env:GITHUB_WORKSPACE/ETL" } + New-Item -ItemType Directory -Path "$env:GITHUB_WORKSPACE/echo/build" | Out-Null + New-Item -ItemType Directory -Path "$env:GITHUB_WORKSPACE/ETL" | Out-Null + + - name: Download echo test tool + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + with: + name: "echo_test" + path: ${{ github.workspace }}/echo + + - name: Download run-echo-test script + shell: pwsh + run: | + Copy-Item -Path "${{ github.workspace }}/.github/scripts/run-echo-test-linux.ps1" -Destination "${{ github.workspace }}/echo/build/run-echo-test-linux.ps1" + + - name: Log input parameters + shell: pwsh + run: | + Write-Output "Input parameters:" + Write-Output " SenderOptions: ${{ inputs.sender_options }}" + Write-Output " ReceiverOptions: ${{ inputs.receiver_options }}" + Write-Output " ref: ${{ inputs.ref }}" + Write-Output " Duration: ${{ inputs.duration }}" + + - name: Diagnostic - List files in echo build directory + shell: pwsh + run: | + Write-Output "Files in echo build directory:" + Get-ChildItem -Path "$env:GITHUB_WORKSPACE/echo/build" -Recurse + + - name: Run echo traffic tests (Linux) + working-directory: ${{ github.workspace }}/echo/build + env: + SENDER_OPTIONS: ${{ inputs.sender_options }} + RECEIVER_OPTIONS: ${{ inputs.receiver_options }} + shell: pwsh + run: | + $cmd = @( + './run-echo-test-linux.ps1', + '-PeerName', '"netperf-peer"', + '-SenderOptions', '"' + $env:SENDER_OPTIONS + '"', + '-ReceiverOptions', '"' + $env:RECEIVER_OPTIONS + '"', + '-Duration', '"${{ inputs.duration }}"' + ) -join ' ' + + Write-Output "Running: $cmd" + Invoke-Expression $cmd + + - name: Upload echo results (Linux) + if: always() + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 + with: + name: echo_test_linux_${{ matrix.vec.env }}_${{ matrix.vec.os }}_${{ matrix.vec.arch }} + path: | + ${{ github.workspace }}/echo/*.csv + ${{ github.workspace }}/echo/*.log + ${{ github.workspace }}/echo/*.json + ${{ github.workspace }}/echo/build/echo_summary.csv + ${{ github.workspace }}/echo/build/echo_client_output.txt + ${{ github.workspace }}/echo/build/server.log + + attempt-reset-lab-linux: + name: Attempting to reset lab (Linux). Status of this job does not indicate result of lab reset. Look at job details. + needs: [test_echo_server_linux] + if: ${{ always() }} + uses: microsoft/netperf/.github/workflows/schedule-lab-reset.yml@main + with: + workflowId: ${{ github.run_id }} diff --git a/.github/workflows/network-traffic-performance.yml b/.github/workflows/network-traffic-performance.yml index 36455472..edf336b5 100644 --- a/.github/workflows/network-traffic-performance.yml +++ b/.github/workflows/network-traffic-performance.yml @@ -27,23 +27,46 @@ on: description: 'Command-line options for the receiver (quoted string)' required: true type: string + test_tool_ref: + description: 'Branch or ref to build the test tool from' + required: false + default: 'main' + type: string + test_tool: + description: 'The network test tool to use' + required: false + default: 'ctsTraffic' + type: choice + options: + - ctsTraffic + - echo + duration: + description: 'Duration of the test in seconds' + required: false + default: '60' + type: string -concurrency: - group: >- - traffic-${{ github.event.client_payload.sha || inputs.ref || github.event.pull_request.number || 'main' }} - cancel-in-progress: true +permissions: write-all + +# concurrency: +# group: >- +# traffic-${{ github.event.client_payload.sha || inputs.ref || github.event.pull_request.number || 'main' }} +# cancel-in-progress: true jobs: build_cts_traffic: + if: ${{ inputs.test_tool == 'ctsTraffic' }} name: Build cts-traffic uses: microsoft/ctsTraffic/.github/workflows/reusable-build.yml@master with: build_artifact: cts-traffic repository: 'microsoft/ctsTraffic' configurations: '["Release"]' - ref: 'master' + # Need to translate main to master for ctsTraffic repo + ref: ${{ inputs.test_tool_ref == 'main' && 'master' || inputs.test_tool_ref }} - test: + test_cts_traffic: + if: ${{ inputs.test_tool == 'ctsTraffic' }} name: Network Traffic Test needs: [build_cts_traffic] strategy: @@ -149,4 +172,204 @@ jobs: uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 with: name: cpu_profile_${{ matrix.vec.env }}_${{ matrix.vec.os }}_${{ matrix.vec.arch }} - path: ${{ github.workspace }}\ETL\cpu_profile-*.etl \ No newline at end of file + path: ${{ github.workspace }}\ETL\cpu_profile-*.etl + + - name: Attempt to reset lab (trigger schedule-lab-reset) + if: always() + uses: peter-evans/workflow-dispatch@v1 + with: + # Repository containing the reusable reset workflow + repository: microsoft/netperf + workflow: .github/workflows/schedule-lab-reset.yml + ref: main + # Forward current run id so the target workflow can correlate + inputs: | + workflowId: ${{ github.run_id }} + token: ${{ secrets.RESET_WORKFLOW_TOKEN }} + + build_echo_test: + if: ${{ inputs.test_tool == 'echo' }} + name: Build echo server + uses: alan-jowett/WinUDPShardedEcho/.github/workflows/reusable-build.yml@main + with: + artifact_name: echo_test + repository: 'alan-jowett/WinUDPShardedEcho' + config: 'RelWithDebInfo' + ref: ${{ inputs.test_tool_ref }} + + test_echo_server: + if: ${{ inputs.test_tool == 'echo' }} + name: Network Traffic Test (echo) + needs: [build_echo_test] + strategy: + fail-fast: false + matrix: + vec: + - env: lab + os: "2022" + arch: x64 + runs-on: + - self-hosted + - ${{ matrix.vec.env }} + - os-windows-${{ matrix.vec.os }} + - ${{ matrix.vec.arch }} + - ebpf + + steps: + - name: Checkout repository + uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 + + - name: Setup workspace + run: |- + Get-ChildItem | % { Remove-Item -Recurse $_ -Force -ErrorAction SilentlyContinue } + if (Test-Path ${{ github.workspace }}\echo) { Remove-Item -Recurse -Force ${{ github.workspace }}\echo } + if (Test-Path ${{ github.workspace }}\ETL) { Remove-Item -Recurse -Force ${{ github.workspace }}\ETL } + New-Item -ItemType Directory -Path ${{ github.workspace }}\echo\RelWithDebInfo + New-Item -ItemType Directory -Path ${{ github.workspace }}\ETL + + - name: Configure Windows Error Reporting to make a local copy of any crashes that occur. + id: configure_windows_error_reporting + run: | + $DumpFolder = "${{ github.workspace }}\echo\RelWithDebInfo" + New-Item -Path "HKLM:\SOFTWARE\Microsoft\Windows\Windows Error Reporting\LocalDumps" -ErrorAction SilentlyContinue + New-ItemProperty -Path "HKLM:\SOFTWARE\Microsoft\Windows\Windows Error Reporting\LocalDumps" -Name "DumpType" -Value 2 -PropertyType DWord -ErrorAction SilentlyContinue + New-ItemProperty -Path "HKLM:\SOFTWARE\Microsoft\Windows\Windows Error Reporting\LocalDumps" -Name "DumpFolder" -Value "$DumpFolder" -PropertyType ExpandString -ErrorAction SilentlyContinue + + - name: Download echo test tool + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + with: + name: "echo_test" + path: ${{ github.workspace }}\echo + + - name: Optional - Start TCPIP tracing + if: ${{ github.event.inputs.tcp_ip_tracing == 'true' }} + run: |- + if (Test-Path "tcpip.wprp") { Remove-Item -Force "tcpip.wprp" } + Invoke-WebRequest -uri "https://raw.githubusercontent.com/microsoft/netperf/${{inputs.ref}}/.github/workflows/tcpip.wprp" -OutFile "tcpip.wprp" + wpr -start tcpip.wprp -filemode + + - name: Download modified CPU profile script + working-directory: ${{ github.workspace }}\ETL + run: | + Invoke-WebRequest -uri "https://raw.githubusercontent.com/microsoft/netperf/${{inputs.ref}}/.github/scripts/cpu.wprp" -OutFile "${{ github.workspace }}\ETL\cpu.wprp" + + - name: Download run-echo-test script + working-directory: ${{ github.workspace }}\echo\RelWithDebInfo + run: | + Invoke-WebRequest -uri "https://raw.githubusercontent.com/microsoft/netperf/${{inputs.ref}}/.github/scripts/run-echo-test.ps1" -OutFile ".\run-echo-test.ps1" + + - name: Log all input parameters for this workflow + run: | + Write-Output "Input parameters:" + Write-Output " Profile: ${{ inputs.profile }}" + Write-Output " SenderOptions: ${{ inputs.sender_options }}" + Write-Output " ReceiverOptions: ${{ inputs.receiver_options }}" + Write-Output " TCP IP Tracing: ${{ inputs.tcp_ip_tracing }}" + Write-Output " ref: ${{ inputs.ref }}" + Write-Output " Duration: ${{ inputs.duration }}" + + - name: Diagonstic - List files in echo RelWithDebInfo directory + run: | + Write-Output "Files in echo RelWithDebInfo directory:" + Get-ChildItem -Path ${{ github.workspace }}\echo\RelWithDebInfo -Recurse + + - name: Log pktmon drop counters before test + run: | + pktmon comp counters --type drop + + - name: Start PktMon drop Capture + run: | + pktmon filter add -p 7 + pktmon filter list + pktmon start --etw -c + + - name: Run traffic tests + working-directory: ${{ github.workspace }}\echo\RelWithDebInfo + env: + PROFILE: ${{ inputs.profile }} + SENDER_OPTIONS: ${{ inputs.sender_options }} + RECEIVER_OPTIONS: ${{ inputs.receiver_options }} + shell: pwsh + run: | + # Build the command and only include -CpuProfile when PROFILE is truthy ('true' or true) + $cpuSwitch = '' + if ($env:PROFILE -eq 'true' -or $env:PROFILE -eq 'True' -or $env:PROFILE -eq 'TRUE' -or $env:PROFILE -eq $true) { + $cpuSwitch = '-CpuProfile' + } + + $cmd = @( + '.\\run-echo-test.ps1', + $cpuSwitch, + '-PeerName', '"netperf-peer"', + '-SenderOptions', '"' + $env:SENDER_OPTIONS + '"', + '-ReceiverOptions', '"' + $env:RECEIVER_OPTIONS + '"' + '-Duration ', '"${{ inputs.duration }}"' + ) -join ' ' + + Write-Output "Running: $cmd" + Invoke-Expression $cmd + + - name: Stop PktMon drop Capture + run: | + pktmon stop + copy pktmon.etl ${{ github.workspace }}\ETL\pktmon_drop_trace.etl + pktmon reset + pktmon filter remove all + + - name: Log pktmon drop counters after test + run: | + pktmon comp counters --type drop + + - name: Optional - Stop TCPIP tracing + if: ${{ github.event.inputs.tcp_ip_tracing == 'true' }} + run: | + wpr -stop ${{ github.workspace }}\ETL\tcpip_trace.etl + + - name: Upload echo results + if: always() + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 + with: + name: echo_test_${{ matrix.vec.env }}_${{ matrix.vec.os }}_${{ matrix.vec.arch }} + path: | + ${{ github.workspace }}\echo\*.csv + ${{ github.workspace }}\echo\*.log + ${{ github.workspace }}\echo\*.json + + - name: Upload TCPIP ETL + if: always() + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 + with: + name: tcpip_trace_${{ matrix.vec.env }}_${{ matrix.vec.os }}_${{ matrix.vec.arch }} + path: ${{ github.workspace }}\ETL\tcpip_trace.etl + + - name: Upload CPU Profile + if: always() + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 + with: + name: cpu_profile_${{ matrix.vec.env }}_${{ matrix.vec.os }}_${{ matrix.vec.arch }} + path: ${{ github.workspace }}\ETL\cpu_profile-*.etl + + - name: Upload PktMon Drop ETL + if: always() + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 + with: + name: pktmon_drop_trace_${{ matrix.vec.env }}_${{ matrix.vec.os }}_${{ matrix.vec.arch }} + path: ${{ github.workspace }}\ETL\pktmon_drop_trace.etl + + - name: Upload Crash Dumps + if: always() + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 + with: + name: crash_dumps_${{ matrix.vec.env }}_${{ matrix.vec.os }}_${{ matrix.vec.arch }} + path: | + ${{ github.workspace }}\echo\RelWithDebInfo + + attempt-reset-lab: + name: Attempting to reset lab. Status of this job does not indicate result of lab reset. Look at job details. + needs: [test_cts_traffic, test_echo_server] + if: ${{ always() }} + uses: microsoft/netperf/.github/workflows/schedule-lab-reset.yml@main + with: + workflowId: ${{ github.run_id }} + + \ No newline at end of file