From 9e57ca7089a29536ef1916861cbc52754002c3c3 Mon Sep 17 00:00:00 2001 From: Alan Jowett Date: Tue, 25 Nov 2025 16:52:32 -0800 Subject: [PATCH 01/70] workflow: add cts_traffic_ref input to build cts-traffic from custom branch --- .github/workflows/network-traffic-performance.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/network-traffic-performance.yml b/.github/workflows/network-traffic-performance.yml index 36455472..d2eba9d1 100644 --- a/.github/workflows/network-traffic-performance.yml +++ b/.github/workflows/network-traffic-performance.yml @@ -27,6 +27,11 @@ on: description: 'Command-line options for the receiver (quoted string)' required: true type: string + cts_traffic_ref: + description: 'Branch or ref to build cts-traffic from (repo: microsoft/ctsTraffic)' + required: false + default: 'master' + type: string concurrency: group: >- @@ -41,7 +46,7 @@ jobs: build_artifact: cts-traffic repository: 'microsoft/ctsTraffic' configurations: '["Release"]' - ref: 'master' + ref: ${{ inputs.cts_traffic_ref }} test: name: Network Traffic Test From 259c74f211db01cef2baa869b6e49df707bca5c0 Mon Sep 17 00:00:00 2001 From: Alan Jowett Date: Mon, 1 Dec 2025 17:49:11 -0800 Subject: [PATCH 02/70] Add option to run echo tools Signed-off-by: Alan Jowett --- .github/scripts/run-echo-test.ps1 | 617 ++++++++++++++++++ .../workflows/network-traffic-performance.yml | 131 +++- 2 files changed, 746 insertions(+), 2 deletions(-) create mode 100644 .github/scripts/run-echo-test.ps1 diff --git a/.github/scripts/run-echo-test.ps1 b/.github/scripts/run-echo-test.ps1 new file mode 100644 index 00000000..746a5833 --- /dev/null +++ b/.github/scripts/run-echo-test.ps1 @@ -0,0 +1,617 @@ +param( + [switch]$CpuProfile, + [string]$PeerName, + [string]$SenderOptions, + [string]$ReceiverOptions +) + +Set-StrictMode -Version Latest + +# Write out the parameters for logging +Write-Host "Parameters:" +Write-Host " CpuProfile: $CpuProfile" +Write-Host " PeerName: $PeerName" +Write-Host " SenderOptions: $SenderOptions" +Write-Host " ReceiverOptions: $ReceiverOptions" + +# Add --server to the sender/client options if not already present +if ($SenderOptions -notmatch '--server') { + $SenderOptions += " --server $PeerName" +} + +# Make errors terminate so catch can handle them +$ErrorActionPreference = 'Stop' +$Session = $null +$exitCode = 0 + +# Ensure local firewall state variable exists so cleanup never errors +$localFwState = $null + +# Helper to parse quoted command-line option strings into an array +function Convert-ArgStringToArray($s) { + if ([string]::IsNullOrEmpty($s)) { return @() } + # Pattern allows quoted strings with backslash-escaped characters, or unquoted tokens + # Matches either: "( (?: \\. | [^"\\] )* )" or [^"\s]+ + $pattern = '("((?:\\.|[^"\\])*)"|[^"\s]+)' + $regexMatches = [regex]::Matches($s, $pattern) + $out = @() + foreach ($m in $regexMatches) { + if ($m.Groups[2].Success) { + # Quoted token; Group 2 contains inner text with possible escapes + $val = $m.Groups[2].Value + # Unescape backslash-escaped sequences commonly used in CLI args + $val = $val -replace '\\', '\' + $val = $val -replace '\"', '"' + } + else { + # Unquoted token in Group 1 + $val = $m.Groups[1].Value + } + $out += $val.Trim() + } + return $out +} + +# Ensure an args array contains a '-target:' entry; replace if present, append if missing +function Set-TargetArg { + param( + [Parameter(Mandatory = $true)] $ArgsArray, + [Parameter(Mandatory = $true)] [string] $TargetName + ) + if ($null -eq $ArgsArray) { $ArgsArray = @() } + $found = $false + for ($i = 0; $i -lt $ArgsArray.Count; $i++) { + if ($ArgsArray[$i] -match '^-target:' ) { + $ArgsArray[$i] = "-target:$TargetName" + $found = $true + break + } + } + if (-not $found) { $ArgsArray += "-target:$TargetName" } + return $ArgsArray +} + +# Normalize tokens: prefix '-' only for standalone tokens that don't look like values +function Normalize-Args { + param([Parameter(Mandatory=$true)][object[]]$Tokens) + if ($null -eq $Tokens) { return @() } + $out = @() + for ($i = 0; $i -lt $Tokens.Count; $i++) { + $t = $Tokens[$i] + if ([string]::IsNullOrEmpty($t)) { continue } + + # Keep tokens that already start with '-' or contain '=' as-is + if ($t -like '-*' -or $t -match '=') { + $out += $t + continue + } + + # If previous token exists and starts with '-', treat this token as that option's value + if ($i -gt 0 -and ($Tokens[$i-1] -is [string]) -and ($Tokens[$i-1] -like '-*')) { + $out += $t + continue + } + + # Otherwise prefix a single '-' + $out += ('-' + $t) + } + return $out +} + +# Rename a local file if it exists; ignore if not present +function Rename-LocalIfExists { + param( + [Parameter(Mandatory=$true)][string]$Path, + [Parameter(Mandatory=$true)][string]$NewName + ) + try { + if (Test-Path $Path) { + # If the desired new name already exists, remove it first so Rename-Item succeeds + if (Test-Path $NewName) { + try { Remove-Item -Path $NewName -Force -ErrorAction Stop } catch { Write-Host "Warning: failed to remove existing '$NewName': $($_.Exception.Message)" } + } + Rename-Item -Path $Path -NewName $NewName -ErrorAction Stop + } + } + catch { + Write-Host "Failed to rename $Path -> $NewName $($_.Exception.Message)" + } +} + +# Print detailed information for an ErrorRecord or Exception. Supports pipeline input. +function Write-DetailedError { + param( + [Parameter(ValueFromPipeline=$true, ValueFromPipelineByPropertyName=$true)] $InputObject + ) + + process { + $er = $InputObject + if ($null -eq $er) { return } + if ($er -is [System.Management.Automation.ErrorRecord]) { + Write-Host "ERROR: $($er.Exception.Message)" + if ($er.Exception.StackTrace) { Write-Host "StackTrace: $($er.Exception.StackTrace)" } + if ($er.InvocationInfo) { Write-Host "Invocation: $($er.InvocationInfo.PositionMessage)" } + Write-Host "ErrorRecord: $er" + } + elseif ($er -is [System.Exception]) { + Write-Host "EXCEPTION: $($er.Message)" + if ($er.StackTrace) { Write-Host "StackTrace: $($er.StackTrace)" } + } + else { + Write-Host $er + } + } +} + +# WPR CPU profiling helpers +$script:WprProfiles = @{} + +function Start-WprCpuProfile { + param([Parameter(Mandatory=$true)][string]$Which) + + if (-not $CpuProfile) { return } + + if (-not $Workspace) { $Workspace = $env:GITHUB_WORKSPACE } + $etlDir = Join-Path $Workspace 'ETL' + if (-not (Test-Path $etlDir)) { New-Item -ItemType Directory -Path $etlDir | Out-Null } + + $outFile = Join-Path $etlDir ("cpu_profile-$Which.etl") + if (Test-Path $outFile) { Remove-Item $outFile -Force -ErrorAction SilentlyContinue } + + Write-Host "Starting WPR CPU profiling -> $outFile" + try { + # Check if WPR is already running to avoid the "profiles are already running" error + $status = $null + try { + $status = & wpr -status 2>&1 + } catch { + $status = $_.ToString() + } + + if ($status -and $status -match 'profile(s)?\s+are\s+already\s+running|Profiles are already running|The profiles are already running') { + Write-Host "WPR already running. Cancelling any existing profiles so we can start a fresh one..." + try { + & wpr -cancel 2>&1 | Out-Null + Start-Sleep -Seconds 1 + } + catch { + Write-Host "Failed to cancel existing WPR session: $($_.Exception.Message). Proceeding to start a new profile anyway." + } + } + + try { + & wpr -start CPU -filemode | Out-Null + } + catch { + Write-Host "wpr -start with custom profile failed: $($_.Exception.Message). Falling back to built-in CPU profile." + try { & wpr -start CPU -filemode | Out-Null } catch { Write-Host "Fallback CPU start also failed: $($_.Exception.Message)" } + } + $script:WprProfiles[$Which] = $outFile + } + catch { + Write-Host "Failed to start WPR: $($_.Exception.Message)" + } +} + +function Stop-WprCpuProfile { + param([Parameter(Mandatory=$true)][string]$Which) + + if (-not $CpuProfile) { return } + + if (-not $script:WprProfiles.ContainsKey($Which)) { + Write-Host "No WPR profile active for '$Which'" + return + } + + $outFile = $script:WprProfiles[$Which] + Write-Host "Stopping WPR CPU profiling, saving to $outFile" + try { + # Attempt to stop WPR and save to the given file. If no profile is running, log and continue. + try { + & wpr -stop $outFile | Out-Null + } + catch { + Write-Host "wpr -stop failed: $($_.Exception.Message). Attempting to query status..." + try { + $s = & wpr -status 2>&1 + Write-Host "WPR status: $s" + } catch { } + } + $script:WprProfiles.Remove($Which) | Out-Null + } + catch { + Write-Host "Failed to stop WPR: $($_.Exception.Message)" + } +} + + +# ========================= +# Remote job helpers +# ========================= +function Invoke-EchoInSession { + param($Session, $RemoteDir, $Name, $Options, $StartDelay, [int]$WaitSeconds = 0) + + $Job = Invoke-Command -Session $Session -ScriptBlock { + param($RemoteDir, $Name, $Options, $StartDelay, $WaitSeconds) + + Set-Location (Join-Path $RemoteDir 'echo') + + $Tool = Join-Path $RemoteDir ("echo\$Name.exe") + Write-Host "[Remote] Running: $Tool" + if ($Options -is [System.Array]) { + Write-Host "[Remote] Arguments (array):" + foreach ($arg in $Options) { Write-Host " $arg" } + $argList = $Options + } + else { + Write-Host "[Remote] Arguments (string):" + Write-Host " $Options" + $argList = @() + if (-not [string]::IsNullOrEmpty($Options)) { $argList = @($Options) } + } + + # If StartDelay is set, wait 10 seconds before starting. + if ($StartDelay) { + Write-Host "[Remote] StartDelay is set; waiting 10 seconds before starting $Name.exe..." + Start-Sleep -Seconds 10 + } + + try { + $proc = Start-Process -FilePath $Tool -ArgumentList $argList -NoNewWindow -PassThru -ErrorAction Stop + Write-Host "[Remote] Started process Id=$($proc.Id)" + + if ($WaitSeconds -and $WaitSeconds -gt 0) { + # Wait for exit with timeout (milliseconds) + $ms = [int]($WaitSeconds * 1000) + Write-Host "[Remote] Waiting up to $WaitSeconds seconds for process to exit..." + $exited = $proc.WaitForExit($ms) + if (-not $exited) { + Write-Host "[Remote] Process did not exit within timeout; killing process Id=$($proc.Id)" + try { $proc | Stop-Process -Force -ErrorAction SilentlyContinue } catch { } + } + else { + Write-Host "[Remote] Process exited with code $($proc.ExitCode)" + if ($proc.ExitCode -ne 0) { throw "Remote $Tool.exe exited with code $($proc.ExitCode)" } + } + } + else { + Write-Host "[Remote] Not waiting for process to exit (no timeout configured)" + } + } + catch { + throw "Failed to launch or monitor process $Tool $($_.Exception.Message)" + } + } -ArgumentList $RemoteDir, $Name, $Options, $StartDelay, $WaitSeconds -AsJob -ErrorAction Stop + + return $Job +} + +function Receive-JobOrThrow { + param([Parameter(Mandatory)] $Job) + + Wait-Job $Job | Out-Null + + # Drain output (keep so we can inspect again if needed) + $null = Receive-Job $Job -Keep + + $errs = @() + foreach ($cj in $Job.ChildJobs) { + if ($cj.Error -and $cj.Error.Count -gt 0) { + $errs += $cj.Error + } + if ($cj.JobStateInfo.State -eq 'Failed' -and $cj.JobStateInfo.Reason) { + $errs += $cj.JobStateInfo.Reason + } + } + + if ($errs.Count -gt 0) { + foreach ($er in $errs) { + if ($er -is [System.Management.Automation.ErrorRecord]) { + $er | Write-DetailedError + } + else { + Write-Host $er + } + } + throw "One or more remote errors occurred (job id: $($Job.Id))." + } + + if ($Job.State -eq 'Failed') { + throw "Remote job failed (job id: $($Job.Id)): $($Job.JobStateInfo.Reason)" + } +} + +# ------------------------- +# Refactored workflow functions +# ------------------------- + +function Create-Session { + param( + [Parameter(Mandatory=$true)][string]$PeerName, + [string]$RemotePSConfiguration = 'PowerShell.7' + ) + + $script:RemotePSConfiguration = $RemotePSConfiguration + $script:RemoteDir = 'C:\_work' + + $Username = (Get-ItemProperty 'HKLM:\Software\Microsoft\Windows NT\CurrentVersion\Winlogon').DefaultUserName + $Password = (Get-ItemProperty 'HKLM:\Software\Microsoft\Windows NT\CurrentVersion\Winlogon').DefaultPassword | ConvertTo-SecureString -AsPlainText -Force + $Creds = New-Object System.Management.Automation.PSCredential ($Username, $Password) + + try { + Write-Host "Creating PSSession to $PeerName using configuration '$RemotePSConfiguration'..." + $s = New-PSSession -ComputerName $PeerName -Credential $Creds -ConfigurationName $RemotePSConfiguration -ErrorAction Stop + Write-Host "Session created using configuration '$RemotePSConfiguration'." + } + catch { + Write-Host "Failed to create session using configuration '$RemotePSConfiguration': $($_.Exception.Message)" + Write-Host "Attempting fallback: creating session without ConfigurationName..." + try { + $s = New-PSSession -ComputerName $PeerName -Credential $Creds -ErrorAction Stop + Write-Host "Session created using default configuration." + } + catch { + Write-Host "Fallback session creation failed: $($_.Exception.Message)" + throw "Failed to create remote session to $PeerName" + } + } + + $script:Session = $s + return $s +} + +function Save-And-Disable-Firewalls { + param([Parameter(Mandatory=$true)]$Session) + + # Coerce possible multi-output (array) from Create-Session into the actual PSSession object. + if ($Session -is [System.Array]) { + $found = $Session | Where-Object { $_ -is [System.Management.Automation.Runspaces.PSSession] } + if ($found -and $found.Count -gt 0) { $Session = $found[0] } + else { $Session = $Session[0] } + } + + if (-not ($Session -is [System.Management.Automation.Runspaces.PSSession])) { + throw "Save-And-Disable-Firewalls requires a PSSession object. Got: $($Session.GetType().FullName) - $Session" + } + + Write-Host "Saving and disabling local firewall profiles..." + $script:localFwState = Get-NetFirewallProfile -Profile Domain, Public, Private | Select-Object Name, Enabled + Set-NetFirewallProfile -Profile Domain, Public, Private -Enabled False + + Write-Host "Disabling firewall on remote machine..." + Invoke-Command -Session $Session -ScriptBlock { + param() + $fw = Get-NetFirewallProfile -Profile Domain, Public, Private | Select-Object Name, Enabled + Set-Variable -Name __SavedFirewallState -Value $fw -Scope Global -Force + Set-NetFirewallProfile -Profile Domain, Public, Private -Enabled False + } -ErrorAction Stop +} + +function Copy-EchoToRemote { + param([Parameter(Mandatory=$true)]$Session) + # Ensure the remote base directory and the 'echo' subdirectory both exist, + # then copy the *contents* of the local directory into the remote folder. + Invoke-Command -Session $Session -ScriptBlock { + param($base, $sub) + if (-not (Test-Path $base)) { New-Item -ItemType Directory -Path $base | Out-Null } + $full = Join-Path $base $sub + if (-not (Test-Path $full)) { New-Item -ItemType Directory -Path $full | Out-Null } + } -ArgumentList $script:RemoteDir, 'echo' -ErrorAction Stop + + $localPath = (Resolve-Path .).Path + Copy-Item -ToSession $Session -Path (Join-Path $localPath '*') -Destination "$script:RemoteDir\echo" -Recurse -Force +} + +# Robust remote file fetch: try Copy-Item -FromSession, fall back to Invoke-Command/Get-Content +function Fetch-RemoteFile { + param( + [Parameter(Mandatory=$true)]$Session, + [Parameter(Mandatory=$true)][string]$RemotePath, + [Parameter(Mandatory=$true)][string]$LocalDestination + ) + + Write-Host "Fetching remote file '$RemotePath' to local '$LocalDestination'..." + + try { + Copy-Item -FromSession $Session -Path $RemotePath -Destination $LocalDestination -ErrorAction Stop + Write-Host "Successfully fetched remote file '$RemotePath' to '$LocalDestination' via Copy-Item -FromSession." + return $true + } + catch { + Write-Host "Copy-Item -FromSession failed for '$RemotePath': $($_.Exception.Message). Attempting Invoke-Command fallback..." + try { + $content = Invoke-Command -Session $Session -ScriptBlock { param($p) Get-Content -Path $p -Raw -ErrorAction Stop } -ArgumentList $RemotePath -ErrorAction Stop + if ($null -ne $content) { + $content | Out-File -FilePath $LocalDestination -Encoding utf8 -Force + return $true + } + else { + Write-Host "Invoke-Command returned no content for '$RemotePath'" + return $false + } + } + catch { + Write-Host "Failed to fetch remote file '$RemotePath' via Invoke-Command: $($_.Exception.Message)" + return $false + } + } +} + +function Run-SendTest { + param( + [Parameter(Mandatory=$true)][string]$PeerName, + [Parameter(Mandatory=$true)]$Session, + [Parameter(Mandatory=$true)][string]$SenderOptions, + [Parameter(Mandatory=$true)][string]$ReceiverOptions + ) + + $serverArgs = Convert-ArgStringToArray $ReceiverOptions + # Normalize server args + $serverArgs = Normalize-Args -Tokens $serverArgs + Write-Host "[Local->Remote] Invoking remote job with arguments:" + if ($serverArgs -is [System.Array]) { foreach ($arg in $serverArgs) { Write-Host " $arg" } } else { Write-Host " $serverArgs" } + $Job = Invoke-EchoInSession -Session $Session -RemoteDir $script:RemoteDir -Name "echo_server" -Options $serverArgs -WaitSeconds 20 + + $clientArgs = Convert-ArgStringToArray $SenderOptions + $clientArgs = Set-TargetArg -ArgsArray $clientArgs -TargetName $PeerName + $clientArgs = Normalize-Args -Tokens $clientArgs + + # Delay 10 seconds to allow remote server to start before client connects + Write-Host "[Local] Waiting 10 seconds before starting send test to allow remote receiver to initialize..." + Start-Sleep -Seconds 10 + + Write-Host "[Local] Running: .\echo_client.exe" + Write-Host "[Local] Arguments:" + foreach ($a in $clientArgs) { Write-Host " $a" } + Start-WprCpuProfile -Which 'send' + & .\echo_client.exe @clientArgs + $script:localExit = $LASTEXITCODE + Stop-WprCpuProfile -Which 'send' + + Receive-JobOrThrow -Job $Job +} + +function Run-RecvTest { + param( + [Parameter(Mandatory=$true)][string]$PeerName, + [Parameter(Mandatory=$true)]$Session, + [Parameter(Mandatory=$true)][string]$SenderOptions, + [Parameter(Mandatory=$true)][string]$ReceiverOptions + ) + + $serverArgs = Convert-ArgStringToArray $SenderOptions + $serverArgs = Set-TargetArg -ArgsArray $serverArgs -TargetName $PeerName + $serverArgs = Normalize-Args -Tokens $serverArgs + Write-Host "[Local->Remote] Invoking remote job with arguments:" + if ($serverArgs -is [System.Array]) { foreach ($arg in $serverArgs) { Write-Host " $arg" } } else { Write-Host " $serverArgs" } + $Job = Invoke-EchoInSession -Session $Session -RemoteDir $script:RemoteDir -Name "echo_client" -Options $serverArgs -StartDelay $true -WaitSeconds 20 + + $clientArgs = Convert-ArgStringToArray $ReceiverOptions + $clientArgs = Normalize-Args -Tokens $clientArgs + + Write-Host "[Local] Running: .\echo_server.exe" + Write-Host "[Local] Arguments:" + foreach ($a in $clientArgs) { Write-Host " $a" } + Start-WprCpuProfile -Which 'recv' + & .\echo_server.exe @clientArgs + $script:localExit = $LASTEXITCODE + Stop-WprCpuProfile -Which 'recv' + + Receive-JobOrThrow -Job $Job +} + +function Restore-FirewallAndCleanup { + param([object]$Session) + + try { + if ($null -ne $Session) { + try { + Write-Host "Restoring firewall state on remote machine..." + Invoke-Command -Session $Session -ScriptBlock { + if (Get-Variable -Name __SavedFirewallState -Scope Global -ErrorAction SilentlyContinue) { + $saved = Get-Variable -Name __SavedFirewallState -Scope Global -ValueOnly + foreach ($p in $saved) { + Set-NetFirewallProfile -Profile $p.Name -Enabled $p.Enabled + } + Remove-Variable -Name __SavedFirewallState -Scope Global -ErrorAction SilentlyContinue + } + else { + Set-NetFirewallProfile -Profile Domain, Public, Private -Enabled True + } + } -ErrorAction SilentlyContinue + } + catch { + $_ | Write-DetailedError + } + + try { + Remove-PSSession $Session -ErrorAction SilentlyContinue + } + catch { + $_ | Write-DetailedError + } + } + + Write-Host "Restoring local firewall state..." + if ($localFwState) { + foreach ($p in $localFwState) { + Set-NetFirewallProfile -Profile $p.Name -Enabled $p.Enabled + } + } + else { + Set-NetFirewallProfile -Profile Domain, Public, Private -Enabled True + } + } + catch { + $_ | Write-DetailedError + } +} + +# ========================= +# Main workflow +# ========================= +$Workspace = $env:GITHUB_WORKSPACE +Write-Host "Workspace: $Workspace" + +try { + if (-not $Workspace) { throw 'GITHUB_WORKSPACE is not set' } + Set-Location (Join-Path $Workspace 'echo') + + # Create remote session + $Session = Create-Session -PeerName $PeerName -RemotePSConfiguration 'PowerShell.7' + + # Save and disable firewalls + Save-And-Disable-Firewalls -Session $Session + + # Copy tool to remote + Copy-EchoToRemote -Session $Session + + # Run tests + Run-SendTest -PeerName $PeerName -Session $Session -SenderOptions $SenderOptions -ReceiverOptions $ReceiverOptions + Run-RecvTest -PeerName $PeerName -Session $Session -SenderOptions $SenderOptions -ReceiverOptions $ReceiverOptions + + Write-Host "echo tests completed successfully." +} +catch { + # $_ is an ErrorRecord; print everything useful + Write-Host "echo tests failed." + Write-Host $_ + $exitCode = 2 +} +finally { + # Use refactored cleanup function + Restore-FirewallAndCleanup -Session $Session + Write-Host "Exiting with code $exitCode" + exit $exitCode +} + +<# + Troubleshooting notes for PowerShell Remoting errors: + + - Add remote host to TrustedHosts (run as Administrator on the client): + ```powershell + Set-Item WSMan:\localhost\Client\TrustedHosts -Value "alanjo-test-2" -Force + # or allow all (less secure): + Set-Item WSMan:\localhost\Client\TrustedHosts -Value "*" -Force + ``` + + - Enable WinRM on the remote machine (run as Administrator on the remote): + ```powershell + Enable-PSRemoting -Force + # Ensure the WinRM service is running: + Set-Service WinRM -StartupType Automatic + Start-Service WinRM + ``` + + - If using PowerShell 7 session configuration, register it on the remote (run on remote machine in PowerShell 7): + ```powershell + # Register a PowerShell 7 endpoint named 'PowerShell.7' + Register-PSSessionConfiguration -Name PowerShell.7 -RunAsCredential (Get-Credential) -Force + # Or use pwsh's implicit registration helper if available: + # pwsh -NoProfile -Command "Register-PSSessionConfiguration -Name PowerShell.7 -Force" + ``` + + - For HTTPS transport, create an HTTPS listener and configure an SSL cert for WinRM on the remote. See `about_Remote_Troubleshooting`. + + Note: Adding hosts to TrustedHosts weakens authentication; prefer HTTPS or domain-joined Kerberos where possible. + #> diff --git a/.github/workflows/network-traffic-performance.yml b/.github/workflows/network-traffic-performance.yml index d2eba9d1..ce0229a5 100644 --- a/.github/workflows/network-traffic-performance.yml +++ b/.github/workflows/network-traffic-performance.yml @@ -32,6 +32,14 @@ on: required: false default: 'master' type: string + test_tool: + description: 'The network test tool to use' + required: false + default: 'ctsTraffic' + type: choice + options: + - ctsTraffic + - echo concurrency: group: >- @@ -40,6 +48,7 @@ concurrency: jobs: build_cts_traffic: + if: ${{ inputs.test_tool == 'ctsTraffic' }} name: Build cts-traffic uses: microsoft/ctsTraffic/.github/workflows/reusable-build.yml@master with: @@ -48,7 +57,8 @@ jobs: configurations: '["Release"]' ref: ${{ inputs.cts_traffic_ref }} - test: + test_cts_traffic: + if: ${{ inputs.test_tool == 'ctsTraffic' }} name: Network Traffic Test needs: [build_cts_traffic] strategy: @@ -154,4 +164,121 @@ jobs: uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 with: name: cpu_profile_${{ matrix.vec.env }}_${{ matrix.vec.os }}_${{ matrix.vec.arch }} - path: ${{ github.workspace }}\ETL\cpu_profile-*.etl \ No newline at end of file + path: ${{ github.workspace }}\ETL\cpu_profile-*.etl + + build_echo_test: + if: ${{ inputs.test_tool == 'echo' }} + name: Build echo server + uses: alan-jowett/scalable_echo_server_demo/.github/workflows/reusable-build.yml@main + with: + artifact_name: echo_test + repository: 'alan-jowett/scalable_echo_server_demo' + + test_echo_server: + if: ${{ inputs.test_tool == 'echo' }} + name: Network Traffic Test (echo) + needs: [build_echo_test] + strategy: + fail-fast: false + matrix: + vec: + - env: lab + os: "2022" + arch: x64 + runs-on: + - self-hosted + - ${{ matrix.vec.env }} + - os-windows-${{ matrix.vec.os }} + - ${{ matrix.vec.arch }} + + steps: + - name: Checkout repository + uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 + + - name: Setup workspace + run: |- + Get-ChildItem | % { Remove-Item -Recurse $_ -Force -ErrorAction SilentlyContinue } + if (Test-Path ${{ github.workspace }}\echo) { Remove-Item -Recurse -Force ${{ github.workspace }}\echo } + if (Test-Path ${{ github.workspace }}\ETL) { Remove-Item -Recurse -Force ${{ github.workspace }}\ETL } + New-Item -ItemType Directory -Path ${{ github.workspace }}\echo + New-Item -ItemType Directory -Path ${{ github.workspace }}\ETL + + - name: Download echo test tool + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + with: + name: "echo_test" + path: ${{ github.workspace }}\echo + + - name: Optional - Start TCPIP tracing + if: ${{ github.event.inputs.tcp_ip_tracing == 'true' }} + run: |- + if (Test-Path "tcpip.wprp") { Remove-Item -Force "tcpip.wprp" } + Invoke-WebRequest -uri "https://raw.githubusercontent.com/microsoft/netperf/${{inputs.ref}}/.github/workflows/tcpip.wprp" -OutFile "tcpip.wprp" + wpr -start tcpip.wprp -filemode + + - name: Download run-echo-test script + working-directory: ${{ github.workspace }}\echo + run: | + Invoke-WebRequest -uri "https://raw.githubusercontent.com/microsoft/netperf/${{inputs.ref}}/.github/scripts/run-echo-test.ps1" -OutFile ".\run-echo-test.ps1" + + - name: Log all input parameters for this workflow + run: | + Write-Output "Input parameters:" + Write-Output " Profile: ${{ inputs.profile }}" + Write-Output " SenderOptions: ${{ inputs.sender_options }}" + Write-Output " ReceiverOptions: ${{ inputs.receiver_options }}" + Write-Output " TCP IP Tracing: ${{ inputs.tcp_ip_tracing }}" + Write-Output " ref: ${{ inputs.ref }}" + + - name: Run traffic tests + working-directory: ${{ github.workspace }}\echo + env: + PROFILE: ${{ inputs.profile }} + SENDER_OPTIONS: ${{ inputs.sender_options }} + RECEIVER_OPTIONS: ${{ inputs.receiver_options }} + shell: pwsh + run: | + # Build the command and only include -CpuProfile when PROFILE is truthy ('true' or true) + $cpuSwitch = '' + if ($env:PROFILE -eq 'true' -or $env:PROFILE -eq 'True' -or $env:PROFILE -eq 'TRUE' -or $env:PROFILE -eq $true) { + $cpuSwitch = '-CpuProfile' + } + + $cmd = @( + '.\\run-echo-test.ps1', + $cpuSwitch, + '-PeerName', '"netperf-peer"', + '-SenderOptions', '"' + $env:SENDER_OPTIONS + '"', + '-ReceiverOptions', '"' + $env:RECEIVER_OPTIONS + '"' + ) -join ' ' + + Write-Output "Running: $cmd" + Invoke-Expression $cmd + + - name: Optional - Stop TCPIP tracing + if: ${{ github.event.inputs.tcp_ip_tracing == 'true' }} + run: | + wpr -stop ${{ github.workspace }}\ETL\tcpip_trace.etl + + - name: Upload echo results + if: always() + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 + with: + name: echo_test_${{ matrix.vec.env }}_${{ matrix.vec.os }}_${{ matrix.vec.arch }} + path: | + ${{ github.workspace }}\echo\*.csv + ${{ github.workspace }}\echo\*.log + + - name: Upload TCPIP ETL + if: always() + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 + with: + name: tcpip_trace_${{ matrix.vec.env }}_${{ matrix.vec.os }}_${{ matrix.vec.arch }} + path: ${{ github.workspace }}\ETL\tcpip_trace.etl + + - name: Upload CPU Profile + if: always() + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 + with: + name: cpu_profile_${{ matrix.vec.env }}_${{ matrix.vec.os }}_${{ matrix.vec.arch }} + path: ${{ github.workspace }}\ETL\cpu_profile-*.etl From 5a92d6d5f2ea7036cd4e4a74f6bcd412459ca12a Mon Sep 17 00:00:00 2001 From: Alan Jowett Date: Mon, 1 Dec 2025 18:18:08 -0800 Subject: [PATCH 03/70] Fix path Signed-off-by: Alan Jowett --- .github/workflows/network-traffic-performance.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/network-traffic-performance.yml b/.github/workflows/network-traffic-performance.yml index ce0229a5..b715f2a2 100644 --- a/.github/workflows/network-traffic-performance.yml +++ b/.github/workflows/network-traffic-performance.yml @@ -231,7 +231,7 @@ jobs: Write-Output " ref: ${{ inputs.ref }}" - name: Run traffic tests - working-directory: ${{ github.workspace }}\echo + working-directory: ${{ github.workspace }}\echo\release env: PROFILE: ${{ inputs.profile }} SENDER_OPTIONS: ${{ inputs.sender_options }} From 94ebde9bf167042a2b7c3d6b3c4a5db5c87563c9 Mon Sep 17 00:00:00 2001 From: Alan Jowett Date: Mon, 1 Dec 2025 18:25:24 -0800 Subject: [PATCH 04/70] Fix path Signed-off-by: Alan Jowett --- .github/workflows/network-traffic-performance.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/network-traffic-performance.yml b/.github/workflows/network-traffic-performance.yml index b715f2a2..fcb45e33 100644 --- a/.github/workflows/network-traffic-performance.yml +++ b/.github/workflows/network-traffic-performance.yml @@ -200,7 +200,7 @@ jobs: Get-ChildItem | % { Remove-Item -Recurse $_ -Force -ErrorAction SilentlyContinue } if (Test-Path ${{ github.workspace }}\echo) { Remove-Item -Recurse -Force ${{ github.workspace }}\echo } if (Test-Path ${{ github.workspace }}\ETL) { Remove-Item -Recurse -Force ${{ github.workspace }}\ETL } - New-Item -ItemType Directory -Path ${{ github.workspace }}\echo + New-Item -ItemType Directory -Path ${{ github.workspace }}\echo\release New-Item -ItemType Directory -Path ${{ github.workspace }}\ETL - name: Download echo test tool @@ -217,7 +217,7 @@ jobs: wpr -start tcpip.wprp -filemode - name: Download run-echo-test script - working-directory: ${{ github.workspace }}\echo + working-directory: ${{ github.workspace }}\echo\release run: | Invoke-WebRequest -uri "https://raw.githubusercontent.com/microsoft/netperf/${{inputs.ref}}/.github/scripts/run-echo-test.ps1" -OutFile ".\run-echo-test.ps1" From d568bb78deec93b452c2fecc1ffe9e6c8d4477c2 Mon Sep 17 00:00:00 2001 From: Alan Jowett Date: Mon, 1 Dec 2025 18:29:30 -0800 Subject: [PATCH 05/70] Fix path Signed-off-by: Alan Jowett --- .github/workflows/network-traffic-performance.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/network-traffic-performance.yml b/.github/workflows/network-traffic-performance.yml index fcb45e33..c360e47d 100644 --- a/.github/workflows/network-traffic-performance.yml +++ b/.github/workflows/network-traffic-performance.yml @@ -230,6 +230,11 @@ jobs: Write-Output " TCP IP Tracing: ${{ inputs.tcp_ip_tracing }}" Write-Output " ref: ${{ inputs.ref }}" + - name: Diagonstic - List files in echo release directory + run: | + Write-Output "Files in echo release directory:" + Get-ChildItem -Path ${{ github.workspace }}\echo\release -Recurse + - name: Run traffic tests working-directory: ${{ github.workspace }}\echo\release env: From adc620fb771223b1ae0095395b577030b3dba424 Mon Sep 17 00:00:00 2001 From: Alan Jowett Date: Mon, 1 Dec 2025 18:33:27 -0800 Subject: [PATCH 06/70] Fix path Signed-off-by: Alan Jowett --- .github/scripts/run-echo-test.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/run-echo-test.ps1 b/.github/scripts/run-echo-test.ps1 index 746a5833..a634bf81 100644 --- a/.github/scripts/run-echo-test.ps1 +++ b/.github/scripts/run-echo-test.ps1 @@ -555,7 +555,7 @@ Write-Host "Workspace: $Workspace" try { if (-not $Workspace) { throw 'GITHUB_WORKSPACE is not set' } - Set-Location (Join-Path $Workspace 'echo') + Set-Location ($Workspace) # Create remote session $Session = Create-Session -PeerName $PeerName -RemotePSConfiguration 'PowerShell.7' From ddc61cbdb319d2fe4da257f69aed5f291b50581e Mon Sep 17 00:00:00 2001 From: Alan Jowett Date: Mon, 1 Dec 2025 18:36:49 -0800 Subject: [PATCH 07/70] Fix path Signed-off-by: Alan Jowett --- .github/scripts/run-echo-test.ps1 | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/scripts/run-echo-test.ps1 b/.github/scripts/run-echo-test.ps1 index a634bf81..de69284e 100644 --- a/.github/scripts/run-echo-test.ps1 +++ b/.github/scripts/run-echo-test.ps1 @@ -550,12 +550,10 @@ function Restore-FirewallAndCleanup { # ========================= # Main workflow # ========================= -$Workspace = $env:GITHUB_WORKSPACE -Write-Host "Workspace: $Workspace" - try { - if (-not $Workspace) { throw 'GITHUB_WORKSPACE is not set' } - Set-Location ($Workspace) + # Print the current working directory + $cwd = (Get-Location).Path + Write-Host "Current working directory: $cwd" # Create remote session $Session = Create-Session -PeerName $PeerName -RemotePSConfiguration 'PowerShell.7' From bfd365922ad19dbd4431aaa624e4e8c429d3e6e4 Mon Sep 17 00:00:00 2001 From: Alan Jowett Date: Mon, 1 Dec 2025 18:54:52 -0800 Subject: [PATCH 08/70] Fix path Signed-off-by: Alan Jowett --- .github/scripts/run-echo-test.ps1 | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/.github/scripts/run-echo-test.ps1 b/.github/scripts/run-echo-test.ps1 index de69284e..750a09b1 100644 --- a/.github/scripts/run-echo-test.ps1 +++ b/.github/scripts/run-echo-test.ps1 @@ -229,10 +229,10 @@ function Stop-WprCpuProfile { # Remote job helpers # ========================= function Invoke-EchoInSession { - param($Session, $RemoteDir, $Name, $Options, $StartDelay, [int]$WaitSeconds = 0) + param($Session, $RemoteDir, $Name, $Options, [int]$WaitSeconds = 0) $Job = Invoke-Command -Session $Session -ScriptBlock { - param($RemoteDir, $Name, $Options, $StartDelay, $WaitSeconds) + param($RemoteDir, $Name, $Options, $WaitSeconds) Set-Location (Join-Path $RemoteDir 'echo') @@ -250,12 +250,6 @@ function Invoke-EchoInSession { if (-not [string]::IsNullOrEmpty($Options)) { $argList = @($Options) } } - # If StartDelay is set, wait 10 seconds before starting. - if ($StartDelay) { - Write-Host "[Remote] StartDelay is set; waiting 10 seconds before starting $Name.exe..." - Start-Sleep -Seconds 10 - } - try { $proc = Start-Process -FilePath $Tool -ArgumentList $argList -NoNewWindow -PassThru -ErrorAction Stop Write-Host "[Remote] Started process Id=$($proc.Id)" @@ -281,7 +275,7 @@ function Invoke-EchoInSession { catch { throw "Failed to launch or monitor process $Tool $($_.Exception.Message)" } - } -ArgumentList $RemoteDir, $Name, $Options, $StartDelay, $WaitSeconds -AsJob -ErrorAction Stop + } -ArgumentList $RemoteDir, $Name, $Options, $WaitSeconds -AsJob -ErrorAction Stop return $Job } @@ -456,10 +450,6 @@ function Run-SendTest { $clientArgs = Set-TargetArg -ArgsArray $clientArgs -TargetName $PeerName $clientArgs = Normalize-Args -Tokens $clientArgs - # Delay 10 seconds to allow remote server to start before client connects - Write-Host "[Local] Waiting 10 seconds before starting send test to allow remote receiver to initialize..." - Start-Sleep -Seconds 10 - Write-Host "[Local] Running: .\echo_client.exe" Write-Host "[Local] Arguments:" foreach ($a in $clientArgs) { Write-Host " $a" } @@ -484,7 +474,7 @@ function Run-RecvTest { $serverArgs = Normalize-Args -Tokens $serverArgs Write-Host "[Local->Remote] Invoking remote job with arguments:" if ($serverArgs -is [System.Array]) { foreach ($arg in $serverArgs) { Write-Host " $arg" } } else { Write-Host " $serverArgs" } - $Job = Invoke-EchoInSession -Session $Session -RemoteDir $script:RemoteDir -Name "echo_client" -Options $serverArgs -StartDelay $true -WaitSeconds 20 + $Job = Invoke-EchoInSession -Session $Session -RemoteDir $script:RemoteDir -Name "echo_client" -Options $serverArgs -WaitSeconds 20 $clientArgs = Convert-ArgStringToArray $ReceiverOptions $clientArgs = Normalize-Args -Tokens $clientArgs From 64574c879919be2dfa5fdc88985a04d9591679df Mon Sep 17 00:00:00 2001 From: Alan Jowett Date: Mon, 1 Dec 2025 18:58:31 -0800 Subject: [PATCH 09/70] Fix path Signed-off-by: Alan Jowett --- .github/workflows/network-traffic-performance.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/network-traffic-performance.yml b/.github/workflows/network-traffic-performance.yml index c360e47d..ee7c6046 100644 --- a/.github/workflows/network-traffic-performance.yml +++ b/.github/workflows/network-traffic-performance.yml @@ -169,10 +169,10 @@ jobs: build_echo_test: if: ${{ inputs.test_tool == 'echo' }} name: Build echo server - uses: alan-jowett/scalable_echo_server_demo/.github/workflows/reusable-build.yml@main + uses: alan-jowett/WinUDPShardedEcho/.github/workflows/reusable-build.yml@main with: artifact_name: echo_test - repository: 'alan-jowett/scalable_echo_server_demo' + repository: 'alan-jowett/WinUDPShardedEcho' test_echo_server: if: ${{ inputs.test_tool == 'echo' }} From ca771f1848404d56d9c56816a21249faef6b8792 Mon Sep 17 00:00:00 2001 From: Alan Jowett Date: Mon, 1 Dec 2025 19:13:14 -0800 Subject: [PATCH 10/70] Fix path Signed-off-by: Alan Jowett --- .github/scripts/run-echo-test.ps1 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/scripts/run-echo-test.ps1 b/.github/scripts/run-echo-test.ps1 index 750a09b1..6b94b806 100644 --- a/.github/scripts/run-echo-test.ps1 +++ b/.github/scripts/run-echo-test.ps1 @@ -541,8 +541,10 @@ function Restore-FirewallAndCleanup { # Main workflow # ========================= try { + # Print the current working directory $cwd = (Get-Location).Path + $Workspace = $env:GITHUB_WORKSPACE Write-Host "Current working directory: $cwd" # Create remote session From 045359f627cfa8236654ec2bb9a741f3df40dba2 Mon Sep 17 00:00:00 2001 From: Alan Jowett Date: Mon, 1 Dec 2025 19:17:15 -0800 Subject: [PATCH 11/70] Fix path Signed-off-by: Alan Jowett --- .github/scripts/run-echo-test.ps1 | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/.github/scripts/run-echo-test.ps1 b/.github/scripts/run-echo-test.ps1 index 6b94b806..39245b77 100644 --- a/.github/scripts/run-echo-test.ps1 +++ b/.github/scripts/run-echo-test.ps1 @@ -151,7 +151,7 @@ function Start-WprCpuProfile { if (-not $CpuProfile) { return } - if (-not $Workspace) { $Workspace = $env:GITHUB_WORKSPACE } + $Workspace = $env:GITHUB_WORKSPACE $etlDir = Join-Path $Workspace 'ETL' if (-not (Test-Path $etlDir)) { New-Item -ItemType Directory -Path $etlDir | Out-Null } @@ -544,7 +544,6 @@ try { # Print the current working directory $cwd = (Get-Location).Path - $Workspace = $env:GITHUB_WORKSPACE Write-Host "Current working directory: $cwd" # Create remote session From f518678a4f32a1f35b55aff1f94acdbbc9ac4ee0 Mon Sep 17 00:00:00 2001 From: Alan Jowett Date: Mon, 1 Dec 2025 19:45:57 -0800 Subject: [PATCH 12/70] Increase buffer size Signed-off-by: Alan Jowett --- .github/scripts/cpu.wprp | 132 ++++++++++++++++++ .github/scripts/run-echo-test.ps1 | 2 +- .../workflows/network-traffic-performance.yml | 5 + 3 files changed, 138 insertions(+), 1 deletion(-) create mode 100644 .github/scripts/cpu.wprp diff --git a/.github/scripts/cpu.wprp b/.github/scripts/cpu.wprp new file mode 100644 index 00000000..84486c1d --- /dev/null +++ b/.github/scripts/cpu.wprp @@ -0,0 +1,132 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/.github/scripts/run-echo-test.ps1 b/.github/scripts/run-echo-test.ps1 index 39245b77..a6387e9c 100644 --- a/.github/scripts/run-echo-test.ps1 +++ b/.github/scripts/run-echo-test.ps1 @@ -180,7 +180,7 @@ function Start-WprCpuProfile { } try { - & wpr -start CPU -filemode | Out-Null + & wpr -start CPU.wprp -filemode | Out-Null } catch { Write-Host "wpr -start with custom profile failed: $($_.Exception.Message). Falling back to built-in CPU profile." diff --git a/.github/workflows/network-traffic-performance.yml b/.github/workflows/network-traffic-performance.yml index ee7c6046..4343de7e 100644 --- a/.github/workflows/network-traffic-performance.yml +++ b/.github/workflows/network-traffic-performance.yml @@ -216,6 +216,11 @@ jobs: Invoke-WebRequest -uri "https://raw.githubusercontent.com/microsoft/netperf/${{inputs.ref}}/.github/workflows/tcpip.wprp" -OutFile "tcpip.wprp" wpr -start tcpip.wprp -filemode + - name: Download modified CPU profile script + working-directory: ${{ github.workspace }}\ETL + run: | + Invoke-WebRequest -uri "https://raw.githubusercontent.com/microsoft/netperf/${{inputs.ref}}/.github/scripts/CPU.wprp" -OutFile ".\cpu.wprp" + - name: Download run-echo-test script working-directory: ${{ github.workspace }}\echo\release run: | From 3eb5e7cdc79659cfb2532a53ec69e9014d8fd5a4 Mon Sep 17 00:00:00 2001 From: Alan Jowett Date: Mon, 1 Dec 2025 19:51:26 -0800 Subject: [PATCH 13/70] Increase buffer size Signed-off-by: Alan Jowett --- .github/workflows/network-traffic-performance.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/network-traffic-performance.yml b/.github/workflows/network-traffic-performance.yml index 4343de7e..db248382 100644 --- a/.github/workflows/network-traffic-performance.yml +++ b/.github/workflows/network-traffic-performance.yml @@ -219,7 +219,7 @@ jobs: - name: Download modified CPU profile script working-directory: ${{ github.workspace }}\ETL run: | - Invoke-WebRequest -uri "https://raw.githubusercontent.com/microsoft/netperf/${{inputs.ref}}/.github/scripts/CPU.wprp" -OutFile ".\cpu.wprp" + Invoke-WebRequest -uri "https://raw.githubusercontent.com/microsoft/netperf/${{inputs.ref}}/.github/scripts/cpu.wprp" -OutFile ".\cpu.wprp" - name: Download run-echo-test script working-directory: ${{ github.workspace }}\echo\release From c7952778e7bf80b8f9095aeec6012bbb515b8a55 Mon Sep 17 00:00:00 2001 From: Alan Jowett Date: Mon, 1 Dec 2025 20:01:38 -0800 Subject: [PATCH 14/70] Increase buffer size Signed-off-by: Alan Jowett --- .github/scripts/run-echo-test.ps1 | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/scripts/run-echo-test.ps1 b/.github/scripts/run-echo-test.ps1 index a6387e9c..63fe4af5 100644 --- a/.github/scripts/run-echo-test.ps1 +++ b/.github/scripts/run-echo-test.ps1 @@ -155,6 +155,8 @@ function Start-WprCpuProfile { $etlDir = Join-Path $Workspace 'ETL' if (-not (Test-Path $etlDir)) { New-Item -ItemType Directory -Path $etlDir | Out-Null } + $WprProfile = Join-Path $etlDir 'cpu.wprp' + $outFile = Join-Path $etlDir ("cpu_profile-$Which.etl") if (Test-Path $outFile) { Remove-Item $outFile -Force -ErrorAction SilentlyContinue } @@ -180,7 +182,7 @@ function Start-WprCpuProfile { } try { - & wpr -start CPU.wprp -filemode | Out-Null + & wpr -start $WprProfile -filemode | Out-Null } catch { Write-Host "wpr -start with custom profile failed: $($_.Exception.Message). Falling back to built-in CPU profile." From 975f2b1185f42298c91bef77088db31e3cef277d Mon Sep 17 00:00:00 2001 From: Alan Jowett Date: Mon, 1 Dec 2025 20:05:26 -0800 Subject: [PATCH 15/70] Increase buffer size Signed-off-by: Alan Jowett --- .github/workflows/network-traffic-performance.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/network-traffic-performance.yml b/.github/workflows/network-traffic-performance.yml index db248382..8e82a4c5 100644 --- a/.github/workflows/network-traffic-performance.yml +++ b/.github/workflows/network-traffic-performance.yml @@ -219,7 +219,7 @@ jobs: - name: Download modified CPU profile script working-directory: ${{ github.workspace }}\ETL run: | - Invoke-WebRequest -uri "https://raw.githubusercontent.com/microsoft/netperf/${{inputs.ref}}/.github/scripts/cpu.wprp" -OutFile ".\cpu.wprp" + Invoke-WebRequest -uri "https://raw.githubusercontent.com/microsoft/netperf/${{inputs.ref}}/.github/scripts/cpu.wprp" -OutFile "${{ github.workspace }}\ETL\cpu.wprp" - name: Download run-echo-test script working-directory: ${{ github.workspace }}\echo\release From 36e3fe722dea1df59fa9e5d1b3da756145c6f52b Mon Sep 17 00:00:00 2001 From: Alan Jowett Date: Tue, 2 Dec 2025 07:53:30 -0800 Subject: [PATCH 16/70] Cancel any tracing Signed-off-by: Alan Jowett --- .github/workflows/ebpf.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/ebpf.yml b/.github/workflows/ebpf.yml index 285bc3a9..c48d6012 100644 --- a/.github/workflows/ebpf.yml +++ b/.github/workflows/ebpf.yml @@ -227,6 +227,10 @@ jobs: name: "cts-traffic Release" path: ${{ github.workspace }}\cts-traffic + - name: Cancel any existing WPR traces + run: | + wpr -cancel 2>$null; $global:LASTEXITCODE = 0 + - name: Start TCPIP tracing - Baseline if: ${{ github.event.inputs.tcp_ip_tracing }} run: | @@ -235,6 +239,7 @@ jobs: Invoke-WebRequest -uri "https://raw.githubusercontent.com/microsoft/netperf/main/.github/workflows/tcpip.wprp" -OutFile "tcpip.wprp" wpr -start tcpip.wprp -filemode + # Run CTS traffic without XDP installed to establish a baseline. - name: Run CTS cts-traffic baseline working-directory: ${{ github.workspace }}\cts-traffic From 7d65a9ce85ba9d406ba25aa47e5f07a64890fb4e Mon Sep 17 00:00:00 2001 From: Alan Jowett Date: Tue, 2 Dec 2025 08:05:24 -0800 Subject: [PATCH 17/70] Diagnositcs Signed-off-by: Alan Jowett --- .github/workflows/ebpf.yml | 6 +----- .../workflows/network-traffic-performance.yml | 19 +++++++++++++++++++ 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/.github/workflows/ebpf.yml b/.github/workflows/ebpf.yml index c48d6012..df4222aa 100644 --- a/.github/workflows/ebpf.yml +++ b/.github/workflows/ebpf.yml @@ -226,11 +226,7 @@ jobs: with: name: "cts-traffic Release" path: ${{ github.workspace }}\cts-traffic - - - name: Cancel any existing WPR traces - run: | - wpr -cancel 2>$null; $global:LASTEXITCODE = 0 - + - name: Start TCPIP tracing - Baseline if: ${{ github.event.inputs.tcp_ip_tracing }} run: | diff --git a/.github/workflows/network-traffic-performance.yml b/.github/workflows/network-traffic-performance.yml index 8e82a4c5..28e0deea 100644 --- a/.github/workflows/network-traffic-performance.yml +++ b/.github/workflows/network-traffic-performance.yml @@ -209,6 +209,25 @@ jobs: name: "echo_test" path: ${{ github.workspace }}\echo + - name: List ETW sessions with providers + run: | + # Enumerate all ETW sessions and their providers + $Sessions = Get-EtwTraceSession -Name * + foreach ($session in $Sessions) { + Write-Host "Session: $($session.Name)" + Write-Host "-----------------------------------" + # Query providers for this session using logman + $providers = logman query $session.Name -ets | Select-String "GUID" -Context 0,1 + if ($providers) { + $providers | ForEach-Object { + Write-Host $_.Line + } + } else { + Write-Host "No providers found." + } + Write-Host "`n" + } + - name: Optional - Start TCPIP tracing if: ${{ github.event.inputs.tcp_ip_tracing == 'true' }} run: |- From 9426505c4fbe557e077a50d93cf50c1ad8b4a3ae Mon Sep 17 00:00:00 2001 From: Alan Jowett Date: Tue, 2 Dec 2025 08:35:04 -0800 Subject: [PATCH 18/70] Log provider names Signed-off-by: Alan Jowett --- .github/workflows/network-traffic-performance.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/network-traffic-performance.yml b/.github/workflows/network-traffic-performance.yml index 28e0deea..cad5b3f7 100644 --- a/.github/workflows/network-traffic-performance.yml +++ b/.github/workflows/network-traffic-performance.yml @@ -217,7 +217,7 @@ jobs: Write-Host "Session: $($session.Name)" Write-Host "-----------------------------------" # Query providers for this session using logman - $providers = logman query $session.Name -ets | Select-String "GUID" -Context 0,1 + $providers = logman query $session.Name -ets if ($providers) { $providers | ForEach-Object { Write-Host $_.Line From 3de39ae3e450090be071f57bb65677443eae4203 Mon Sep 17 00:00:00 2001 From: Alan Jowett Date: Tue, 2 Dec 2025 08:43:20 -0800 Subject: [PATCH 19/70] Rever Signed-off-by: Alan Jowett --- .github/workflows/network-traffic-performance.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/network-traffic-performance.yml b/.github/workflows/network-traffic-performance.yml index cad5b3f7..28e0deea 100644 --- a/.github/workflows/network-traffic-performance.yml +++ b/.github/workflows/network-traffic-performance.yml @@ -217,7 +217,7 @@ jobs: Write-Host "Session: $($session.Name)" Write-Host "-----------------------------------" # Query providers for this session using logman - $providers = logman query $session.Name -ets + $providers = logman query $session.Name -ets | Select-String "GUID" -Context 0,1 if ($providers) { $providers | ForEach-Object { Write-Host $_.Line From ac1a21af18b8096d85ece15524de0fe031107f72 Mon Sep 17 00:00:00 2001 From: Alan Jowett Date: Tue, 2 Dec 2025 09:27:07 -0800 Subject: [PATCH 20/70] Remove Microsoft-Windows-Networking-Correlation Signed-off-by: Alan Jowett --- .github/scripts/cpu.wprp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/scripts/cpu.wprp b/.github/scripts/cpu.wprp index 84486c1d..9a0feec8 100644 --- a/.github/scripts/cpu.wprp +++ b/.github/scripts/cpu.wprp @@ -64,7 +64,9 @@ + @@ -100,7 +102,7 @@ - + From f8309c40083258d2db308b6b058dd44921537176 Mon Sep 17 00:00:00 2001 From: Alan Jowett Date: Tue, 2 Dec 2025 09:33:41 -0800 Subject: [PATCH 21/70] Cleanup Signed-off-by: Alan Jowett --- .../workflows/network-traffic-performance.yml | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/.github/workflows/network-traffic-performance.yml b/.github/workflows/network-traffic-performance.yml index 28e0deea..8e82a4c5 100644 --- a/.github/workflows/network-traffic-performance.yml +++ b/.github/workflows/network-traffic-performance.yml @@ -209,25 +209,6 @@ jobs: name: "echo_test" path: ${{ github.workspace }}\echo - - name: List ETW sessions with providers - run: | - # Enumerate all ETW sessions and their providers - $Sessions = Get-EtwTraceSession -Name * - foreach ($session in $Sessions) { - Write-Host "Session: $($session.Name)" - Write-Host "-----------------------------------" - # Query providers for this session using logman - $providers = logman query $session.Name -ets | Select-String "GUID" -Context 0,1 - if ($providers) { - $providers | ForEach-Object { - Write-Host $_.Line - } - } else { - Write-Host "No providers found." - } - Write-Host "`n" - } - - name: Optional - Start TCPIP tracing if: ${{ github.event.inputs.tcp_ip_tracing == 'true' }} run: |- From b600f21c33df502b479f217b991f4913662be7da Mon Sep 17 00:00:00 2001 From: Alan Jowett Date: Tue, 2 Dec 2025 12:30:13 -0800 Subject: [PATCH 22/70] Don't timeout server Signed-off-by: Alan Jowett --- .github/scripts/run-echo-test.ps1 | 62 ++++++++++++++++++++++++------- 1 file changed, 49 insertions(+), 13 deletions(-) diff --git a/.github/scripts/run-echo-test.ps1 b/.github/scripts/run-echo-test.ps1 index 63fe4af5..7661b0db 100644 --- a/.github/scripts/run-echo-test.ps1 +++ b/.github/scripts/run-echo-test.ps1 @@ -253,25 +253,61 @@ function Invoke-EchoInSession { } try { - $proc = Start-Process -FilePath $Tool -ArgumentList $argList -NoNewWindow -PassThru -ErrorAction Stop - Write-Host "[Remote] Started process Id=$($proc.Id)" - + # Invoke the tool directly. When a timeout is requested, run the invocation + # inside a PowerShell background job so we can enforce a timeout and cancel + # the job (and any matching process) if it doesn't finish in time. if ($WaitSeconds -and $WaitSeconds -gt 0) { - # Wait for exit with timeout (milliseconds) - $ms = [int]($WaitSeconds * 1000) - Write-Host "[Remote] Waiting up to $WaitSeconds seconds for process to exit..." - $exited = $proc.WaitForExit($ms) - if (-not $exited) { - Write-Host "[Remote] Process did not exit within timeout; killing process Id=$($proc.Id)" - try { $proc | Stop-Process -Force -ErrorAction SilentlyContinue } catch { } + Write-Host "[Remote] Starting tool as background job for timeout control..." + $jobScript = { + param($ToolPath, $ArgList) + if ($ArgList -is [System.Array]) { + & $ToolPath @ArgList + } + elseif (-not [string]::IsNullOrEmpty($ArgList)) { + & $ToolPath $ArgList + } + else { + & $ToolPath + } + return $LASTEXITCODE + } + + $j = Start-Job -ScriptBlock $jobScript -ArgumentList $Tool, $argList -ErrorAction Stop + Write-Host "[Remote] Started job Id=$($j.Id)" + + # Wait-Job uses seconds for timeout + $waitSec = [int]$WaitSeconds + $completed = $j | Wait-Job -Timeout $waitSec + if (-not $completed) { + Write-Host "[Remote] Process did not exit within timeout; stopping job and attempting to kill matching process(es)" + try { + $j | Stop-Job -Force -ErrorAction SilentlyContinue + } catch { } + + try { + $pname = [System.IO.Path]::GetFileNameWithoutExtension($Tool) + $procs = Get-Process -Name $pname -ErrorAction SilentlyContinue + foreach ($p in $procs) { + try { $p | Stop-Process -Force -ErrorAction SilentlyContinue } catch { } + } + } catch { } + + throw "Remote $Tool did not exit within timeout" } else { - Write-Host "[Remote] Process exited with code $($proc.ExitCode)" - if ($proc.ExitCode -ne 0) { throw "Remote $Tool.exe exited with code $($proc.ExitCode)" } + $output = Receive-Job $j -Keep + # The job returns the tool's exit code as the last object + $rc = $output | Where-Object { ($_ -is [int]) -or ($_ -match '^[0-9]+$') } | Select-Object -Last 1 + if ($rc -eq $null) { $rc = 0 } + Write-Host "[Remote] Process (job) exited with code $rc" + if ($rc -ne 0) { throw "Remote $Tool.exe exited with code $rc" } } } else { - Write-Host "[Remote] Not waiting for process to exit (no timeout configured)" + Write-Host "[Remote] Running tool in foreground (no timeout)..." + if ($argList -is [System.Array]) { & $Tool @argList } elseif (-not [string]::IsNullOrEmpty($argList)) { & $Tool $argList } else { & $Tool } + Write-Host "[Remote] Process exited with code $LASTEXITCODE" + if ($LASTEXITCODE -ne 0) { throw "Remote $Tool.exe exited with code $LASTEXITCODE" } } } catch { From 9951dd740702f330ebd7b4ae99f1463104b7bd5b Mon Sep 17 00:00:00 2001 From: Alan Jowett Date: Tue, 2 Dec 2025 12:39:12 -0800 Subject: [PATCH 23/70] Don't timeout server Signed-off-by: Alan Jowett --- .github/scripts/run-echo-test.ps1 | 41 +++++++++---------------------- 1 file changed, 11 insertions(+), 30 deletions(-) diff --git a/.github/scripts/run-echo-test.ps1 b/.github/scripts/run-echo-test.ps1 index 7661b0db..ff8db1a5 100644 --- a/.github/scripts/run-echo-test.ps1 +++ b/.github/scripts/run-echo-test.ps1 @@ -231,7 +231,7 @@ function Stop-WprCpuProfile { # Remote job helpers # ========================= function Invoke-EchoInSession { - param($Session, $RemoteDir, $Name, $Options, [int]$WaitSeconds = 0) + param($Session, $RemoteDir, $Name, $Options) $Job = Invoke-Command -Session $Session -ScriptBlock { param($RemoteDir, $Name, $Options, $WaitSeconds) @@ -276,32 +276,13 @@ function Invoke-EchoInSession { Write-Host "[Remote] Started job Id=$($j.Id)" # Wait-Job uses seconds for timeout - $waitSec = [int]$WaitSeconds - $completed = $j | Wait-Job -Timeout $waitSec - if (-not $completed) { - Write-Host "[Remote] Process did not exit within timeout; stopping job and attempting to kill matching process(es)" - try { - $j | Stop-Job -Force -ErrorAction SilentlyContinue - } catch { } - - try { - $pname = [System.IO.Path]::GetFileNameWithoutExtension($Tool) - $procs = Get-Process -Name $pname -ErrorAction SilentlyContinue - foreach ($p in $procs) { - try { $p | Stop-Process -Force -ErrorAction SilentlyContinue } catch { } - } - } catch { } - - throw "Remote $Tool did not exit within timeout" - } - else { - $output = Receive-Job $j -Keep - # The job returns the tool's exit code as the last object - $rc = $output | Where-Object { ($_ -is [int]) -or ($_ -match '^[0-9]+$') } | Select-Object -Last 1 - if ($rc -eq $null) { $rc = 0 } - Write-Host "[Remote] Process (job) exited with code $rc" - if ($rc -ne 0) { throw "Remote $Tool.exe exited with code $rc" } - } + $completed = $j | Wait-Job + $output = Receive-Job $j -Keep + # The job returns the tool's exit code as the last object + $rc = $output | Where-Object { ($_ -is [int]) -or ($_ -match '^[0-9]+$') } | Select-Object -Last 1 + if ($rc -eq $null) { $rc = 0 } + Write-Host "[Remote] Process (job) exited with code $rc" + if ($rc -ne 0) { throw "Remote $Tool.exe exited with code $rc" } } else { Write-Host "[Remote] Running tool in foreground (no timeout)..." @@ -313,7 +294,7 @@ function Invoke-EchoInSession { catch { throw "Failed to launch or monitor process $Tool $($_.Exception.Message)" } - } -ArgumentList $RemoteDir, $Name, $Options, $WaitSeconds -AsJob -ErrorAction Stop + } -ArgumentList $RemoteDir, $Name, $Options -AsJob -ErrorAction Stop return $Job } @@ -482,7 +463,7 @@ function Run-SendTest { $serverArgs = Normalize-Args -Tokens $serverArgs Write-Host "[Local->Remote] Invoking remote job with arguments:" if ($serverArgs -is [System.Array]) { foreach ($arg in $serverArgs) { Write-Host " $arg" } } else { Write-Host " $serverArgs" } - $Job = Invoke-EchoInSession -Session $Session -RemoteDir $script:RemoteDir -Name "echo_server" -Options $serverArgs -WaitSeconds 20 + $Job = Invoke-EchoInSession -Session $Session -RemoteDir $script:RemoteDir -Name "echo_server" -Options $serverArgs $clientArgs = Convert-ArgStringToArray $SenderOptions $clientArgs = Set-TargetArg -ArgsArray $clientArgs -TargetName $PeerName @@ -512,7 +493,7 @@ function Run-RecvTest { $serverArgs = Normalize-Args -Tokens $serverArgs Write-Host "[Local->Remote] Invoking remote job with arguments:" if ($serverArgs -is [System.Array]) { foreach ($arg in $serverArgs) { Write-Host " $arg" } } else { Write-Host " $serverArgs" } - $Job = Invoke-EchoInSession -Session $Session -RemoteDir $script:RemoteDir -Name "echo_client" -Options $serverArgs -WaitSeconds 20 + $Job = Invoke-EchoInSession -Session $Session -RemoteDir $script:RemoteDir -Name "echo_client" -Options $serverArgs $clientArgs = Convert-ArgStringToArray $ReceiverOptions $clientArgs = Normalize-Args -Tokens $clientArgs From 8febbbc8ad7549ff2c073c99eca0b3a4de76b071 Mon Sep 17 00:00:00 2001 From: Alan Jowett Date: Tue, 2 Dec 2025 15:31:06 -0800 Subject: [PATCH 24/70] Remove redundant parameter Signed-off-by: Alan Jowett --- .github/scripts/run-echo-test.ps1 | 21 --------------------- 1 file changed, 21 deletions(-) diff --git a/.github/scripts/run-echo-test.ps1 b/.github/scripts/run-echo-test.ps1 index ff8db1a5..17cb79a7 100644 --- a/.github/scripts/run-echo-test.ps1 +++ b/.github/scripts/run-echo-test.ps1 @@ -52,25 +52,6 @@ function Convert-ArgStringToArray($s) { return $out } -# Ensure an args array contains a '-target:' entry; replace if present, append if missing -function Set-TargetArg { - param( - [Parameter(Mandatory = $true)] $ArgsArray, - [Parameter(Mandatory = $true)] [string] $TargetName - ) - if ($null -eq $ArgsArray) { $ArgsArray = @() } - $found = $false - for ($i = 0; $i -lt $ArgsArray.Count; $i++) { - if ($ArgsArray[$i] -match '^-target:' ) { - $ArgsArray[$i] = "-target:$TargetName" - $found = $true - break - } - } - if (-not $found) { $ArgsArray += "-target:$TargetName" } - return $ArgsArray -} - # Normalize tokens: prefix '-' only for standalone tokens that don't look like values function Normalize-Args { param([Parameter(Mandatory=$true)][object[]]$Tokens) @@ -466,7 +447,6 @@ function Run-SendTest { $Job = Invoke-EchoInSession -Session $Session -RemoteDir $script:RemoteDir -Name "echo_server" -Options $serverArgs $clientArgs = Convert-ArgStringToArray $SenderOptions - $clientArgs = Set-TargetArg -ArgsArray $clientArgs -TargetName $PeerName $clientArgs = Normalize-Args -Tokens $clientArgs Write-Host "[Local] Running: .\echo_client.exe" @@ -489,7 +469,6 @@ function Run-RecvTest { ) $serverArgs = Convert-ArgStringToArray $SenderOptions - $serverArgs = Set-TargetArg -ArgsArray $serverArgs -TargetName $PeerName $serverArgs = Normalize-Args -Tokens $serverArgs Write-Host "[Local->Remote] Invoking remote job with arguments:" if ($serverArgs -is [System.Array]) { foreach ($arg in $serverArgs) { Write-Host " $arg" } } else { Write-Host " $serverArgs" } From f6c80fe641ba32b71a5621a950fb396da4998e19 Mon Sep 17 00:00:00 2001 From: Alan Jowett Date: Tue, 2 Dec 2025 15:36:45 -0800 Subject: [PATCH 25/70] Build with symbols Signed-off-by: Alan Jowett --- .github/workflows/network-traffic-performance.yml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.github/workflows/network-traffic-performance.yml b/.github/workflows/network-traffic-performance.yml index 8e82a4c5..7f64e77b 100644 --- a/.github/workflows/network-traffic-performance.yml +++ b/.github/workflows/network-traffic-performance.yml @@ -173,6 +173,7 @@ jobs: with: artifact_name: echo_test repository: 'alan-jowett/WinUDPShardedEcho' + config: 'RelWithDebInfo' test_echo_server: if: ${{ inputs.test_tool == 'echo' }} @@ -200,7 +201,7 @@ jobs: Get-ChildItem | % { Remove-Item -Recurse $_ -Force -ErrorAction SilentlyContinue } if (Test-Path ${{ github.workspace }}\echo) { Remove-Item -Recurse -Force ${{ github.workspace }}\echo } if (Test-Path ${{ github.workspace }}\ETL) { Remove-Item -Recurse -Force ${{ github.workspace }}\ETL } - New-Item -ItemType Directory -Path ${{ github.workspace }}\echo\release + New-Item -ItemType Directory -Path ${{ github.workspace }}\echo\RelWithDebInfo New-Item -ItemType Directory -Path ${{ github.workspace }}\ETL - name: Download echo test tool @@ -222,7 +223,7 @@ jobs: Invoke-WebRequest -uri "https://raw.githubusercontent.com/microsoft/netperf/${{inputs.ref}}/.github/scripts/cpu.wprp" -OutFile "${{ github.workspace }}\ETL\cpu.wprp" - name: Download run-echo-test script - working-directory: ${{ github.workspace }}\echo\release + working-directory: ${{ github.workspace }}\echo\RelWithDebInfo run: | Invoke-WebRequest -uri "https://raw.githubusercontent.com/microsoft/netperf/${{inputs.ref}}/.github/scripts/run-echo-test.ps1" -OutFile ".\run-echo-test.ps1" @@ -235,13 +236,13 @@ jobs: Write-Output " TCP IP Tracing: ${{ inputs.tcp_ip_tracing }}" Write-Output " ref: ${{ inputs.ref }}" - - name: Diagonstic - List files in echo release directory + - name: Diagonstic - List files in echo RelWithDebInfo directory run: | - Write-Output "Files in echo release directory:" - Get-ChildItem -Path ${{ github.workspace }}\echo\release -Recurse + Write-Output "Files in echo RelWithDebInfo directory:" + Get-ChildItem -Path ${{ github.workspace }}\echo\RelWithDebInfo -Recurse - name: Run traffic tests - working-directory: ${{ github.workspace }}\echo\release + working-directory: ${{ github.workspace }}\echo\RelWithDebInfo env: PROFILE: ${{ inputs.profile }} SENDER_OPTIONS: ${{ inputs.sender_options }} From 80509d44bb20f09612904214cad2a67e4035f933 Mon Sep 17 00:00:00 2001 From: Alan Jowett Date: Tue, 2 Dec 2025 16:01:53 -0800 Subject: [PATCH 26/70] Fix test tool reference Signed-off-by: Alan Jowett --- .github/workflows/network-traffic-performance.yml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/network-traffic-performance.yml b/.github/workflows/network-traffic-performance.yml index 7f64e77b..d7018291 100644 --- a/.github/workflows/network-traffic-performance.yml +++ b/.github/workflows/network-traffic-performance.yml @@ -27,10 +27,10 @@ on: description: 'Command-line options for the receiver (quoted string)' required: true type: string - cts_traffic_ref: - description: 'Branch or ref to build cts-traffic from (repo: microsoft/ctsTraffic)' + test_tool_ref: + description: 'Branch or ref to build the test tool from' required: false - default: 'master' + default: 'main' type: string test_tool: description: 'The network test tool to use' @@ -55,7 +55,8 @@ jobs: build_artifact: cts-traffic repository: 'microsoft/ctsTraffic' configurations: '["Release"]' - ref: ${{ inputs.cts_traffic_ref }} + # Need to translate main to master for ctsTraffic repo + ref: ${{ inputs.test_tool_ref == 'main' && 'master' || inputs.test_tool_ref }} test_cts_traffic: if: ${{ inputs.test_tool == 'ctsTraffic' }} @@ -174,6 +175,7 @@ jobs: artifact_name: echo_test repository: 'alan-jowett/WinUDPShardedEcho' config: 'RelWithDebInfo' + ref: ${{ inputs.test_tool_ref }} test_echo_server: if: ${{ inputs.test_tool == 'echo' }} From 03e6d995b51a1f51a13dca6f0b30276d3a55e3e6 Mon Sep 17 00:00:00 2001 From: Alan Jowett Date: Tue, 2 Dec 2025 16:28:33 -0800 Subject: [PATCH 27/70] Upload crash dumps Signed-off-by: Alan Jowett --- .../workflows/network-traffic-performance.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.github/workflows/network-traffic-performance.yml b/.github/workflows/network-traffic-performance.yml index d7018291..629d0684 100644 --- a/.github/workflows/network-traffic-performance.yml +++ b/.github/workflows/network-traffic-performance.yml @@ -206,6 +206,14 @@ jobs: New-Item -ItemType Directory -Path ${{ github.workspace }}\echo\RelWithDebInfo New-Item -ItemType Directory -Path ${{ github.workspace }}\ETL + - name: Configure Windows Error Reporting to make a local copy of any crashes that occur. + id: configure_windows_error_reporting + run: | + $DumpFolder = "${{ github.workspace }}\echo\RelWithDebInfo" + New-Item -Path "HKLM:\SOFTWARE\Microsoft\Windows\Windows Error Reporting\LocalDumps" -ErrorAction SilentlyContinue + New-ItemProperty -Path "HKLM:\SOFTWARE\Microsoft\Windows\Windows Error Reporting\LocalDumps" -Name "DumpType" -Value 2 -PropertyType DWord -ErrorAction SilentlyContinue + New-ItemProperty -Path "HKLM:\SOFTWARE\Microsoft\Windows\Windows Error Reporting\LocalDumps" -Name "DumpFolder" -Value "$DumpFolder" -PropertyType ExpandString -ErrorAction SilentlyContinue + - name: Download echo test tool uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 with: @@ -295,3 +303,11 @@ jobs: with: name: cpu_profile_${{ matrix.vec.env }}_${{ matrix.vec.os }}_${{ matrix.vec.arch }} path: ${{ github.workspace }}\ETL\cpu_profile-*.etl + + - name: Upload Crash Dumps + if: always() + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 + with: + name: crash_dumps_${{ matrix.vec.env }}_${{ matrix.vec.os }}_${{ matrix.vec.arch }} + path: | + ${{ github.workspace }}\echo\RelWithDebInfo \ No newline at end of file From ca73dccacc07ccd1cf83047bd65866767ffb6564 Mon Sep 17 00:00:00 2001 From: Alan Jowett Date: Tue, 2 Dec 2025 17:19:09 -0800 Subject: [PATCH 28/70] Remove concurrency limit Signed-off-by: Alan Jowett --- .github/workflows/network-traffic-performance.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/network-traffic-performance.yml b/.github/workflows/network-traffic-performance.yml index 629d0684..ec2091f2 100644 --- a/.github/workflows/network-traffic-performance.yml +++ b/.github/workflows/network-traffic-performance.yml @@ -41,10 +41,10 @@ on: - ctsTraffic - echo -concurrency: - group: >- - traffic-${{ github.event.client_payload.sha || inputs.ref || github.event.pull_request.number || 'main' }} - cancel-in-progress: true +# concurrency: +# group: >- +# traffic-${{ github.event.client_payload.sha || inputs.ref || github.event.pull_request.number || 'main' }} +# cancel-in-progress: true jobs: build_cts_traffic: From e5f9989b89cf63725bd47ee53104bd1780f0b678 Mon Sep 17 00:00:00 2001 From: Alan Jowett Date: Wed, 3 Dec 2025 09:00:00 -0800 Subject: [PATCH 29/70] Capture average CPU usage Signed-off-by: Alan Jowett --- .github/scripts/run-echo-test.ps1 | 58 ++++++++++++++++++- .../workflows/network-traffic-performance.yml | 6 ++ 2 files changed, 63 insertions(+), 1 deletion(-) diff --git a/.github/scripts/run-echo-test.ps1 b/.github/scripts/run-echo-test.ps1 index 17cb79a7..de8870fd 100644 --- a/.github/scripts/run-echo-test.ps1 +++ b/.github/scripts/run-echo-test.ps1 @@ -2,7 +2,8 @@ param( [switch]$CpuProfile, [string]$PeerName, [string]$SenderOptions, - [string]$ReceiverOptions + [string]$ReceiverOptions, + [string]$Duration = "60" # Duration in seconds for the test runs (default 60s) ) Set-StrictMode -Version Latest @@ -19,6 +20,15 @@ if ($SenderOptions -notmatch '--server') { $SenderOptions += " --server $PeerName" } +# Add duration option if specified and not already present to both sender and receiver +if ($Duration -and $Duration -gt 0 -and $SenderOptions -notmatch '--duration') { + $SenderOptions += " --duration $Duration" +} + +if ($Duration -and $Duration -gt 0 -and $ReceiverOptions -notmatch '--duration') { + $ReceiverOptions += " --duration $Duration" +} + # Make errors terminate so catch can handle them $ErrorActionPreference = 'Stop' $Session = $null @@ -488,6 +498,37 @@ function Run-RecvTest { Receive-JobOrThrow -Job $Job } +function CaptureCpuUsagePerformanceMonitorAsJob { + param( + [Parameter(Mandatory=$true)][string]$DurationSeconds + ) + # Ensure we pass a numeric duration into the job and use that value inside + $intDuration = [int]::Parse($DurationSeconds) + + $cpuMonitorJob = Start-Job -ScriptBlock { + param($duration) + + $counter = '\Processor(_Total)\% Processor Time' + $d = [int]$duration + + try { + $samples = Get-Counter -Counter $counter -SampleInterval 1 -MaxSamples $d -ErrorAction Stop + $values = $samples.CounterSamples | ForEach-Object { [double]$_.CookedValue } + } + catch { + $values = @(0) + } + + + $average = ($values | Measure-Object -Average).Average + + # Emit a raw numeric value so the caller can parse it reliably + Write-Output $average + } -ArgumentList $intDuration + + return $cpuMonitorJob +} + function Restore-FirewallAndCleanup { param([object]$Session) @@ -553,10 +594,25 @@ try { # Copy tool to remote Copy-EchoToRemote -Session $Session + # Launch CaptureCpuUsagePerformanceMonitor as a background job + $cpuMonitorJob = CaptureCpuUsagePerformanceMonitorAsJob $Duration + # Run tests Run-SendTest -PeerName $PeerName -Session $Session -SenderOptions $SenderOptions -ReceiverOptions $ReceiverOptions + + # Recover CPU usage data + $cpuUsage = Receive-Job -Job $cpuMonitorJob -Wait -AutoRemoveJob + Write-Host "Average CPU Usage during test: $([math]::Round($cpuUsage, 2)) %" + + # Launch another CPU monitor for the recv test + $cpuMonitorJob = CaptureCpuUsagePerformanceMonitorAsJob $Duration + Run-RecvTest -PeerName $PeerName -Session $Session -SenderOptions $SenderOptions -ReceiverOptions $ReceiverOptions + # Recover CPU usage data + $cpuUsage = Receive-Job -Job $cpuMonitorJob -Wait -AutoRemoveJob + Write-Host "Average CPU Usage during test: $([math]::Round($cpuUsage, 2)) %" + Write-Host "echo tests completed successfully." } catch { diff --git a/.github/workflows/network-traffic-performance.yml b/.github/workflows/network-traffic-performance.yml index ec2091f2..bb68dd28 100644 --- a/.github/workflows/network-traffic-performance.yml +++ b/.github/workflows/network-traffic-performance.yml @@ -40,6 +40,11 @@ on: options: - ctsTraffic - echo + duration: + description: 'Duration of the test in seconds' + required: false + default: '60' + type: string # concurrency: # group: >- @@ -271,6 +276,7 @@ jobs: '-PeerName', '"netperf-peer"', '-SenderOptions', '"' + $env:SENDER_OPTIONS + '"', '-ReceiverOptions', '"' + $env:RECEIVER_OPTIONS + '"' + '-Duration ', ${{ inputs.duration }} ) -join ' ' Write-Output "Running: $cmd" From ca6561a04a66a921f1ba0ec4551a3c11d5c4f009 Mon Sep 17 00:00:00 2001 From: Alan Jowett Date: Wed, 3 Dec 2025 09:25:08 -0800 Subject: [PATCH 30/70] Add per CPU stats Signed-off-by: Alan Jowett --- .github/scripts/performance_utlilites.psm1 | 300 ++++++++++++++++++ .github/scripts/run-echo-test.ps1 | 79 ++++- .../workflows/network-traffic-performance.yml | 1 + 3 files changed, 370 insertions(+), 10 deletions(-) create mode 100644 .github/scripts/performance_utlilites.psm1 diff --git a/.github/scripts/performance_utlilites.psm1 b/.github/scripts/performance_utlilites.psm1 new file mode 100644 index 00000000..b698c9e5 --- /dev/null +++ b/.github/scripts/performance_utlilites.psm1 @@ -0,0 +1,300 @@ +# WPR CPU profiling helpers +$script:WprProfiles = @{} + +function Start-WprCpuProfile { + param([Parameter(Mandatory=$true)][string]$Which) + + if (-not $CpuProfile) { return } + + $Workspace = $env:GITHUB_WORKSPACE + $etlDir = Join-Path $Workspace 'ETL' + if (-not (Test-Path $etlDir)) { New-Item -ItemType Directory -Path $etlDir | Out-Null } + + $WprProfile = Join-Path $etlDir 'cpu.wprp' + + $outFile = Join-Path $etlDir ("cpu_profile-$Which.etl") + if (Test-Path $outFile) { Remove-Item $outFile -Force -ErrorAction SilentlyContinue } + + Write-Host "Starting WPR CPU profiling -> $outFile" + try { + # Check if WPR is already running to avoid the "profiles are already running" error + $status = $null + try { + $status = & wpr -status 2>&1 + } catch { + $status = $_.ToString() + } + + if ($status -and $status -match 'profile(s)?\s+are\s+already\s+running|Profiles are already running|The profiles are already running') { + Write-Host "WPR already running. Cancelling any existing profiles so we can start a fresh one..." + try { + & wpr -cancel 2>&1 | Out-Null + Start-Sleep -Seconds 1 + } + catch { + Write-Host "Failed to cancel existing WPR session: $($_.Exception.Message). Proceeding to start a new profile anyway." + } + } + + try { + & wpr -start $WprProfile -filemode | Out-Null + } + catch { + Write-Host "wpr -start with custom profile failed: $($_.Exception.Message). Falling back to built-in CPU profile." + try { & wpr -start CPU -filemode | Out-Null } catch { Write-Host "Fallback CPU start also failed: $($_.Exception.Message)" } + } + $script:WprProfiles[$Which] = $outFile + } + catch { + Write-Host "Failed to start WPR: $($_.Exception.Message)" + } +} + +function Stop-WprCpuProfile { + param([Parameter(Mandatory=$true)][string]$Which) + + if (-not $CpuProfile) { return } + + if (-not $script:WprProfiles.ContainsKey($Which)) { + Write-Host "No WPR profile active for '$Which'" + return + } + + $outFile = $script:WprProfiles[$Which] + Write-Host "Stopping WPR CPU profiling, saving to $outFile" + try { + # Attempt to stop WPR and save to the given file. If no profile is running, log and continue. + try { + & wpr -stop $outFile | Out-Null + } + catch { + Write-Host "wpr -stop failed: $($_.Exception.Message). Attempting to query status..." + try { + $s = & wpr -status 2>&1 + Write-Host "WPR status: $s" + } catch { } + } + $script:WprProfiles.Remove($Which) | Out-Null + } + catch { + Write-Host "Failed to stop WPR: $($_.Exception.Message)" + } +} + +function Receive-JobOrThrow { + param([Parameter(Mandatory)] $Job) + + Wait-Job $Job | Out-Null + + # Drain output (keep so we can inspect again if needed) + $null = Receive-Job $Job -Keep + + $errs = @() + foreach ($cj in $Job.ChildJobs) { + if ($cj.Error -and $cj.Error.Count -gt 0) { + $errs += $cj.Error + } + if ($cj.JobStateInfo.State -eq 'Failed' -and $cj.JobStateInfo.Reason) { + $errs += $cj.JobStateInfo.Reason + } + } + + if ($errs.Count -gt 0) { + foreach ($er in $errs) { + if ($er -is [System.Management.Automation.ErrorRecord]) { + $er | Write-DetailedError + } + else { + Write-Host $er + } + } + throw "One or more remote errors occurred (job id: $($Job.Id))." + } + + if ($Job.State -eq 'Failed') { + throw "Remote job failed (job id: $($Job.Id)): $($Job.JobStateInfo.Reason)" + } +} + +function Create-Session { + param( + [Parameter(Mandatory=$true)][string]$PeerName, + [string]$RemotePSConfiguration = 'PowerShell.7' + ) + + $script:RemotePSConfiguration = $RemotePSConfiguration + $script:RemoteDir = 'C:\_work' + + $Username = (Get-ItemProperty 'HKLM:\Software\Microsoft\Windows NT\CurrentVersion\Winlogon').DefaultUserName + $Password = (Get-ItemProperty 'HKLM:\Software\Microsoft\Windows NT\CurrentVersion\Winlogon').DefaultPassword | ConvertTo-SecureString -AsPlainText -Force + $Creds = New-Object System.Management.Automation.PSCredential ($Username, $Password) + + try { + Write-Host "Creating PSSession to $PeerName using configuration '$RemotePSConfiguration'..." + $s = New-PSSession -ComputerName $PeerName -Credential $Creds -ConfigurationName $RemotePSConfiguration -ErrorAction Stop + Write-Host "Session created using configuration '$RemotePSConfiguration'." + } + catch { + Write-Host "Failed to create session using configuration '$RemotePSConfiguration': $($_.Exception.Message)" + Write-Host "Attempting fallback: creating session without ConfigurationName..." + try { + $s = New-PSSession -ComputerName $PeerName -Credential $Creds -ErrorAction Stop + Write-Host "Session created using default configuration." + } + catch { + Write-Host "Fallback session creation failed: $($_.Exception.Message)" + throw "Failed to create remote session to $PeerName" + } + } + + $script:Session = $s + return $s +} + +function Save-And-Disable-Firewalls { + param([Parameter(Mandatory=$true)]$Session) + + # Coerce possible multi-output (array) from Create-Session into the actual PSSession object. + if ($Session -is [System.Array]) { + $found = $Session | Where-Object { $_ -is [System.Management.Automation.Runspaces.PSSession] } + if ($found -and $found.Count -gt 0) { $Session = $found[0] } + else { $Session = $Session[0] } + } + + if (-not ($Session -is [System.Management.Automation.Runspaces.PSSession])) { + throw "Save-And-Disable-Firewalls requires a PSSession object. Got: $($Session.GetType().FullName) - $Session" + } + + Write-Host "Saving and disabling local firewall profiles..." + $script:localFwState = Get-NetFirewallProfile -Profile Domain, Public, Private | Select-Object Name, Enabled + Set-NetFirewallProfile -Profile Domain, Public, Private -Enabled False + + Write-Host "Disabling firewall on remote machine..." + Invoke-Command -Session $Session -ScriptBlock { + param() + $fw = Get-NetFirewallProfile -Profile Domain, Public, Private | Select-Object Name, Enabled + Set-Variable -Name __SavedFirewallState -Value $fw -Scope Global -Force + Set-NetFirewallProfile -Profile Domain, Public, Private -Enabled False + } -ErrorAction Stop +} + + +function CaptureCpuUsagePerformanceMonitorAsJob { + param( + [Parameter(Mandatory=$true)][string]$DurationSeconds + ) + # Ensure we pass a numeric duration into the job and use that value inside + $intDuration = [int]::Parse($DurationSeconds) + + $cpuMonitorJob = Start-Job -ScriptBlock { + param($duration) + + $counter = '\Processor(_Total)\% Processor Time' + $d = [int]$duration + + try { + $samples = Get-Counter -Counter $counter -SampleInterval 1 -MaxSamples $d -ErrorAction Stop + $values = $samples.CounterSamples | ForEach-Object { [double]$_.CookedValue } + } + catch { + $values = @(0) + } + + + $average = ($values | Measure-Object -Average).Average + + # Emit a raw numeric value so the caller can parse it reliably + Write-Output $average + } -ArgumentList $intDuration + + return $cpuMonitorJob +} + +function CaptureIndividualCpuUsagePerformanceMonitorAsJob { + param( + [Parameter(Mandatory=$true)][string]$DurationSeconds + ) + + # Ensure we pass a numeric duration into the job and use that value inside + $intDuration = [int]::Parse($DurationSeconds) + + $cpuMonitorJob = Start-Job -ScriptBlock { + param($duration) + + $counter = '\Processor(*)\% Processor Time' + $d = [int]$duration + + try { + $samples = Get-Counter -Counter $counter -SampleInterval 1 -MaxSamples $d -ErrorAction Stop + # Group samples by instance (processor index) and compute average per instance + $grouped = $samples.CounterSamples | Group-Object -Property InstanceName + $results = @() + foreach ($g in $grouped) { + $vals = $g.Group | ForEach-Object { [double]$_.CookedValue } + $avg = ($vals | Measure-Object -Average).Average + $results += [PSCustomObject]@{ Processor = $g.Name; Average = $avg } + } + # Sort by processor name to have consistent ordering (e.g., _Total last or first) + $sorted = $results | Sort-Object @{Expression={$_.Processor -replace '^CPU',''}},Processor + # Emit numeric array (only per-CPU numeric averages, excluding the _Total instance) + $numeric = $sorted | Where-Object { $_.Processor -ne '_Total' } | ForEach-Object { [double]$_.Average } + } + catch { + $numeric = @(0) + } + + # Emit the numeric array so the caller receives per-CPU averages + Write-Output $numeric + } -ArgumentList $intDuration + + return $cpuMonitorJob +} + + + +function Restore-FirewallAndCleanup { + param([object]$Session) + + try { + if ($null -ne $Session) { + try { + Write-Host "Restoring firewall state on remote machine..." + Invoke-Command -Session $Session -ScriptBlock { + if (Get-Variable -Name __SavedFirewallState -Scope Global -ErrorAction SilentlyContinue) { + $saved = Get-Variable -Name __SavedFirewallState -Scope Global -ValueOnly + foreach ($p in $saved) { + Set-NetFirewallProfile -Profile $p.Name -Enabled $p.Enabled + } + Remove-Variable -Name __SavedFirewallState -Scope Global -ErrorAction SilentlyContinue + } + else { + Set-NetFirewallProfile -Profile Domain, Public, Private -Enabled True + } + } -ErrorAction SilentlyContinue + } + catch { + $_ | Write-DetailedError + } + + try { + Remove-PSSession $Session -ErrorAction SilentlyContinue + } + catch { + $_ | Write-DetailedError + } + } + + Write-Host "Restoring local firewall state..." + if ($localFwState) { + foreach ($p in $localFwState) { + Set-NetFirewallProfile -Profile $p.Name -Enabled $p.Enabled + } + } + else { + Set-NetFirewallProfile -Profile Domain, Public, Private -Enabled True + } + } + catch { + $_ | Write-DetailedError + } +} diff --git a/.github/scripts/run-echo-test.ps1 b/.github/scripts/run-echo-test.ps1 index de8870fd..10b8cf46 100644 --- a/.github/scripts/run-echo-test.ps1 +++ b/.github/scripts/run-echo-test.ps1 @@ -529,6 +529,47 @@ function CaptureCpuUsagePerformanceMonitorAsJob { return $cpuMonitorJob } +function CaptureIndividualCpuUsagePerformanceMonitorAsJob { + param( + [Parameter(Mandatory=$true)][string]$DurationSeconds + ) + + # Ensure we pass a numeric duration into the job and use that value inside + $intDuration = [int]::Parse($DurationSeconds) + + $cpuMonitorJob = Start-Job -ScriptBlock { + param($duration) + + $counter = '\Processor(*)\% Processor Time' + $d = [int]$duration + + try { + $samples = Get-Counter -Counter $counter -SampleInterval 1 -MaxSamples $d -ErrorAction Stop + # Group samples by instance (processor index) and compute average per instance + $grouped = $samples.CounterSamples | Group-Object -Property InstanceName + $results = @() + foreach ($g in $grouped) { + $vals = $g.Group | ForEach-Object { [double]$_.CookedValue } + $avg = ($vals | Measure-Object -Average).Average + $results += [PSCustomObject]@{ Processor = $g.Name; Average = $avg } + } + # Sort by processor name to have consistent ordering (e.g., _Total last or first) + $sorted = $results | Sort-Object @{Expression={$_.Processor -replace '^CPU',''}},Processor + # Emit numeric array (only per-CPU numeric averages, excluding the _Total instance) + $numeric = $sorted | Where-Object { $_.Processor -ne '_Total' } | ForEach-Object { [double]$_.Average } + } + catch { + $numeric = @(0) + } + + # Emit the numeric array so the caller receives per-CPU averages + Write-Output $numeric + } -ArgumentList $intDuration + + return $cpuMonitorJob +} + + function Restore-FirewallAndCleanup { param([object]$Session) @@ -594,24 +635,42 @@ try { # Copy tool to remote Copy-EchoToRemote -Session $Session - # Launch CaptureCpuUsagePerformanceMonitor as a background job - $cpuMonitorJob = CaptureCpuUsagePerformanceMonitorAsJob $Duration + # Launch per-CPU usage monitor as a background job (returns array of per-CPU averages) + $cpuMonitorJob = CaptureIndividualCpuUsagePerformanceMonitorAsJob $Duration # Run tests Run-SendTest -PeerName $PeerName -Session $Session -SenderOptions $SenderOptions -ReceiverOptions $ReceiverOptions - # Recover CPU usage data - $cpuUsage = Receive-Job -Job $cpuMonitorJob -Wait -AutoRemoveJob - Write-Host "Average CPU Usage during test: $([math]::Round($cpuUsage, 2)) %" + # Recover CPU usage data (monitor returns per-CPU averages). Print per-CPU values. + $cpuUsagePerCpu = Receive-Job -Job $cpuMonitorJob -Wait -AutoRemoveJob + if ($cpuUsagePerCpu -is [System.Array]) { + $i = 0 + foreach ($val in $cpuUsagePerCpu) { + $i++ + Write-Host "CPU$i: $([math]::Round([double]$val, 2)) %" + } + } + else { + Write-Host "CPU1: $([math]::Round([double]$cpuUsagePerCpu, 2)) %" + } - # Launch another CPU monitor for the recv test - $cpuMonitorJob = CaptureCpuUsagePerformanceMonitorAsJob $Duration + # Launch another per-CPU usage monitor for the recv test + $cpuMonitorJob = CaptureIndividualCpuUsagePerformanceMonitorAsJob $Duration Run-RecvTest -PeerName $PeerName -Session $Session -SenderOptions $SenderOptions -ReceiverOptions $ReceiverOptions - # Recover CPU usage data - $cpuUsage = Receive-Job -Job $cpuMonitorJob -Wait -AutoRemoveJob - Write-Host "Average CPU Usage during test: $([math]::Round($cpuUsage, 2)) %" + # Recover CPU usage data (monitor returns per-CPU averages). Print per-CPU values. + $cpuUsagePerCpu = Receive-Job -Job $cpuMonitorJob -Wait -AutoRemoveJob + if ($cpuUsagePerCpu -is [System.Array]) { + $i = 0 + foreach ($val in $cpuUsagePerCpu) { + $i++ + Write-Host "CPU$i: $([math]::Round([double]$val, 2)) %" + } + } + else { + Write-Host "CPU1: $([math]::Round([double]$cpuUsagePerCpu, 2)) %" + } Write-Host "echo tests completed successfully." } diff --git a/.github/workflows/network-traffic-performance.yml b/.github/workflows/network-traffic-performance.yml index bb68dd28..3e2ae515 100644 --- a/.github/workflows/network-traffic-performance.yml +++ b/.github/workflows/network-traffic-performance.yml @@ -250,6 +250,7 @@ jobs: Write-Output " ReceiverOptions: ${{ inputs.receiver_options }}" Write-Output " TCP IP Tracing: ${{ inputs.tcp_ip_tracing }}" Write-Output " ref: ${{ inputs.ref }}" + Write-Output " Duration: ${{ inputs.duration }}" - name: Diagonstic - List files in echo RelWithDebInfo directory run: | From 57e222ba640e7499357a4c6dcb9d2a61511fb5a7 Mon Sep 17 00:00:00 2001 From: Alan Jowett Date: Wed, 3 Dec 2025 09:34:15 -0800 Subject: [PATCH 31/70] Add per CPU stats Signed-off-by: Alan Jowett --- .github/workflows/network-traffic-performance.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/network-traffic-performance.yml b/.github/workflows/network-traffic-performance.yml index 3e2ae515..82f3cc35 100644 --- a/.github/workflows/network-traffic-performance.yml +++ b/.github/workflows/network-traffic-performance.yml @@ -277,7 +277,7 @@ jobs: '-PeerName', '"netperf-peer"', '-SenderOptions', '"' + $env:SENDER_OPTIONS + '"', '-ReceiverOptions', '"' + $env:RECEIVER_OPTIONS + '"' - '-Duration ', ${{ inputs.duration }} + '-Duration ', '"${{ inputs.duration }}"' ) -join ' ' Write-Output "Running: $cmd" From 9185a18a7b299d7d0f4de6fbffcefc29080ecba8 Mon Sep 17 00:00:00 2001 From: Alan Jowett Date: Wed, 3 Dec 2025 09:36:39 -0800 Subject: [PATCH 32/70] Add per CPU stats Signed-off-by: Alan Jowett --- .github/workflows/network-traffic-performance.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/network-traffic-performance.yml b/.github/workflows/network-traffic-performance.yml index 82f3cc35..e2dba083 100644 --- a/.github/workflows/network-traffic-performance.yml +++ b/.github/workflows/network-traffic-performance.yml @@ -281,7 +281,7 @@ jobs: ) -join ' ' Write-Output "Running: $cmd" - Invoke-Expression $cmd + & $cmd - name: Optional - Stop TCPIP tracing if: ${{ github.event.inputs.tcp_ip_tracing == 'true' }} From 398e3f9726d46d7d91c0dc7255a55f8995fcf8a4 Mon Sep 17 00:00:00 2001 From: Alan Jowett Date: Wed, 3 Dec 2025 09:38:16 -0800 Subject: [PATCH 33/70] Add per CPU stats Signed-off-by: Alan Jowett --- .github/scripts/run-echo-test.ps1 | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/scripts/run-echo-test.ps1 b/.github/scripts/run-echo-test.ps1 index 10b8cf46..13bc9363 100644 --- a/.github/scripts/run-echo-test.ps1 +++ b/.github/scripts/run-echo-test.ps1 @@ -647,11 +647,11 @@ try { $i = 0 foreach ($val in $cpuUsagePerCpu) { $i++ - Write-Host "CPU$i: $([math]::Round([double]$val, 2)) %" + Write-Host "CPU$i $([math]::Round([double]$val, 2)) %" } } else { - Write-Host "CPU1: $([math]::Round([double]$cpuUsagePerCpu, 2)) %" + Write-Host "CPU1 $([math]::Round([double]$cpuUsagePerCpu, 2)) %" } # Launch another per-CPU usage monitor for the recv test @@ -665,11 +665,11 @@ try { $i = 0 foreach ($val in $cpuUsagePerCpu) { $i++ - Write-Host "CPU$i: $([math]::Round([double]$val, 2)) %" + Write-Host "CPU$i $([math]::Round([double]$val, 2)) %" } } else { - Write-Host "CPU1: $([math]::Round([double]$cpuUsagePerCpu, 2)) %" + Write-Host "CPU1 $([math]::Round([double]$cpuUsagePerCpu, 2)) %" } Write-Host "echo tests completed successfully." From 05853a11715bb5740a698fffee258f2bb9604a60 Mon Sep 17 00:00:00 2001 From: Alan Jowett Date: Wed, 3 Dec 2025 09:40:07 -0800 Subject: [PATCH 34/70] Add per CPU stats Signed-off-by: Alan Jowett --- .github/workflows/network-traffic-performance.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/network-traffic-performance.yml b/.github/workflows/network-traffic-performance.yml index e2dba083..82f3cc35 100644 --- a/.github/workflows/network-traffic-performance.yml +++ b/.github/workflows/network-traffic-performance.yml @@ -281,7 +281,7 @@ jobs: ) -join ' ' Write-Output "Running: $cmd" - & $cmd + Invoke-Expression $cmd - name: Optional - Stop TCPIP tracing if: ${{ github.event.inputs.tcp_ip_tracing == 'true' }} From c2e494c11a5c9a14d4930d87913fb0bd40dae309 Mon Sep 17 00:00:00 2001 From: Alan Jowett Date: Wed, 3 Dec 2025 09:58:39 -0800 Subject: [PATCH 35/70] Add per CPU stats Signed-off-by: Alan Jowett --- .github/scripts/run-echo-test.ps1 | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/.github/scripts/run-echo-test.ps1 b/.github/scripts/run-echo-test.ps1 index 13bc9363..31c7b14c 100644 --- a/.github/scripts/run-echo-test.ps1 +++ b/.github/scripts/run-echo-test.ps1 @@ -649,6 +649,9 @@ try { $i++ Write-Host "CPU$i $([math]::Round([double]$val, 2)) %" } + # Compute and print overall average across all CPUs + $overall = (($cpuUsagePerCpu | Measure-Object -Average).Average) + Write-Host "Overall average CPU Usage: $([math]::Round($overall, 2)) %" } else { Write-Host "CPU1 $([math]::Round([double]$cpuUsagePerCpu, 2)) %" @@ -667,6 +670,9 @@ try { $i++ Write-Host "CPU$i $([math]::Round([double]$val, 2)) %" } + # Compute and print overall average across all CPUs + $overall = (($cpuUsagePerCpu | Measure-Object -Average).Average) + Write-Host "Overall average CPU Usage: $([math]::Round($overall, 2)) %" } else { Write-Host "CPU1 $([math]::Round([double]$cpuUsagePerCpu, 2)) %" From 83d8e85021f33567468291c4561532ecc1757465 Mon Sep 17 00:00:00 2001 From: Alan Jowett Date: Wed, 3 Dec 2025 10:27:01 -0800 Subject: [PATCH 36/70] Reset hyper-v checkpoint and gather all groups Signed-off-by: Alan Jowett --- .github/scripts/run-echo-test.ps1 | 57 +++++++------------ .../workflows/network-traffic-performance.yml | 18 +++++- 2 files changed, 36 insertions(+), 39 deletions(-) diff --git a/.github/scripts/run-echo-test.ps1 b/.github/scripts/run-echo-test.ps1 index 31c7b14c..7a2ba15d 100644 --- a/.github/scripts/run-echo-test.ps1 +++ b/.github/scripts/run-echo-test.ps1 @@ -498,37 +498,6 @@ function Run-RecvTest { Receive-JobOrThrow -Job $Job } -function CaptureCpuUsagePerformanceMonitorAsJob { - param( - [Parameter(Mandatory=$true)][string]$DurationSeconds - ) - # Ensure we pass a numeric duration into the job and use that value inside - $intDuration = [int]::Parse($DurationSeconds) - - $cpuMonitorJob = Start-Job -ScriptBlock { - param($duration) - - $counter = '\Processor(_Total)\% Processor Time' - $d = [int]$duration - - try { - $samples = Get-Counter -Counter $counter -SampleInterval 1 -MaxSamples $d -ErrorAction Stop - $values = $samples.CounterSamples | ForEach-Object { [double]$_.CookedValue } - } - catch { - $values = @(0) - } - - - $average = ($values | Measure-Object -Average).Average - - # Emit a raw numeric value so the caller can parse it reliably - Write-Output $average - } -ArgumentList $intDuration - - return $cpuMonitorJob -} - function CaptureIndividualCpuUsagePerformanceMonitorAsJob { param( [Parameter(Mandatory=$true)][string]$DurationSeconds @@ -540,23 +509,35 @@ function CaptureIndividualCpuUsagePerformanceMonitorAsJob { $cpuMonitorJob = Start-Job -ScriptBlock { param($duration) - $counter = '\Processor(*)\% Processor Time' + # Use the Processor Information counter which contains CPU instances across all groups + # (e.g., "0,0", "0,1", "1,0" etc.) so we capture CPUs from every group, not just group 0. + $counter = '\Processor Information(*)\% Processor Time' $d = [int]$duration try { $samples = Get-Counter -Counter $counter -SampleInterval 1 -MaxSamples $d -ErrorAction Stop - # Group samples by instance (processor index) and compute average per instance + # Group samples by instance (processor information name) and compute average per instance. + # InstanceName for Processor Information uses formats like "0,0" (group,index) or descriptive names. $grouped = $samples.CounterSamples | Group-Object -Property InstanceName $results = @() foreach ($g in $grouped) { + $instName = $g.Name + # Normalize instance names: skip the _Total instance and any empty names + if ([string]::IsNullOrEmpty($instName) -or $instName -eq '_Total') { continue } $vals = $g.Group | ForEach-Object { [double]$_.CookedValue } $avg = ($vals | Measure-Object -Average).Average - $results += [PSCustomObject]@{ Processor = $g.Name; Average = $avg } + $results += [PSCustomObject]@{ Processor = $instName; Average = $avg } } - # Sort by processor name to have consistent ordering (e.g., _Total last or first) - $sorted = $results | Sort-Object @{Expression={$_.Processor -replace '^CPU',''}},Processor - # Emit numeric array (only per-CPU numeric averages, excluding the _Total instance) - $numeric = $sorted | Where-Object { $_.Processor -ne '_Total' } | ForEach-Object { [double]$_.Average } + # Sort by numeric ordering where possible, otherwise by name for consistent output + $sorted = $results | Sort-Object @{Expression={ + $n = $_.Processor -replace '[^0-9,]','' + # If the processor string contains a comma (group,index), split and compute a sortable key + if ($n -match ',') { $parts = $n -split ','; return ([int]$parts[0]*1000 + [int]$parts[1]) } + if ($n -match '^[0-9]+$') { return [int]$n } + return $_.Processor + }},Processor + # Emit numeric array of per-CPU numeric averages + $numeric = $sorted | ForEach-Object { [double]$_.Average } } catch { $numeric = @(0) diff --git a/.github/workflows/network-traffic-performance.yml b/.github/workflows/network-traffic-performance.yml index 82f3cc35..c9f14fcf 100644 --- a/.github/workflows/network-traffic-performance.yml +++ b/.github/workflows/network-traffic-performance.yml @@ -317,4 +317,20 @@ jobs: with: name: crash_dumps_${{ matrix.vec.env }}_${{ matrix.vec.os }}_${{ matrix.vec.arch }} path: | - ${{ github.workspace }}\echo\RelWithDebInfo \ No newline at end of file + ${{ github.workspace }}\echo\RelWithDebInfo + + attempt-reset-labecho: + name: Attempting to reset lab. Status of this job does not indicate result of lab reset. Look at job details. + needs: [test_echo_server] + if: ${{ always() }} + uses: microsoft/netperf/.github/workflows/schedule-lab-reset.yml@main + with: + workflowId: ${{ github.run_id }} + + attempt-reset-lab-cts: + name: Attempting to reset lab. Status of this job does not indicate result of lab reset. Look at job details. + needs: [test_cts_traffic] + if: ${{ always() }} + uses: microsoft/netperf/.github/workflows/schedule-lab-reset.yml@main + with: + workflowId: ${{ github.run_id }} \ No newline at end of file From 4762060b96a409e6dd1fa6f8755060ecc976058f Mon Sep 17 00:00:00 2001 From: Alan Jowett Date: Wed, 3 Dec 2025 10:28:46 -0800 Subject: [PATCH 37/70] Reset hyper-v checkpoint and gather all groups Signed-off-by: Alan Jowett --- .github/workflows/network-traffic-performance.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/network-traffic-performance.yml b/.github/workflows/network-traffic-performance.yml index c9f14fcf..29b6a72d 100644 --- a/.github/workflows/network-traffic-performance.yml +++ b/.github/workflows/network-traffic-performance.yml @@ -46,6 +46,8 @@ on: default: '60' type: string +permissions: write-all + # concurrency: # group: >- # traffic-${{ github.event.client_payload.sha || inputs.ref || github.event.pull_request.number || 'main' }} From 711afafd2d1f5575224979eefbdc0dd157409aee Mon Sep 17 00:00:00 2001 From: Alan Jowett Date: Wed, 3 Dec 2025 10:30:24 -0800 Subject: [PATCH 38/70] Reset hyper-v checkpoint and gather all groups Signed-off-by: Alan Jowett --- .github/workflows/network-traffic-performance.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/network-traffic-performance.yml b/.github/workflows/network-traffic-performance.yml index 29b6a72d..9c293960 100644 --- a/.github/workflows/network-traffic-performance.yml +++ b/.github/workflows/network-traffic-performance.yml @@ -322,6 +322,7 @@ jobs: ${{ github.workspace }}\echo\RelWithDebInfo attempt-reset-labecho: + if: ${{ inputs.test_tool == 'echo' }} name: Attempting to reset lab. Status of this job does not indicate result of lab reset. Look at job details. needs: [test_echo_server] if: ${{ always() }} @@ -330,6 +331,7 @@ jobs: workflowId: ${{ github.run_id }} attempt-reset-lab-cts: + if: ${{ inputs.test_tool == 'ctsTraffic' }} name: Attempting to reset lab. Status of this job does not indicate result of lab reset. Look at job details. needs: [test_cts_traffic] if: ${{ always() }} From dd5d333728c495a5d34a3af5df7f16772cc8fe3f Mon Sep 17 00:00:00 2001 From: Alan Jowett Date: Wed, 3 Dec 2025 10:34:11 -0800 Subject: [PATCH 39/70] Reset hyper-v checkpoint and gather all groups Signed-off-by: Alan Jowett --- .../workflows/network-traffic-performance.yml | 39 +++++++++++-------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/.github/workflows/network-traffic-performance.yml b/.github/workflows/network-traffic-performance.yml index 9c293960..f93fea6c 100644 --- a/.github/workflows/network-traffic-performance.yml +++ b/.github/workflows/network-traffic-performance.yml @@ -174,6 +174,19 @@ jobs: name: cpu_profile_${{ matrix.vec.env }}_${{ matrix.vec.os }}_${{ matrix.vec.arch }} path: ${{ github.workspace }}\ETL\cpu_profile-*.etl + - name: Attempt to reset lab (trigger schedule-lab-reset) + if: always() + uses: peter-evans/workflow-dispatch@v1 + with: + # Repository containing the reusable reset workflow + repository: microsoft/netperf + workflow: .github/workflows/schedule-lab-reset.yml + ref: main + # Forward current run id so the target workflow can correlate + inputs: | + workflowId: ${{ github.run_id }} + token: ${{ secrets.RESET_WORKFLOW_TOKEN }} + build_echo_test: if: ${{ inputs.test_tool == 'echo' }} name: Build echo server @@ -321,20 +334,14 @@ jobs: path: | ${{ github.workspace }}\echo\RelWithDebInfo - attempt-reset-labecho: - if: ${{ inputs.test_tool == 'echo' }} - name: Attempting to reset lab. Status of this job does not indicate result of lab reset. Look at job details. - needs: [test_echo_server] - if: ${{ always() }} - uses: microsoft/netperf/.github/workflows/schedule-lab-reset.yml@main - with: - workflowId: ${{ github.run_id }} + - name: Attempt to reset lab (trigger schedule-lab-reset) + if: always() + uses: peter-evans/workflow-dispatch@v1 + with: + repository: microsoft/netperf + workflow: .github/workflows/schedule-lab-reset.yml + ref: main + inputs: | + workflowId: ${{ github.run_id }} + token: ${{ secrets.RESET_WORKFLOW_TOKEN }} - attempt-reset-lab-cts: - if: ${{ inputs.test_tool == 'ctsTraffic' }} - name: Attempting to reset lab. Status of this job does not indicate result of lab reset. Look at job details. - needs: [test_cts_traffic] - if: ${{ always() }} - uses: microsoft/netperf/.github/workflows/schedule-lab-reset.yml@main - with: - workflowId: ${{ github.run_id }} \ No newline at end of file From 55e54e1ccde0b051124d92c1fb354f14c1639485 Mon Sep 17 00:00:00 2001 From: Alan Jowett Date: Wed, 3 Dec 2025 10:39:11 -0800 Subject: [PATCH 40/70] Reset hyper-v checkpoint and gather all groups Signed-off-by: Alan Jowett --- .../workflows/network-traffic-performance.yml | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/.github/workflows/network-traffic-performance.yml b/.github/workflows/network-traffic-performance.yml index f93fea6c..1aa4fe5e 100644 --- a/.github/workflows/network-traffic-performance.yml +++ b/.github/workflows/network-traffic-performance.yml @@ -334,14 +334,12 @@ jobs: path: | ${{ github.workspace }}\echo\RelWithDebInfo - - name: Attempt to reset lab (trigger schedule-lab-reset) - if: always() - uses: peter-evans/workflow-dispatch@v1 - with: - repository: microsoft/netperf - workflow: .github/workflows/schedule-lab-reset.yml - ref: main - inputs: | - workflowId: ${{ github.run_id }} - token: ${{ secrets.RESET_WORKFLOW_TOKEN }} + attempt-reset-lab: + name: Attempting to reset lab. Status of this job does not indicate result of lab reset. Look at job details. + needs: [test_cts_traffic, test_echo_server] + if: ${{ always() }} + uses: microsoft/netperf/.github/workflows/schedule-lab-reset.yml@main + with: + workflowId: ${{ github.run_id }} + \ No newline at end of file From c57d3947cb2388f008f8c290857b97d87287937a Mon Sep 17 00:00:00 2001 From: Alan Jowett Date: Wed, 3 Dec 2025 10:49:19 -0800 Subject: [PATCH 41/70] Apply RSS settings Signed-off-by: Alan Jowett --- .github/scripts/run-echo-test.ps1 | 45 +++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/.github/scripts/run-echo-test.ps1 b/.github/scripts/run-echo-test.ps1 index 7a2ba15d..372641ab 100644 --- a/.github/scripts/run-echo-test.ps1 +++ b/.github/scripts/run-echo-test.ps1 @@ -134,6 +134,49 @@ function Write-DetailedError { } } +function Set-RssOnAllCpu { + param([Parameter(Mandatory=$true)][string]$TargetServer) + + # Get all NICs with IPv4 addresses + $NICs = Get-NetIPAddress | Where-Object { $_.AddressFamily -eq 'IPv4' -and $_.IPAddress -notlike '169.*' } + + $ReachableNIC = $null + + Write-Host "Testing connectivity to $TargetServer from each NIC..." + + foreach ($nic in $NICs) { + $result = Test-NetConnection -ComputerName $TargetServer -SourceAddress $nic.IPAddress -WarningAction SilentlyContinue + if ($result.PingSucceeded) { + Write-Host "NIC '$($nic.InterfaceAlias)' with IP $($nic.IPAddress) can reach $TargetServer" + $ReachableNIC = $nic.InterfaceAlias + break + } + } + + if (-not $ReachableNIC) { + Write-Host "No NIC could reach $TargetServer. Exiting." + exit + } + + Write-Host "`nConfiguring RSS on NIC: $ReachableNIC" + + # Enable RSS + Enable-NetAdapterRss -Name $ReachableNIC + + # Get RSS capabilities to determine CPU range + $cap = Get-NetAdapterRssCapabilities -Name $ReachableNIC + $maxCPU = $cap.MaxProcessorNumber + $maxGroup = $cap.MaxProcessorGroup + + # We assume group 0 exists and use its full range + Write-Host "Setting RSS to use all CPUs in group 0 (0..$maxCPU)" + Set-NetAdapterRss -Name $ReachableNIC -BaseProcessorGroup 0 -MaxProcessorNumber $maxCPU + + # Verify settings + Write-Host "`nUpdated RSS settings:" + Get-NetAdapterRss -Name $ReachableNIC +} + # WPR CPU profiling helpers $script:WprProfiles = @{} @@ -607,6 +650,8 @@ try { $cwd = (Get-Location).Path Write-Host "Current working directory: $cwd" + Set-RssOnAllCpu -TargetServer $PeerName + # Create remote session $Session = Create-Session -PeerName $PeerName -RemotePSConfiguration 'PowerShell.7' From 0f657d58b7c6c34f62b2e92c9e9f5d685a95b212 Mon Sep 17 00:00:00 2001 From: Alan Jowett Date: Wed, 3 Dec 2025 11:24:13 -0800 Subject: [PATCH 42/70] Apply RSS settings Signed-off-by: Alan Jowett --- .github/scripts/run-echo-test.ps1 | 219 +++++++++++++++++++++++++----- 1 file changed, 185 insertions(+), 34 deletions(-) diff --git a/.github/scripts/run-echo-test.ps1 b/.github/scripts/run-echo-test.ps1 index 372641ab..2056ec02 100644 --- a/.github/scripts/run-echo-test.ps1 +++ b/.github/scripts/run-echo-test.ps1 @@ -3,7 +3,8 @@ param( [string]$PeerName, [string]$SenderOptions, [string]$ReceiverOptions, - [string]$Duration = "60" # Duration in seconds for the test runs (default 60s) + [string]$Duration = "60", # Duration in seconds for the test runs (default 60s) + [int]$RssCpuCount = 0 # Number of CPUs to enable RSS on the remote server (0 = max available ) Set-StrictMode -Version Latest @@ -134,49 +135,199 @@ function Write-DetailedError { } } -function Set-RssOnAllCpu { - param([Parameter(Mandatory=$true)][string]$TargetServer) +function Set-RssOnCpus { + param( + [Parameter(Mandatory=$true)][string]$TargetServer, + [Parameter(Mandatory=$false)][int]$CpuCount + ) + + Write-Host "Checking connectivity to '$TargetServer'..." + + function Get-RssCapabilities { + param([string]$AdapterName) + + # Prefer the convenient cmdlet if available + if (Get-Command -Name Get-NetAdapterRssCapabilities -ErrorAction SilentlyContinue) { + try { + $res = Get-NetAdapterRssCapabilities -Name $AdapterName -ErrorAction SilentlyContinue + if ($res) { + return [PSCustomObject]@{ + MaxProcessorNumber = [int]$res.MaxProcessorNumber + MaxProcessorGroup = [int]$res.MaxProcessorGroup + } + } + return $null + } catch { + return $null + } + } - # Get all NICs with IPv4 addresses - $NICs = Get-NetIPAddress | Where-Object { $_.AddressFamily -eq 'IPv4' -and $_.IPAddress -notlike '169.*' } + # Fallback: query the CIM class directly + try { + $filter = "Name='$AdapterName'" + $obj = Get-CimInstance -Namespace root/StandardCimv2 -ClassName MSFT_NetAdapterRssSettingData -Filter $filter -ErrorAction SilentlyContinue + if ($obj) { + return [PSCustomObject]@{ + MaxProcessorNumber = [int]$obj.MaxProcessorNumber + MaxProcessorGroup = [int]$obj.MaxProcessorGroup + } + } + } catch { + # ignore + } + return $null + } - $ReachableNIC = $null + # Simple reachability test: use Test-NetConnection once. If it succeeds, pick a usable NIC. + try { + $result = Test-NetConnection -ComputerName $TargetServer -WarningAction SilentlyContinue + } catch { + $result = $null + } - Write-Host "Testing connectivity to $TargetServer from each NIC..." + if (-not ($result -and $result.PingSucceeded)) { + Write-Host "Target '$TargetServer' is not reachable from this host. Returning without changes." + return + } - foreach ($nic in $NICs) { - $result = Test-NetConnection -ComputerName $TargetServer -SourceAddress $nic.IPAddress -WarningAction SilentlyContinue - if ($result.PingSucceeded) { - Write-Host "NIC '$($nic.InterfaceAlias)' with IP $($nic.IPAddress) can reach $TargetServer" - $ReachableNIC = $nic.InterfaceAlias - break - } - } + # Choose the first 'Up' physical adapter as the target NIC + $adapter = Get-NetAdapter -Physical | Where-Object { $_.Status -eq 'Up' } | Select-Object -First 1 + if (-not $adapter) { + Write-Host "No operational physical network adapter found. Returning." + return + } - if (-not $ReachableNIC) { - Write-Host "No NIC could reach $TargetServer. Exiting." - exit - } + $ReachableNIC = $adapter.Name + Write-Host "Configuring RSS on adapter: $ReachableNIC" + + # Check RSS capabilities (cmdlet or CIM fallback) + $capCheck = Get-RssCapabilities -AdapterName $ReachableNIC + if (-not $capCheck) { + Write-Host "Adapter '$ReachableNIC' does not expose RSS capabilities or does not support RSS. Skipping RSS configuration." + return + } - Write-Host "`nConfiguring RSS on NIC: $ReachableNIC" + # Enable RSS if the cmdlet exists; otherwise inform and continue to capability-only flow + if (Get-Command -Name Enable-NetAdapterRss -ErrorAction SilentlyContinue) { + try { + Enable-NetAdapterRss -Name $ReachableNIC -ErrorAction Stop + } catch { + Write-Host "Failed to enable RSS on '$ReachableNIC': $($_.Exception.Message)" + return + } + } else { + Write-Host "Enable-NetAdapterRss cmdlet not present; skipping enable step." + } + + # Get RSS capabilities to determine CPU range + # Re-read capabilities (cmdlet or CIM fallback) + $cap = Get-RssCapabilities -AdapterName $ReachableNIC + if (-not $cap) { + Write-Host "No RSS capability information returned. Returning." + return + } + + $maxCPU = $cap.MaxProcessorNumber + $maxGroup = $cap.MaxProcessorGroup + + if (-not ($maxCPU -is [int]) -or -not ($maxGroup -is [int])) { + Write-Host "Unexpected RSS capability values. Returning." + return + } - # Enable RSS - Enable-NetAdapterRss -Name $ReachableNIC + if ($maxGroup -lt 1) { + Write-Host "No processor groups reported. Assuming group 0." + $useGroup = 0 + } else { + $useGroup = 0 + } + if ($maxCPU -lt 0) { + Write-Host "Invalid MaxProcessorNumber ($maxCPU). Returning." + return + } - # Get RSS capabilities to determine CPU range - $cap = Get-NetAdapterRssCapabilities -Name $ReachableNIC - $maxCPU = $cap.MaxProcessorNumber - $maxGroup = $cap.MaxProcessorGroup + if (Get-Command -Name Set-NetAdapterRss -ErrorAction SilentlyContinue) { + # Determine how many CPUs to set. Use provided CpuCount if valid, otherwise the adapter max. + if ($CpuCount) { + if ($CpuCount -lt 1) { + Write-Host "Provided CpuCount ($CpuCount) is invalid; must be >= 1. Returning." + return + } + if ($CpuCount -gt ($maxCPU + 1)) { + Write-Host "Provided CpuCount ($CpuCount) exceeds adapter MaxProcessorNumber ($maxCPU + 1). Clamping to max." + $useMax = $maxCPU + } else { + # Convert CpuCount (count) to MaxProcessorNumber (index) + $useMax = [int]($CpuCount - 1) + } + } else { + $useMax = $maxCPU + } - # We assume group 0 exists and use its full range - Write-Host "Setting RSS to use all CPUs in group 0 (0..$maxCPU)" - Set-NetAdapterRss -Name $ReachableNIC -BaseProcessorGroup 0 -MaxProcessorNumber $maxCPU + Write-Host "Setting RSS to use CPUs 0..$useMax in group $useGroup" + try { + Set-NetAdapterRss -Name $ReachableNIC -BaseProcessorGroup $useGroup -MaxProcessorNumber $useMax -ErrorAction Stop + } catch { + Write-Host "Failed to set RSS on '$ReachableNIC': $($_.Exception.Message)" + return + } - # Verify settings - Write-Host "`nUpdated RSS settings:" - Get-NetAdapterRss -Name $ReachableNIC + if (Get-Command -Name Get-NetAdapterRss -ErrorAction SilentlyContinue) { + Write-Host "Updated RSS settings for '$ReachableNIC':" + Get-NetAdapterRss -Name $ReachableNIC + } else { + Write-Host "Set successful (no Get-NetAdapterRss cmdlet present to display settings)." + } + } else { + Write-Host "Set-NetAdapterRss cmdlet not present; cannot modify RSS settings. Displaying reported capabilities instead:" + Write-Host "MaxProcessorNumber: $maxCPU, MaxProcessorGroup: $maxGroup" + } } + +# function Set-RssOnAllCpu { +# param([Parameter(Mandatory=$true)][string]$TargetServer) + +# # Get all NICs with IPv4 addresses +# $NICs = Get-NetIPAddress | Where-Object { $_.AddressFamily -eq 'IPv4' -and $_.IPAddress -notlike '169.*' } + +# $ReachableNIC = $null + +# Write-Host "Testing connectivity to $TargetServer from each NIC..." + +# foreach ($nic in $NICs) { +# $result = Test-NetConnection -ComputerName $TargetServer -SourceAddress $nic.IPAddress -WarningAction SilentlyContinue +# if ($result.PingSucceeded) { +# Write-Host "NIC '$($nic.InterfaceAlias)' with IP $($nic.IPAddress) can reach $TargetServer" +# $ReachableNIC = $nic.InterfaceAlias +# break +# } +# } + +# if (-not $ReachableNIC) { +# Write-Host "No NIC could reach $TargetServer. Exiting." +# exit +# } + +# Write-Host "`nConfiguring RSS on NIC: $ReachableNIC" + +# # Enable RSS +# Enable-NetAdapterRss -Name $ReachableNIC + +# # Get RSS capabilities to determine CPU range +# $cap = Get-NetAdapterRssCapabilities -Name $ReachableNIC +# $maxCPU = $cap.MaxProcessorNumber +# $maxGroup = $cap.MaxProcessorGroup + +# # We assume group 0 exists and use its full range +# Write-Host "Setting RSS to use all CPUs in group 0 (0..$maxCPU)" +# Set-NetAdapterRss -Name $ReachableNIC -BaseProcessorGroup 0 -MaxProcessorNumber $maxCPU + +# # Verify settings +# Write-Host "`nUpdated RSS settings:" +# Get-NetAdapterRss -Name $ReachableNIC +# } + # WPR CPU profiling helpers $script:WprProfiles = @{} @@ -650,8 +801,8 @@ try { $cwd = (Get-Location).Path Write-Host "Current working directory: $cwd" - Set-RssOnAllCpu -TargetServer $PeerName - + Set-RssOnCpus -TargetServer $PeerName -CpuCount $RssCpuCount + # Create remote session $Session = Create-Session -PeerName $PeerName -RemotePSConfiguration 'PowerShell.7' From 6805ff4f3a98ae870323e3d3df1650d09f0a57e7 Mon Sep 17 00:00:00 2001 From: Alan Jowett Date: Wed, 3 Dec 2025 11:33:12 -0800 Subject: [PATCH 43/70] Apply RSS settings Signed-off-by: Alan Jowett --- .github/scripts/run-echo-test.ps1 | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.github/scripts/run-echo-test.ps1 b/.github/scripts/run-echo-test.ps1 index 2056ec02..3443d273 100644 --- a/.github/scripts/run-echo-test.ps1 +++ b/.github/scripts/run-echo-test.ps1 @@ -272,6 +272,22 @@ function Set-RssOnCpus { return } + # Disable then re-enable the adapter to ensure settings apply + if ((Get-Command -Name Disable-NetAdapter -ErrorAction SilentlyContinue) -and (Get-Command -Name Enable-NetAdapter -ErrorAction SilentlyContinue)) { + try { + Write-Host "Disabling adapter '$ReachableNIC' to apply settings..." + Disable-NetAdapter -Name $ReachableNIC -Confirm:$false -ErrorAction Stop + Start-Sleep -Seconds 2 + Write-Host "Re-enabling adapter '$ReachableNIC'..." + Enable-NetAdapter -Name $ReachableNIC -Confirm:$false -ErrorAction Stop + Start-Sleep -Seconds 2 + } catch { + Write-Host "Warning: failed to toggle adapter '$ReachableNIC': $($_.Exception.Message)" + } + } else { + Write-Host "Disable/Enable adapter cmdlets not present; skipping adapter toggle." + } + if (Get-Command -Name Get-NetAdapterRss -ErrorAction SilentlyContinue) { Write-Host "Updated RSS settings for '$ReachableNIC':" Get-NetAdapterRss -Name $ReachableNIC From 3fbe49a19902e9b8084ace27d85234973882825b Mon Sep 17 00:00:00 2001 From: Alan Jowett Date: Wed, 3 Dec 2025 11:39:16 -0800 Subject: [PATCH 44/70] Apply RSS settings Signed-off-by: Alan Jowett --- .github/scripts/run-echo-test.ps1 | 58 ++++++++----------------------- 1 file changed, 14 insertions(+), 44 deletions(-) diff --git a/.github/scripts/run-echo-test.ps1 b/.github/scripts/run-echo-test.ps1 index 3443d273..f0911a75 100644 --- a/.github/scripts/run-echo-test.ps1 +++ b/.github/scripts/run-echo-test.ps1 @@ -143,6 +143,20 @@ function Set-RssOnCpus { Write-Host "Checking connectivity to '$TargetServer'..." + # Debug: list adapters and IPv4 addresses to help with troubleshooting + Write-Host "\n[Debug] Network adapters:" + try { + Get-NetAdapter | Format-Table Name, Status, LinkSpeed, InterfaceDescription -AutoSize + } catch { + Write-Host "[Debug] Get-NetAdapter not available: $($_.Exception.Message)" + } + Write-Host "\n[Debug] IPv4 addresses:" + try { + Get-NetIPAddress -AddressFamily IPv4 | Format-Table InterfaceAlias, IPAddress, PrefixLength -AutoSize + } catch { + Write-Host "[Debug] Get-NetIPAddress not available: $($_.Exception.Message)" + } + function Get-RssCapabilities { param([string]$AdapterName) @@ -300,50 +314,6 @@ function Set-RssOnCpus { } } - -# function Set-RssOnAllCpu { -# param([Parameter(Mandatory=$true)][string]$TargetServer) - -# # Get all NICs with IPv4 addresses -# $NICs = Get-NetIPAddress | Where-Object { $_.AddressFamily -eq 'IPv4' -and $_.IPAddress -notlike '169.*' } - -# $ReachableNIC = $null - -# Write-Host "Testing connectivity to $TargetServer from each NIC..." - -# foreach ($nic in $NICs) { -# $result = Test-NetConnection -ComputerName $TargetServer -SourceAddress $nic.IPAddress -WarningAction SilentlyContinue -# if ($result.PingSucceeded) { -# Write-Host "NIC '$($nic.InterfaceAlias)' with IP $($nic.IPAddress) can reach $TargetServer" -# $ReachableNIC = $nic.InterfaceAlias -# break -# } -# } - -# if (-not $ReachableNIC) { -# Write-Host "No NIC could reach $TargetServer. Exiting." -# exit -# } - -# Write-Host "`nConfiguring RSS on NIC: $ReachableNIC" - -# # Enable RSS -# Enable-NetAdapterRss -Name $ReachableNIC - -# # Get RSS capabilities to determine CPU range -# $cap = Get-NetAdapterRssCapabilities -Name $ReachableNIC -# $maxCPU = $cap.MaxProcessorNumber -# $maxGroup = $cap.MaxProcessorGroup - -# # We assume group 0 exists and use its full range -# Write-Host "Setting RSS to use all CPUs in group 0 (0..$maxCPU)" -# Set-NetAdapterRss -Name $ReachableNIC -BaseProcessorGroup 0 -MaxProcessorNumber $maxCPU - -# # Verify settings -# Write-Host "`nUpdated RSS settings:" -# Get-NetAdapterRss -Name $ReachableNIC -# } - # WPR CPU profiling helpers $script:WprProfiles = @{} From 3c165ef43abb9de94636f047251531ea5e11e27a Mon Sep 17 00:00:00 2001 From: Alan Jowett Date: Wed, 3 Dec 2025 12:27:21 -0800 Subject: [PATCH 45/70] Apply RSS settings Signed-off-by: Alan Jowett --- .github/scripts/run-echo-test.ps1 | 57 ++++++++++++++++--------------- 1 file changed, 29 insertions(+), 28 deletions(-) diff --git a/.github/scripts/run-echo-test.ps1 b/.github/scripts/run-echo-test.ps1 index f0911a75..bf0c159a 100644 --- a/.github/scripts/run-echo-test.ps1 +++ b/.github/scripts/run-echo-test.ps1 @@ -137,25 +137,11 @@ function Write-DetailedError { function Set-RssOnCpus { param( - [Parameter(Mandatory=$true)][string]$TargetServer, + [Parameter(Mandatory=$true)][string]$AdapterName, [Parameter(Mandatory=$false)][int]$CpuCount ) - Write-Host "Checking connectivity to '$TargetServer'..." - - # Debug: list adapters and IPv4 addresses to help with troubleshooting - Write-Host "\n[Debug] Network adapters:" - try { - Get-NetAdapter | Format-Table Name, Status, LinkSpeed, InterfaceDescription -AutoSize - } catch { - Write-Host "[Debug] Get-NetAdapter not available: $($_.Exception.Message)" - } - Write-Host "\n[Debug] IPv4 addresses:" - try { - Get-NetIPAddress -AddressFamily IPv4 | Format-Table InterfaceAlias, IPAddress, PrefixLength -AutoSize - } catch { - Write-Host "[Debug] Get-NetIPAddress not available: $($_.Exception.Message)" - } + Write-Host "Applying RSS settings to adapter '$AdapterName'..." function Get-RssCapabilities { param([string]$AdapterName) @@ -192,22 +178,18 @@ function Set-RssOnCpus { return $null } - # Simple reachability test: use Test-NetConnection once. If it succeeds, pick a usable NIC. + # Use the adapter name provided by the caller and validate it exists and is operational try { - $result = Test-NetConnection -ComputerName $TargetServer -WarningAction SilentlyContinue + $adapter = Get-NetAdapter -Name $AdapterName -ErrorAction SilentlyContinue } catch { - $result = $null + $adapter = $null } - - if (-not ($result -and $result.PingSucceeded)) { - Write-Host "Target '$TargetServer' is not reachable from this host. Returning without changes." + if (-not $adapter) { + Write-Host "Adapter '$AdapterName' not found. Returning." return } - - # Choose the first 'Up' physical adapter as the target NIC - $adapter = Get-NetAdapter -Physical | Where-Object { $_.Status -eq 'Up' } | Select-Object -First 1 - if (-not $adapter) { - Write-Host "No operational physical network adapter found. Returning." + if ($adapter.Status -ne 'Up') { + Write-Host "Adapter '$AdapterName' is not operational (Status: $($adapter.Status)). Returning." return } @@ -787,7 +769,26 @@ try { $cwd = (Get-Location).Path Write-Host "Current working directory: $cwd" - Set-RssOnCpus -TargetServer $PeerName -CpuCount $RssCpuCount + # Debug: list adapters and IPv4 addresses to help with troubleshooting + Write-Host "\n[Debug] Network adapters:" + try { + Get-NetAdapter | Format-Table Name, Status, LinkSpeed, InterfaceDescription -AutoSize + } catch { + Write-Host "[Debug] Get-NetAdapter not available: $($_.Exception.Message)" + } + Write-Host "\n[Debug] IPv4 addresses:" + try { + Get-NetIPAddress -AddressFamily IPv4 | Format-Table InterfaceAlias, IPAddress, PrefixLength -AutoSize + } catch { + Write-Host "[Debug] Get-NetIPAddress not available: $($_.Exception.Message)" + } + + + # Get NIC adapter with at least 10 Gbps link speed + $Nic = (Get-NetAdapter | where-object -Property LinkSpeed -GE 10).Name + + Write-Output "Configuring RSS on adapter '$Nic' to use $RssCpuCount CPUs..." + Set-RssOnCpus -AdapterName $Nic -CpuCount $RssCpuCount # Create remote session $Session = Create-Session -PeerName $PeerName -RemotePSConfiguration 'PowerShell.7' From f483217636199620ced336ea80e199ef96691946 Mon Sep 17 00:00:00 2001 From: Alan Jowett Date: Wed, 3 Dec 2025 12:42:04 -0800 Subject: [PATCH 46/70] Apply RSS settings Signed-off-by: Alan Jowett --- .github/scripts/run-echo-test.ps1 | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/scripts/run-echo-test.ps1 b/.github/scripts/run-echo-test.ps1 index bf0c159a..9eefc6f0 100644 --- a/.github/scripts/run-echo-test.ps1 +++ b/.github/scripts/run-echo-test.ps1 @@ -783,12 +783,14 @@ try { Write-Host "[Debug] Get-NetIPAddress not available: $($_.Exception.Message)" } - - # Get NIC adapter with at least 10 Gbps link speed + # Determine the NIC to configure RSS on (10Gbps or higher) $Nic = (Get-NetAdapter | where-object -Property LinkSpeed -GE 10).Name - Write-Output "Configuring RSS on adapter '$Nic' to use $RssCpuCount CPUs..." - Set-RssOnCpus -AdapterName $Nic -CpuCount $RssCpuCount + # Set this on each NIC that meets the criteria + foreach ($n in $Nic) { + Write-Output "Configuring RSS on adapter '$Nic' to use $RssCpuCount CPUs..." + Set-RssOnCpus -AdapterName $n -CpuCount $RssCpuCount + } # Create remote session $Session = Create-Session -PeerName $PeerName -RemotePSConfiguration 'PowerShell.7' From 5545e2e83890884539d501ab2d4c1a02534d33d2 Mon Sep 17 00:00:00 2001 From: Alan Jowett Date: Wed, 3 Dec 2025 12:57:30 -0800 Subject: [PATCH 47/70] Apply RSS settings Signed-off-by: Alan Jowett --- .github/scripts/run-echo-test.ps1 | 8 +++++++- .github/workflows/network-traffic-performance.yml | 2 ++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/scripts/run-echo-test.ps1 b/.github/scripts/run-echo-test.ps1 index 9eefc6f0..316698d4 100644 --- a/.github/scripts/run-echo-test.ps1 +++ b/.github/scripts/run-echo-test.ps1 @@ -788,10 +788,16 @@ try { # Set this on each NIC that meets the criteria foreach ($n in $Nic) { - Write-Output "Configuring RSS on adapter '$Nic' to use $RssCpuCount CPUs..." + Write-Host "Configuring RSS on adapter '$Nic' to use $RssCpuCount CPUs..." Set-RssOnCpus -AdapterName $n -CpuCount $RssCpuCount } + # Debug RSS state for all NICs + Write-Host "\n[Debug] Current RSS settings for all adapters:" + Get-NetAdapterRss + + Write-Host "\nStarting echo tests to peer '$PeerName' with duration $Duration seconds..." + # Create remote session $Session = Create-Session -PeerName $PeerName -RemotePSConfiguration 'PowerShell.7' diff --git a/.github/workflows/network-traffic-performance.yml b/.github/workflows/network-traffic-performance.yml index 1aa4fe5e..22a71ef2 100644 --- a/.github/workflows/network-traffic-performance.yml +++ b/.github/workflows/network-traffic-performance.yml @@ -208,11 +208,13 @@ jobs: - env: lab os: "2022" arch: x64 + id: ["vm-01-02", "vm-03-04", "vm-05-06", "vm-07-08", "vm-09-10"] runs-on: - self-hosted - ${{ matrix.vec.env }} - os-windows-${{ matrix.vec.os }} - ${{ matrix.vec.arch }} + - ${ matrix.id } steps: - name: Checkout repository From 3284d25c5783869f1f05485924c0a37e5ab14d40 Mon Sep 17 00:00:00 2001 From: Alan Jowett Date: Wed, 3 Dec 2025 13:00:24 -0800 Subject: [PATCH 48/70] Apply RSS settings Signed-off-by: Alan Jowett --- .github/workflows/network-traffic-performance.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/network-traffic-performance.yml b/.github/workflows/network-traffic-performance.yml index 22a71ef2..1aa4fe5e 100644 --- a/.github/workflows/network-traffic-performance.yml +++ b/.github/workflows/network-traffic-performance.yml @@ -208,13 +208,11 @@ jobs: - env: lab os: "2022" arch: x64 - id: ["vm-01-02", "vm-03-04", "vm-05-06", "vm-07-08", "vm-09-10"] runs-on: - self-hosted - ${{ matrix.vec.env }} - os-windows-${{ matrix.vec.os }} - ${{ matrix.vec.arch }} - - ${ matrix.id } steps: - name: Checkout repository From bd080f5325079cd091ffb86a48193b9e38314994 Mon Sep 17 00:00:00 2001 From: Alan Jowett Date: Wed, 3 Dec 2025 13:04:12 -0800 Subject: [PATCH 49/70] Apply RSS settings Signed-off-by: Alan Jowett --- .github/scripts/run-echo-test.ps1 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/scripts/run-echo-test.ps1 b/.github/scripts/run-echo-test.ps1 index 316698d4..f44aae25 100644 --- a/.github/scripts/run-echo-test.ps1 +++ b/.github/scripts/run-echo-test.ps1 @@ -794,7 +794,8 @@ try { # Debug RSS state for all NICs Write-Host "\n[Debug] Current RSS settings for all adapters:" - Get-NetAdapterRss + $RssState = Get-NetAdapterRss + Write-Host $RssState Write-Host "\nStarting echo tests to peer '$PeerName' with duration $Duration seconds..." From 10672510ace5832df7f6840fc712d657409da8ea Mon Sep 17 00:00:00 2001 From: Alan Jowett Date: Wed, 3 Dec 2025 13:10:31 -0800 Subject: [PATCH 50/70] Apply RSS settings Signed-off-by: Alan Jowett --- .github/scripts/run-echo-test.ps1 | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/scripts/run-echo-test.ps1 b/.github/scripts/run-echo-test.ps1 index f44aae25..487b8147 100644 --- a/.github/scripts/run-echo-test.ps1 +++ b/.github/scripts/run-echo-test.ps1 @@ -135,7 +135,7 @@ function Write-DetailedError { } } -function Set-RssOnCpus { +function Set-RssSettings { param( [Parameter(Mandatory=$true)][string]$AdapterName, [Parameter(Mandatory=$false)][int]$CpuCount @@ -262,7 +262,7 @@ function Set-RssOnCpus { Write-Host "Setting RSS to use CPUs 0..$useMax in group $useGroup" try { - Set-NetAdapterRss -Name $ReachableNIC -BaseProcessorGroup $useGroup -MaxProcessorNumber $useMax -ErrorAction Stop + Set-NetAdapterRss -Name $ReachableNIC -BaseProcessorGroup $useGroup -MaxProcessorNumber $useMax -MaxProcessors CpuCount -Profile NUMAStatic -ErrorAction Stop } catch { Write-Host "Failed to set RSS on '$ReachableNIC': $($_.Exception.Message)" return @@ -789,7 +789,7 @@ try { # Set this on each NIC that meets the criteria foreach ($n in $Nic) { Write-Host "Configuring RSS on adapter '$Nic' to use $RssCpuCount CPUs..." - Set-RssOnCpus -AdapterName $n -CpuCount $RssCpuCount + Set-RssSettings -AdapterName $n -CpuCount $RssCpuCount } # Debug RSS state for all NICs From 5d6b69b05b93966ab9cb4f952434180279ca58e0 Mon Sep 17 00:00:00 2001 From: Alan Jowett Date: Wed, 3 Dec 2025 13:12:39 -0800 Subject: [PATCH 51/70] Apply RSS settings Signed-off-by: Alan Jowett --- .github/workflows/network-traffic-performance.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/network-traffic-performance.yml b/.github/workflows/network-traffic-performance.yml index 1aa4fe5e..793e4229 100644 --- a/.github/workflows/network-traffic-performance.yml +++ b/.github/workflows/network-traffic-performance.yml @@ -213,6 +213,7 @@ jobs: - ${{ matrix.vec.env }} - os-windows-${{ matrix.vec.os }} - ${{ matrix.vec.arch }} + - ebpf steps: - name: Checkout repository From 4b67a699f3624186e078ec19ca0c603b7f113956 Mon Sep 17 00:00:00 2001 From: Alan Jowett Date: Wed, 3 Dec 2025 13:14:47 -0800 Subject: [PATCH 52/70] Apply RSS settings Signed-off-by: Alan Jowett --- .github/scripts/run-echo-test.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/run-echo-test.ps1 b/.github/scripts/run-echo-test.ps1 index 487b8147..e9718f78 100644 --- a/.github/scripts/run-echo-test.ps1 +++ b/.github/scripts/run-echo-test.ps1 @@ -262,7 +262,7 @@ function Set-RssSettings { Write-Host "Setting RSS to use CPUs 0..$useMax in group $useGroup" try { - Set-NetAdapterRss -Name $ReachableNIC -BaseProcessorGroup $useGroup -MaxProcessorNumber $useMax -MaxProcessors CpuCount -Profile NUMAStatic -ErrorAction Stop + Set-NetAdapterRss -Name $ReachableNIC -BaseProcessorGroup $useGroup -MaxProcessorNumber $useMax -MaxProcessors $CpuCount -Profile NUMAStatic -ErrorAction Stop } catch { Write-Host "Failed to set RSS on '$ReachableNIC': $($_.Exception.Message)" return From 5246458acb00d5b3c87a8a21324952ad602e2117 Mon Sep 17 00:00:00 2001 From: Alan Jowett Date: Wed, 3 Dec 2025 13:21:46 -0800 Subject: [PATCH 53/70] Apply RSS settings Signed-off-by: Alan Jowett --- .github/scripts/run-echo-test.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/run-echo-test.ps1 b/.github/scripts/run-echo-test.ps1 index e9718f78..e41c5a94 100644 --- a/.github/scripts/run-echo-test.ps1 +++ b/.github/scripts/run-echo-test.ps1 @@ -262,7 +262,7 @@ function Set-RssSettings { Write-Host "Setting RSS to use CPUs 0..$useMax in group $useGroup" try { - Set-NetAdapterRss -Name $ReachableNIC -BaseProcessorGroup $useGroup -MaxProcessorNumber $useMax -MaxProcessors $CpuCount -Profile NUMAStatic -ErrorAction Stop + Set-NetAdapterRss -Name $ReachableNIC -BaseProcessorGroup $useGroup -MaxProcessorNumber $useMax -MaxProcessors ($useMax+1) -Profile NUMAStatic -ErrorAction Stop } catch { Write-Host "Failed to set RSS on '$ReachableNIC': $($_.Exception.Message)" return From 38eb7e6ef7e3bf89ee777b6a5b578652f6b07782 Mon Sep 17 00:00:00 2001 From: Alan Jowett Date: Wed, 3 Dec 2025 13:35:31 -0800 Subject: [PATCH 54/70] Apply RSS settings Signed-off-by: Alan Jowett --- .github/scripts/run-echo-test.ps1 | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/.github/scripts/run-echo-test.ps1 b/.github/scripts/run-echo-test.ps1 index e41c5a94..746131ed 100644 --- a/.github/scripts/run-echo-test.ps1 +++ b/.github/scripts/run-echo-test.ps1 @@ -260,9 +260,11 @@ function Set-RssSettings { $useMax = $maxCPU } + $CpuCount = $useMax + 1 + Write-Host "Setting RSS to use CPUs 0..$useMax in group $useGroup" try { - Set-NetAdapterRss -Name $ReachableNIC -BaseProcessorGroup $useGroup -MaxProcessorNumber $useMax -MaxProcessors ($useMax+1) -Profile NUMAStatic -ErrorAction Stop + Set-NetAdapterRss -Name $ReachableNIC -BaseProcessorGroup $useGroup -MaxProcessorNumber $useMax -MaxProcessors $CpuCount -Profile NUMAStatic -ErrorAction Stop } catch { Write-Host "Failed to set RSS on '$ReachableNIC': $($_.Exception.Message)" return @@ -795,7 +797,7 @@ try { # Debug RSS state for all NICs Write-Host "\n[Debug] Current RSS settings for all adapters:" $RssState = Get-NetAdapterRss - Write-Host $RssState + Write-Host $RssState | Format-Table -AutoSize Write-Host "\nStarting echo tests to peer '$PeerName' with duration $Duration seconds..." From 6be759106c6c4cdf820e0ce67f01baadc76817e4 Mon Sep 17 00:00:00 2001 From: Alan Jowett Date: Wed, 3 Dec 2025 13:45:07 -0800 Subject: [PATCH 55/70] Revert RSS settings due to VF limitations Signed-off-by: Alan Jowett --- .github/scripts/run-echo-test.ps1 | 373 +++++++++++++++--------------- 1 file changed, 188 insertions(+), 185 deletions(-) diff --git a/.github/scripts/run-echo-test.ps1 b/.github/scripts/run-echo-test.ps1 index 746131ed..2709f04c 100644 --- a/.github/scripts/run-echo-test.ps1 +++ b/.github/scripts/run-echo-test.ps1 @@ -135,168 +135,169 @@ function Write-DetailedError { } } -function Set-RssSettings { - param( - [Parameter(Mandatory=$true)][string]$AdapterName, - [Parameter(Mandatory=$false)][int]$CpuCount - ) - - Write-Host "Applying RSS settings to adapter '$AdapterName'..." - - function Get-RssCapabilities { - param([string]$AdapterName) - - # Prefer the convenient cmdlet if available - if (Get-Command -Name Get-NetAdapterRssCapabilities -ErrorAction SilentlyContinue) { - try { - $res = Get-NetAdapterRssCapabilities -Name $AdapterName -ErrorAction SilentlyContinue - if ($res) { - return [PSCustomObject]@{ - MaxProcessorNumber = [int]$res.MaxProcessorNumber - MaxProcessorGroup = [int]$res.MaxProcessorGroup - } - } - return $null - } catch { - return $null - } - } - - # Fallback: query the CIM class directly - try { - $filter = "Name='$AdapterName'" - $obj = Get-CimInstance -Namespace root/StandardCimv2 -ClassName MSFT_NetAdapterRssSettingData -Filter $filter -ErrorAction SilentlyContinue - if ($obj) { - return [PSCustomObject]@{ - MaxProcessorNumber = [int]$obj.MaxProcessorNumber - MaxProcessorGroup = [int]$obj.MaxProcessorGroup - } - } - } catch { - # ignore - } - return $null - } - - # Use the adapter name provided by the caller and validate it exists and is operational - try { - $adapter = Get-NetAdapter -Name $AdapterName -ErrorAction SilentlyContinue - } catch { - $adapter = $null - } - if (-not $adapter) { - Write-Host "Adapter '$AdapterName' not found. Returning." - return - } - if ($adapter.Status -ne 'Up') { - Write-Host "Adapter '$AdapterName' is not operational (Status: $($adapter.Status)). Returning." - return - } - - $ReachableNIC = $adapter.Name - Write-Host "Configuring RSS on adapter: $ReachableNIC" - - # Check RSS capabilities (cmdlet or CIM fallback) - $capCheck = Get-RssCapabilities -AdapterName $ReachableNIC - if (-not $capCheck) { - Write-Host "Adapter '$ReachableNIC' does not expose RSS capabilities or does not support RSS. Skipping RSS configuration." - return - } - - # Enable RSS if the cmdlet exists; otherwise inform and continue to capability-only flow - if (Get-Command -Name Enable-NetAdapterRss -ErrorAction SilentlyContinue) { - try { - Enable-NetAdapterRss -Name $ReachableNIC -ErrorAction Stop - } catch { - Write-Host "Failed to enable RSS on '$ReachableNIC': $($_.Exception.Message)" - return - } - } else { - Write-Host "Enable-NetAdapterRss cmdlet not present; skipping enable step." - } - - # Get RSS capabilities to determine CPU range - # Re-read capabilities (cmdlet or CIM fallback) - $cap = Get-RssCapabilities -AdapterName $ReachableNIC - if (-not $cap) { - Write-Host "No RSS capability information returned. Returning." - return - } - - $maxCPU = $cap.MaxProcessorNumber - $maxGroup = $cap.MaxProcessorGroup - - if (-not ($maxCPU -is [int]) -or -not ($maxGroup -is [int])) { - Write-Host "Unexpected RSS capability values. Returning." - return - } - - if ($maxGroup -lt 1) { - Write-Host "No processor groups reported. Assuming group 0." - $useGroup = 0 - } else { - $useGroup = 0 - } - if ($maxCPU -lt 0) { - Write-Host "Invalid MaxProcessorNumber ($maxCPU). Returning." - return - } - - if (Get-Command -Name Set-NetAdapterRss -ErrorAction SilentlyContinue) { - # Determine how many CPUs to set. Use provided CpuCount if valid, otherwise the adapter max. - if ($CpuCount) { - if ($CpuCount -lt 1) { - Write-Host "Provided CpuCount ($CpuCount) is invalid; must be >= 1. Returning." - return - } - if ($CpuCount -gt ($maxCPU + 1)) { - Write-Host "Provided CpuCount ($CpuCount) exceeds adapter MaxProcessorNumber ($maxCPU + 1). Clamping to max." - $useMax = $maxCPU - } else { - # Convert CpuCount (count) to MaxProcessorNumber (index) - $useMax = [int]($CpuCount - 1) - } - } else { - $useMax = $maxCPU - } - - $CpuCount = $useMax + 1 - - Write-Host "Setting RSS to use CPUs 0..$useMax in group $useGroup" - try { - Set-NetAdapterRss -Name $ReachableNIC -BaseProcessorGroup $useGroup -MaxProcessorNumber $useMax -MaxProcessors $CpuCount -Profile NUMAStatic -ErrorAction Stop - } catch { - Write-Host "Failed to set RSS on '$ReachableNIC': $($_.Exception.Message)" - return - } - - # Disable then re-enable the adapter to ensure settings apply - if ((Get-Command -Name Disable-NetAdapter -ErrorAction SilentlyContinue) -and (Get-Command -Name Enable-NetAdapter -ErrorAction SilentlyContinue)) { - try { - Write-Host "Disabling adapter '$ReachableNIC' to apply settings..." - Disable-NetAdapter -Name $ReachableNIC -Confirm:$false -ErrorAction Stop - Start-Sleep -Seconds 2 - Write-Host "Re-enabling adapter '$ReachableNIC'..." - Enable-NetAdapter -Name $ReachableNIC -Confirm:$false -ErrorAction Stop - Start-Sleep -Seconds 2 - } catch { - Write-Host "Warning: failed to toggle adapter '$ReachableNIC': $($_.Exception.Message)" - } - } else { - Write-Host "Disable/Enable adapter cmdlets not present; skipping adapter toggle." - } - - if (Get-Command -Name Get-NetAdapterRss -ErrorAction SilentlyContinue) { - Write-Host "Updated RSS settings for '$ReachableNIC':" - Get-NetAdapterRss -Name $ReachableNIC - } else { - Write-Host "Set successful (no Get-NetAdapterRss cmdlet present to display settings)." - } - } else { - Write-Host "Set-NetAdapterRss cmdlet not present; cannot modify RSS settings. Displaying reported capabilities instead:" - Write-Host "MaxProcessorNumber: $maxCPU, MaxProcessorGroup: $maxGroup" - } -} +# Mellanox CX5 limits RSS to 8 CPUs even if more are available, so setting this is pointless. +# function Set-RssSettings { +# param( +# [Parameter(Mandatory=$true)][string]$AdapterName, +# [Parameter(Mandatory=$false)][int]$CpuCount +# ) + +# Write-Host "Applying RSS settings to adapter '$AdapterName'..." + +# function Get-RssCapabilities { +# param([string]$AdapterName) + +# # Prefer the convenient cmdlet if available +# if (Get-Command -Name Get-NetAdapterRssCapabilities -ErrorAction SilentlyContinue) { +# try { +# $res = Get-NetAdapterRssCapabilities -Name $AdapterName -ErrorAction SilentlyContinue +# if ($res) { +# return [PSCustomObject]@{ +# MaxProcessorNumber = [int]$res.MaxProcessorNumber +# MaxProcessorGroup = [int]$res.MaxProcessorGroup +# } +# } +# return $null +# } catch { +# return $null +# } +# } + +# # Fallback: query the CIM class directly +# try { +# $filter = "Name='$AdapterName'" +# $obj = Get-CimInstance -Namespace root/StandardCimv2 -ClassName MSFT_NetAdapterRssSettingData -Filter $filter -ErrorAction SilentlyContinue +# if ($obj) { +# return [PSCustomObject]@{ +# MaxProcessorNumber = [int]$obj.MaxProcessorNumber +# MaxProcessorGroup = [int]$obj.MaxProcessorGroup +# } +# } +# } catch { +# # ignore +# } +# return $null +# } + +# # Use the adapter name provided by the caller and validate it exists and is operational +# try { +# $adapter = Get-NetAdapter -Name $AdapterName -ErrorAction SilentlyContinue +# } catch { +# $adapter = $null +# } +# if (-not $adapter) { +# Write-Host "Adapter '$AdapterName' not found. Returning." +# return +# } +# if ($adapter.Status -ne 'Up') { +# Write-Host "Adapter '$AdapterName' is not operational (Status: $($adapter.Status)). Returning." +# return +# } + +# $ReachableNIC = $adapter.Name +# Write-Host "Configuring RSS on adapter: $ReachableNIC" + +# # Check RSS capabilities (cmdlet or CIM fallback) +# $capCheck = Get-RssCapabilities -AdapterName $ReachableNIC +# if (-not $capCheck) { +# Write-Host "Adapter '$ReachableNIC' does not expose RSS capabilities or does not support RSS. Skipping RSS configuration." +# return +# } + +# # Enable RSS if the cmdlet exists; otherwise inform and continue to capability-only flow +# if (Get-Command -Name Enable-NetAdapterRss -ErrorAction SilentlyContinue) { +# try { +# Enable-NetAdapterRss -Name $ReachableNIC -ErrorAction Stop +# } catch { +# Write-Host "Failed to enable RSS on '$ReachableNIC': $($_.Exception.Message)" +# return +# } +# } else { +# Write-Host "Enable-NetAdapterRss cmdlet not present; skipping enable step." +# } + +# # Get RSS capabilities to determine CPU range +# # Re-read capabilities (cmdlet or CIM fallback) +# $cap = Get-RssCapabilities -AdapterName $ReachableNIC +# if (-not $cap) { +# Write-Host "No RSS capability information returned. Returning." +# return +# } + +# $maxCPU = $cap.MaxProcessorNumber +# $maxGroup = $cap.MaxProcessorGroup + +# if (-not ($maxCPU -is [int]) -or -not ($maxGroup -is [int])) { +# Write-Host "Unexpected RSS capability values. Returning." +# return +# } + +# if ($maxGroup -lt 1) { +# Write-Host "No processor groups reported. Assuming group 0." +# $useGroup = 0 +# } else { +# $useGroup = 0 +# } +# if ($maxCPU -lt 0) { +# Write-Host "Invalid MaxProcessorNumber ($maxCPU). Returning." +# return +# } + +# if (Get-Command -Name Set-NetAdapterRss -ErrorAction SilentlyContinue) { +# # Determine how many CPUs to set. Use provided CpuCount if valid, otherwise the adapter max. +# if ($CpuCount) { +# if ($CpuCount -lt 1) { +# Write-Host "Provided CpuCount ($CpuCount) is invalid; must be >= 1. Returning." +# return +# } +# if ($CpuCount -gt ($maxCPU + 1)) { +# Write-Host "Provided CpuCount ($CpuCount) exceeds adapter MaxProcessorNumber ($maxCPU + 1). Clamping to max." +# $useMax = $maxCPU +# } else { +# # Convert CpuCount (count) to MaxProcessorNumber (index) +# $useMax = [int]($CpuCount - 1) +# } +# } else { +# $useMax = $maxCPU +# } + +# $CpuCount = $useMax + 1 + +# Write-Host "Setting RSS to use CPUs 0..$useMax in group $useGroup" +# try { +# Set-NetAdapterRss -Name $ReachableNIC -BaseProcessorGroup $useGroup -MaxProcessorNumber $useMax -MaxProcessors $CpuCount -Profile NUMAStatic -ErrorAction Stop +# } catch { +# Write-Host "Failed to set RSS on '$ReachableNIC': $($_.Exception.Message)" +# return +# } + +# # Disable then re-enable the adapter to ensure settings apply +# if ((Get-Command -Name Disable-NetAdapter -ErrorAction SilentlyContinue) -and (Get-Command -Name Enable-NetAdapter -ErrorAction SilentlyContinue)) { +# try { +# Write-Host "Disabling adapter '$ReachableNIC' to apply settings..." +# Disable-NetAdapter -Name $ReachableNIC -Confirm:$false -ErrorAction Stop +# Start-Sleep -Seconds 2 +# Write-Host "Re-enabling adapter '$ReachableNIC'..." +# Enable-NetAdapter -Name $ReachableNIC -Confirm:$false -ErrorAction Stop +# Start-Sleep -Seconds 2 +# } catch { +# Write-Host "Warning: failed to toggle adapter '$ReachableNIC': $($_.Exception.Message)" +# } +# } else { +# Write-Host "Disable/Enable adapter cmdlets not present; skipping adapter toggle." +# } + +# if (Get-Command -Name Get-NetAdapterRss -ErrorAction SilentlyContinue) { +# Write-Host "Updated RSS settings for '$ReachableNIC':" +# Get-NetAdapterRss -Name $ReachableNIC +# } else { +# Write-Host "Set successful (no Get-NetAdapterRss cmdlet present to display settings)." +# } +# } else { +# Write-Host "Set-NetAdapterRss cmdlet not present; cannot modify RSS settings. Displaying reported capabilities instead:" +# Write-Host "MaxProcessorNumber: $maxCPU, MaxProcessorGroup: $maxGroup" +# } +# } # WPR CPU profiling helpers $script:WprProfiles = @{} @@ -771,33 +772,35 @@ try { $cwd = (Get-Location).Path Write-Host "Current working directory: $cwd" - # Debug: list adapters and IPv4 addresses to help with troubleshooting - Write-Host "\n[Debug] Network adapters:" - try { - Get-NetAdapter | Format-Table Name, Status, LinkSpeed, InterfaceDescription -AutoSize - } catch { - Write-Host "[Debug] Get-NetAdapter not available: $($_.Exception.Message)" - } - Write-Host "\n[Debug] IPv4 addresses:" - try { - Get-NetIPAddress -AddressFamily IPv4 | Format-Table InterfaceAlias, IPAddress, PrefixLength -AutoSize - } catch { - Write-Host "[Debug] Get-NetIPAddress not available: $($_.Exception.Message)" - } - - # Determine the NIC to configure RSS on (10Gbps or higher) - $Nic = (Get-NetAdapter | where-object -Property LinkSpeed -GE 10).Name - - # Set this on each NIC that meets the criteria - foreach ($n in $Nic) { - Write-Host "Configuring RSS on adapter '$Nic' to use $RssCpuCount CPUs..." - Set-RssSettings -AdapterName $n -CpuCount $RssCpuCount - } + # # Debug: list adapters and IPv4 addresses to help with troubleshooting + # Write-Host "\n[Debug] Network adapters:" + # try { + # Get-NetAdapter | Format-Table Name, Status, LinkSpeed, InterfaceDescription -AutoSize + # } catch { + # Write-Host "[Debug] Get-NetAdapter not available: $($_.Exception.Message)" + # } + # Write-Host "\n[Debug] IPv4 addresses:" + # try { + # Get-NetIPAddress -AddressFamily IPv4 | Format-Table InterfaceAlias, IPAddress, PrefixLength -AutoSize + # } catch { + # Write-Host "[Debug] Get-NetIPAddress not available: $($_.Exception.Message)" + # } + + # # Determine the NIC to configure RSS on (10Gbps or higher) + # $Nic = (Get-NetAdapter | where-object -Property LinkSpeed -GE 10).Name + + # # Set this on each NIC that meets the criteria + # foreach ($n in $Nic) { + # Write-Host "Configuring RSS on adapter '$Nic' to use $RssCpuCount CPUs..." + # Set-RssSettings -AdapterName $n -CpuCount $RssCpuCount + # } # Debug RSS state for all NICs Write-Host "\n[Debug] Current RSS settings for all adapters:" $RssState = Get-NetAdapterRss - Write-Host $RssState | Format-Table -AutoSize + # Format the output nicely + $output = $RssState | Format-Table Name, Enabled, BaseProcessorGroup, MaxProcessorNumber, MaxProcessors, Profile -AutoSize + Write-Host $output Write-Host "\nStarting echo tests to peer '$PeerName' with duration $Duration seconds..." From fec866d2c834398b445a02cdbbff711faaba06d6 Mon Sep 17 00:00:00 2001 From: Alan Jowett Date: Wed, 3 Dec 2025 14:19:56 -0800 Subject: [PATCH 56/70] Add option to print stats to file Signed-off-by: Alan Jowett --- .github/scripts/run-echo-test.ps1 | 2 ++ .github/workflows/network-traffic-performance.yml | 1 + 2 files changed, 3 insertions(+) diff --git a/.github/scripts/run-echo-test.ps1 b/.github/scripts/run-echo-test.ps1 index 2709f04c..9177eb13 100644 --- a/.github/scripts/run-echo-test.ps1 +++ b/.github/scripts/run-echo-test.ps1 @@ -624,6 +624,8 @@ function Run-SendTest { $clientArgs = Convert-ArgStringToArray $SenderOptions $clientArgs = Normalize-Args -Tokens $clientArgs + $clientArgs += @('--stats-file', 'echo_client_stats.json') + Write-Host "[Local] Running: .\echo_client.exe" Write-Host "[Local] Arguments:" foreach ($a in $clientArgs) { Write-Host " $a" } diff --git a/.github/workflows/network-traffic-performance.yml b/.github/workflows/network-traffic-performance.yml index 793e4229..0ec163c1 100644 --- a/.github/workflows/network-traffic-performance.yml +++ b/.github/workflows/network-traffic-performance.yml @@ -312,6 +312,7 @@ jobs: path: | ${{ github.workspace }}\echo\*.csv ${{ github.workspace }}\echo\*.log + ${{ github.workspace }}\echo\*.json - name: Upload TCPIP ETL if: always() From 9714e096cfb3d14759f476850e1209fc6872b21d Mon Sep 17 00:00:00 2001 From: Alan Jowett Date: Wed, 3 Dec 2025 15:14:27 -0800 Subject: [PATCH 57/70] Add additional counters Signed-off-by: Alan Jowett --- .github/scripts/run-echo-test.ps1 | 87 +++++++++++++++++++++++++++++-- 1 file changed, 84 insertions(+), 3 deletions(-) diff --git a/.github/scripts/run-echo-test.ps1 b/.github/scripts/run-echo-test.ps1 index 9177eb13..590e0202 100644 --- a/.github/scripts/run-echo-test.ps1 +++ b/.github/scripts/run-echo-test.ps1 @@ -718,6 +718,56 @@ function CaptureIndividualCpuUsagePerformanceMonitorAsJob { } +function CapturePerformanceMonitorAsJob { + param( + [Parameter(Mandatory=$true)][string]$DurationSeconds, + [Parameter(Mandatory=$false)][string[]]$Counters = @('\Processor Information(*)\% Processor Time') + ) + + # Ensure numeric duration + $intDuration = [int]::Parse($DurationSeconds) + + $perfJob = Start-Job -ScriptBlock { + param($duration, $counters) + + $d = [int]$duration + if (-not $counters -or $counters.Count -eq 0) { + $counters = @('\Processor Information(*)\% Processor Time') + } + + try { + # Collect all requested counters in one Get-Counter call if possible + $samples = Get-Counter -Counter $counters -SampleInterval 1 -MaxSamples $d -ErrorAction Stop + + # Group by counter path and instance name, compute per-instance averages + # CounterSamples contain Path, InstanceName, CounterName, CookedValue + $groupedByCounter = $samples.CounterSamples | Group-Object -Property Path + + $results = @() + foreach ($counterGroup in $groupedByCounter) { + $path = $counterGroup.Name + $innerGroups = $counterGroup.Group | Group-Object -Property InstanceName + foreach ($inst in $innerGroups) { + $instName = $inst.Name + if ([string]::IsNullOrEmpty($instName) -or $instName -eq '_Total') { continue } + $vals = $inst.Group | ForEach-Object { [double]$_.CookedValue } + $avg = ($vals | Measure-Object -Average).Average + $results += [PSCustomObject]@{ Counter = $path; Instance = $instName; Average = $avg } + } + } + + # Emit structured results: an array of PSObjects with Counter, Instance, Average + $results + } + catch { + Write-Output (@([PSCustomObject]@{ Counter = 'error'; Instance = ''; Average = 0 })) + } + } -ArgumentList $intDuration, $Counters + + return $perfJob +} + + function Restore-FirewallAndCleanup { param([object]$Session) @@ -765,6 +815,27 @@ function Restore-FirewallAndCleanup { } } +$PerformanceCounters = +@( + '\UDPv4\Datagrams Received Errors', + '\UDPv6\Datagrams Received Errors', + + '\Network Interface(*)\Packets Received Errors', + '\Network Interface(*)\Packets Received Discarded', + '\Network Interface(*)\Packets Outbound Discarded', + + '\IPv4\Datagrams Received Discarded', + '\IPv4\Datagrams Received Header Errors', + '\IPv4\Datagrams Received Address Errors', + + '\IPv6\Datagrams Received Discarded', + '\IPv6\Datagrams Received Header Errors', + '\IPv6\Datagrams Received Address Errors', + + '\WFPv4\Packets Discarded/sec', + '\WFPv6\Packets Discarded/sec' +) + # ========================= # Main workflow # ========================= @@ -797,9 +868,9 @@ try { # Set-RssSettings -AdapterName $n -CpuCount $RssCpuCount # } - # Debug RSS state for all NICs - Write-Host "\n[Debug] Current RSS settings for all adapters:" - $RssState = Get-NetAdapterRss + # # Debug RSS state for all NICs + # Write-Host "\n[Debug] Current RSS settings for all adapters:" + # $RssState = Get-NetAdapterRss # Format the output nicely $output = $RssState | Format-Table Name, Enabled, BaseProcessorGroup, MaxProcessorNumber, MaxProcessors, Profile -AutoSize Write-Host $output @@ -817,6 +888,7 @@ try { # Launch per-CPU usage monitor as a background job (returns array of per-CPU averages) $cpuMonitorJob = CaptureIndividualCpuUsagePerformanceMonitorAsJob $Duration + $perfCounterJob = CapturePerformanceMonitorAsJob -DurationSeconds $Duration -Counters $PerformanceCounters # Run tests Run-SendTest -PeerName $PeerName -Session $Session -SenderOptions $SenderOptions -ReceiverOptions $ReceiverOptions @@ -837,8 +909,13 @@ try { Write-Host "CPU1 $([math]::Round([double]$cpuUsagePerCpu, 2)) %" } + # Write the performance counter results as a JSON file + $perfResults = Receive-Job -Job $perfCounterJob -Wait -AutoRemoveJob + $perfJsonPath = Join-Path $cwd 'echo_client_perf_counters.json' + # Launch another per-CPU usage monitor for the recv test $cpuMonitorJob = CaptureIndividualCpuUsagePerformanceMonitorAsJob $Duration + $perfCounterJob = CapturePerformanceMonitorAsJob -DurationSeconds $Duration -Counters $PerformanceCounters Run-RecvTest -PeerName $PeerName -Session $Session -SenderOptions $SenderOptions -ReceiverOptions $ReceiverOptions @@ -858,6 +935,10 @@ try { Write-Host "CPU1 $([math]::Round([double]$cpuUsagePerCpu, 2)) %" } + # Write the performance counter results as a JSON file + $perfResults = Receive-Job -Job $perfCounterJob -Wait -AutoRemoveJob + $perfJsonPath = Join-Path $cwd 'echo_server_perf_counters.json' + Write-Host "echo tests completed successfully." } catch { From 6140571ec8bedcc476274cf6375d314863139ea5 Mon Sep 17 00:00:00 2001 From: Alan Jowett Date: Wed, 3 Dec 2025 15:17:53 -0800 Subject: [PATCH 58/70] Add additional counters Signed-off-by: Alan Jowett --- .github/scripts/run-echo-test.ps1 | 32 ++----------------------------- 1 file changed, 2 insertions(+), 30 deletions(-) diff --git a/.github/scripts/run-echo-test.ps1 b/.github/scripts/run-echo-test.ps1 index 590e0202..4301779c 100644 --- a/.github/scripts/run-echo-test.ps1 +++ b/.github/scripts/run-echo-test.ps1 @@ -845,36 +845,8 @@ try { $cwd = (Get-Location).Path Write-Host "Current working directory: $cwd" - # # Debug: list adapters and IPv4 addresses to help with troubleshooting - # Write-Host "\n[Debug] Network adapters:" - # try { - # Get-NetAdapter | Format-Table Name, Status, LinkSpeed, InterfaceDescription -AutoSize - # } catch { - # Write-Host "[Debug] Get-NetAdapter not available: $($_.Exception.Message)" - # } - # Write-Host "\n[Debug] IPv4 addresses:" - # try { - # Get-NetIPAddress -AddressFamily IPv4 | Format-Table InterfaceAlias, IPAddress, PrefixLength -AutoSize - # } catch { - # Write-Host "[Debug] Get-NetIPAddress not available: $($_.Exception.Message)" - # } - - # # Determine the NIC to configure RSS on (10Gbps or higher) - # $Nic = (Get-NetAdapter | where-object -Property LinkSpeed -GE 10).Name - - # # Set this on each NIC that meets the criteria - # foreach ($n in $Nic) { - # Write-Host "Configuring RSS on adapter '$Nic' to use $RssCpuCount CPUs..." - # Set-RssSettings -AdapterName $n -CpuCount $RssCpuCount - # } - - # # Debug RSS state for all NICs - # Write-Host "\n[Debug] Current RSS settings for all adapters:" - # $RssState = Get-NetAdapterRss - # Format the output nicely - $output = $RssState | Format-Table Name, Enabled, BaseProcessorGroup, MaxProcessorNumber, MaxProcessors, Profile -AutoSize - Write-Host $output - + Get-NetAdapterRss + Write-Host "\nStarting echo tests to peer '$PeerName' with duration $Duration seconds..." # Create remote session From 914d104a042c73963e7e6c1609c6f12cfd8cad3c Mon Sep 17 00:00:00 2001 From: Alan Jowett Date: Wed, 3 Dec 2025 15:24:02 -0800 Subject: [PATCH 59/70] Add pktmon capture Signed-off-by: Alan Jowett --- .github/workflows/network-traffic-performance.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.github/workflows/network-traffic-performance.yml b/.github/workflows/network-traffic-performance.yml index 0ec163c1..b0955a8d 100644 --- a/.github/workflows/network-traffic-performance.yml +++ b/.github/workflows/network-traffic-performance.yml @@ -273,6 +273,12 @@ jobs: Write-Output "Files in echo RelWithDebInfo directory:" Get-ChildItem -Path ${{ github.workspace }}\echo\RelWithDebInfo -Recurse + - name: Start PktMon drop Capture + run: | + pktmon filter add -p 7 + pktmon filter list + pktmon start --etw -c + - name: Run traffic tests working-directory: ${{ github.workspace }}\echo\RelWithDebInfo env: @@ -299,6 +305,13 @@ jobs: Write-Output "Running: $cmd" Invoke-Expression $cmd + - name: Stop PktMon drop Capture + run: | + pktmon stop + copy pktmon.etl ${{ github.workspace }}\ETL\pktmon_drop_trace.etl + pktmon reset + pktmon filter remove all + - name: Optional - Stop TCPIP tracing if: ${{ github.event.inputs.tcp_ip_tracing == 'true' }} run: | From b6961e93402f03230c36f06426d9d3dbf574ba06 Mon Sep 17 00:00:00 2001 From: Alan Jowett Date: Wed, 3 Dec 2025 15:25:46 -0800 Subject: [PATCH 60/70] Add pktmon capture Signed-off-by: Alan Jowett --- .github/scripts/performance_utlilites.psm1 | 165 +++++++++++++++++++- .github/scripts/run-echo-test.ps1 | 167 +-------------------- 2 files changed, 164 insertions(+), 168 deletions(-) diff --git a/.github/scripts/performance_utlilites.psm1 b/.github/scripts/performance_utlilites.psm1 index b698c9e5..fbdcdec9 100644 --- a/.github/scripts/performance_utlilites.psm1 +++ b/.github/scripts/performance_utlilites.psm1 @@ -250,8 +250,6 @@ function CaptureIndividualCpuUsagePerformanceMonitorAsJob { return $cpuMonitorJob } - - function Restore-FirewallAndCleanup { param([object]$Session) @@ -298,3 +296,166 @@ function Restore-FirewallAndCleanup { $_ | Write-DetailedError } } + +function Set-RssSettings { + param( + [Parameter(Mandatory=$true)][string]$AdapterName, + [Parameter(Mandatory=$false)][int]$CpuCount + ) + + Write-Host "Applying RSS settings to adapter '$AdapterName'..." + + function Get-RssCapabilities { + param([string]$AdapterName) + + # Prefer the convenient cmdlet if available + if (Get-Command -Name Get-NetAdapterRssCapabilities -ErrorAction SilentlyContinue) { + try { + $res = Get-NetAdapterRssCapabilities -Name $AdapterName -ErrorAction SilentlyContinue + if ($res) { + return [PSCustomObject]@{ + MaxProcessorNumber = [int]$res.MaxProcessorNumber + MaxProcessorGroup = [int]$res.MaxProcessorGroup + } + } + return $null + } catch { + return $null + } + } + + # Fallback: query the CIM class directly + try { + $filter = "Name='$AdapterName'" + $obj = Get-CimInstance -Namespace root/StandardCimv2 -ClassName MSFT_NetAdapterRssSettingData -Filter $filter -ErrorAction SilentlyContinue + if ($obj) { + return [PSCustomObject]@{ + MaxProcessorNumber = [int]$obj.MaxProcessorNumber + MaxProcessorGroup = [int]$obj.MaxProcessorGroup + } + } + } catch { + # ignore + } + return $null + } + + # Use the adapter name provided by the caller and validate it exists and is operational + try { + $adapter = Get-NetAdapter -Name $AdapterName -ErrorAction SilentlyContinue + } catch { + $adapter = $null + } + if (-not $adapter) { + Write-Host "Adapter '$AdapterName' not found. Returning." + return + } + if ($adapter.Status -ne 'Up') { + Write-Host "Adapter '$AdapterName' is not operational (Status: $($adapter.Status)). Returning." + return + } + + $ReachableNIC = $adapter.Name + Write-Host "Configuring RSS on adapter: $ReachableNIC" + + # Check RSS capabilities (cmdlet or CIM fallback) + $capCheck = Get-RssCapabilities -AdapterName $ReachableNIC + if (-not $capCheck) { + Write-Host "Adapter '$ReachableNIC' does not expose RSS capabilities or does not support RSS. Skipping RSS configuration." + return + } + + # Enable RSS if the cmdlet exists; otherwise inform and continue to capability-only flow + if (Get-Command -Name Enable-NetAdapterRss -ErrorAction SilentlyContinue) { + try { + Enable-NetAdapterRss -Name $ReachableNIC -ErrorAction Stop + } catch { + Write-Host "Failed to enable RSS on '$ReachableNIC': $($_.Exception.Message)" + return + } + } else { + Write-Host "Enable-NetAdapterRss cmdlet not present; skipping enable step." + } + + # Get RSS capabilities to determine CPU range + # Re-read capabilities (cmdlet or CIM fallback) + $cap = Get-RssCapabilities -AdapterName $ReachableNIC + if (-not $cap) { + Write-Host "No RSS capability information returned. Returning." + return + } + + $maxCPU = $cap.MaxProcessorNumber + $maxGroup = $cap.MaxProcessorGroup + + if (-not ($maxCPU -is [int]) -or -not ($maxGroup -is [int])) { + Write-Host "Unexpected RSS capability values. Returning." + return + } + + if ($maxGroup -lt 1) { + Write-Host "No processor groups reported. Assuming group 0." + $useGroup = 0 + } else { + $useGroup = 0 + } + if ($maxCPU -lt 0) { + Write-Host "Invalid MaxProcessorNumber ($maxCPU). Returning." + return + } + + if (Get-Command -Name Set-NetAdapterRss -ErrorAction SilentlyContinue) { + # Determine how many CPUs to set. Use provided CpuCount if valid, otherwise the adapter max. + if ($CpuCount) { + if ($CpuCount -lt 1) { + Write-Host "Provided CpuCount ($CpuCount) is invalid; must be >= 1. Returning." + return + } + if ($CpuCount -gt ($maxCPU + 1)) { + Write-Host "Provided CpuCount ($CpuCount) exceeds adapter MaxProcessorNumber ($maxCPU + 1). Clamping to max." + $useMax = $maxCPU + } else { + # Convert CpuCount (count) to MaxProcessorNumber (index) + $useMax = [int]($CpuCount - 1) + } + } else { + $useMax = $maxCPU + } + + $CpuCount = $useMax + 1 + + Write-Host "Setting RSS to use CPUs 0..$useMax in group $useGroup" + try { + Set-NetAdapterRss -Name $ReachableNIC -BaseProcessorGroup $useGroup -MaxProcessorNumber $useMax -MaxProcessors $CpuCount -Profile NUMAStatic -ErrorAction Stop + } catch { + Write-Host "Failed to set RSS on '$ReachableNIC': $($_.Exception.Message)" + return + } + + # Disable then re-enable the adapter to ensure settings apply + if ((Get-Command -Name Disable-NetAdapter -ErrorAction SilentlyContinue) -and (Get-Command -Name Enable-NetAdapter -ErrorAction SilentlyContinue)) { + try { + Write-Host "Disabling adapter '$ReachableNIC' to apply settings..." + Disable-NetAdapter -Name $ReachableNIC -Confirm:$false -ErrorAction Stop + Start-Sleep -Seconds 2 + Write-Host "Re-enabling adapter '$ReachableNIC'..." + Enable-NetAdapter -Name $ReachableNIC -Confirm:$false -ErrorAction Stop + Start-Sleep -Seconds 2 + } catch { + Write-Host "Warning: failed to toggle adapter '$ReachableNIC': $($_.Exception.Message)" + } + } else { + Write-Host "Disable/Enable adapter cmdlets not present; skipping adapter toggle." + } + + if (Get-Command -Name Get-NetAdapterRss -ErrorAction SilentlyContinue) { + Write-Host "Updated RSS settings for '$ReachableNIC':" + Get-NetAdapterRss -Name $ReachableNIC + } else { + Write-Host "Set successful (no Get-NetAdapterRss cmdlet present to display settings)." + } + } else { + Write-Host "Set-NetAdapterRss cmdlet not present; cannot modify RSS settings. Displaying reported capabilities instead:" + Write-Host "MaxProcessorNumber: $maxCPU, MaxProcessorGroup: $maxGroup" + } +} \ No newline at end of file diff --git a/.github/scripts/run-echo-test.ps1 b/.github/scripts/run-echo-test.ps1 index 4301779c..979e0bc6 100644 --- a/.github/scripts/run-echo-test.ps1 +++ b/.github/scripts/run-echo-test.ps1 @@ -3,8 +3,7 @@ param( [string]$PeerName, [string]$SenderOptions, [string]$ReceiverOptions, - [string]$Duration = "60", # Duration in seconds for the test runs (default 60s) - [int]$RssCpuCount = 0 # Number of CPUs to enable RSS on the remote server (0 = max available + [string]$Duration = "60" ) Set-StrictMode -Version Latest @@ -135,170 +134,6 @@ function Write-DetailedError { } } -# Mellanox CX5 limits RSS to 8 CPUs even if more are available, so setting this is pointless. -# function Set-RssSettings { -# param( -# [Parameter(Mandatory=$true)][string]$AdapterName, -# [Parameter(Mandatory=$false)][int]$CpuCount -# ) - -# Write-Host "Applying RSS settings to adapter '$AdapterName'..." - -# function Get-RssCapabilities { -# param([string]$AdapterName) - -# # Prefer the convenient cmdlet if available -# if (Get-Command -Name Get-NetAdapterRssCapabilities -ErrorAction SilentlyContinue) { -# try { -# $res = Get-NetAdapterRssCapabilities -Name $AdapterName -ErrorAction SilentlyContinue -# if ($res) { -# return [PSCustomObject]@{ -# MaxProcessorNumber = [int]$res.MaxProcessorNumber -# MaxProcessorGroup = [int]$res.MaxProcessorGroup -# } -# } -# return $null -# } catch { -# return $null -# } -# } - -# # Fallback: query the CIM class directly -# try { -# $filter = "Name='$AdapterName'" -# $obj = Get-CimInstance -Namespace root/StandardCimv2 -ClassName MSFT_NetAdapterRssSettingData -Filter $filter -ErrorAction SilentlyContinue -# if ($obj) { -# return [PSCustomObject]@{ -# MaxProcessorNumber = [int]$obj.MaxProcessorNumber -# MaxProcessorGroup = [int]$obj.MaxProcessorGroup -# } -# } -# } catch { -# # ignore -# } -# return $null -# } - -# # Use the adapter name provided by the caller and validate it exists and is operational -# try { -# $adapter = Get-NetAdapter -Name $AdapterName -ErrorAction SilentlyContinue -# } catch { -# $adapter = $null -# } -# if (-not $adapter) { -# Write-Host "Adapter '$AdapterName' not found. Returning." -# return -# } -# if ($adapter.Status -ne 'Up') { -# Write-Host "Adapter '$AdapterName' is not operational (Status: $($adapter.Status)). Returning." -# return -# } - -# $ReachableNIC = $adapter.Name -# Write-Host "Configuring RSS on adapter: $ReachableNIC" - -# # Check RSS capabilities (cmdlet or CIM fallback) -# $capCheck = Get-RssCapabilities -AdapterName $ReachableNIC -# if (-not $capCheck) { -# Write-Host "Adapter '$ReachableNIC' does not expose RSS capabilities or does not support RSS. Skipping RSS configuration." -# return -# } - -# # Enable RSS if the cmdlet exists; otherwise inform and continue to capability-only flow -# if (Get-Command -Name Enable-NetAdapterRss -ErrorAction SilentlyContinue) { -# try { -# Enable-NetAdapterRss -Name $ReachableNIC -ErrorAction Stop -# } catch { -# Write-Host "Failed to enable RSS on '$ReachableNIC': $($_.Exception.Message)" -# return -# } -# } else { -# Write-Host "Enable-NetAdapterRss cmdlet not present; skipping enable step." -# } - -# # Get RSS capabilities to determine CPU range -# # Re-read capabilities (cmdlet or CIM fallback) -# $cap = Get-RssCapabilities -AdapterName $ReachableNIC -# if (-not $cap) { -# Write-Host "No RSS capability information returned. Returning." -# return -# } - -# $maxCPU = $cap.MaxProcessorNumber -# $maxGroup = $cap.MaxProcessorGroup - -# if (-not ($maxCPU -is [int]) -or -not ($maxGroup -is [int])) { -# Write-Host "Unexpected RSS capability values. Returning." -# return -# } - -# if ($maxGroup -lt 1) { -# Write-Host "No processor groups reported. Assuming group 0." -# $useGroup = 0 -# } else { -# $useGroup = 0 -# } -# if ($maxCPU -lt 0) { -# Write-Host "Invalid MaxProcessorNumber ($maxCPU). Returning." -# return -# } - -# if (Get-Command -Name Set-NetAdapterRss -ErrorAction SilentlyContinue) { -# # Determine how many CPUs to set. Use provided CpuCount if valid, otherwise the adapter max. -# if ($CpuCount) { -# if ($CpuCount -lt 1) { -# Write-Host "Provided CpuCount ($CpuCount) is invalid; must be >= 1. Returning." -# return -# } -# if ($CpuCount -gt ($maxCPU + 1)) { -# Write-Host "Provided CpuCount ($CpuCount) exceeds adapter MaxProcessorNumber ($maxCPU + 1). Clamping to max." -# $useMax = $maxCPU -# } else { -# # Convert CpuCount (count) to MaxProcessorNumber (index) -# $useMax = [int]($CpuCount - 1) -# } -# } else { -# $useMax = $maxCPU -# } - -# $CpuCount = $useMax + 1 - -# Write-Host "Setting RSS to use CPUs 0..$useMax in group $useGroup" -# try { -# Set-NetAdapterRss -Name $ReachableNIC -BaseProcessorGroup $useGroup -MaxProcessorNumber $useMax -MaxProcessors $CpuCount -Profile NUMAStatic -ErrorAction Stop -# } catch { -# Write-Host "Failed to set RSS on '$ReachableNIC': $($_.Exception.Message)" -# return -# } - -# # Disable then re-enable the adapter to ensure settings apply -# if ((Get-Command -Name Disable-NetAdapter -ErrorAction SilentlyContinue) -and (Get-Command -Name Enable-NetAdapter -ErrorAction SilentlyContinue)) { -# try { -# Write-Host "Disabling adapter '$ReachableNIC' to apply settings..." -# Disable-NetAdapter -Name $ReachableNIC -Confirm:$false -ErrorAction Stop -# Start-Sleep -Seconds 2 -# Write-Host "Re-enabling adapter '$ReachableNIC'..." -# Enable-NetAdapter -Name $ReachableNIC -Confirm:$false -ErrorAction Stop -# Start-Sleep -Seconds 2 -# } catch { -# Write-Host "Warning: failed to toggle adapter '$ReachableNIC': $($_.Exception.Message)" -# } -# } else { -# Write-Host "Disable/Enable adapter cmdlets not present; skipping adapter toggle." -# } - -# if (Get-Command -Name Get-NetAdapterRss -ErrorAction SilentlyContinue) { -# Write-Host "Updated RSS settings for '$ReachableNIC':" -# Get-NetAdapterRss -Name $ReachableNIC -# } else { -# Write-Host "Set successful (no Get-NetAdapterRss cmdlet present to display settings)." -# } -# } else { -# Write-Host "Set-NetAdapterRss cmdlet not present; cannot modify RSS settings. Displaying reported capabilities instead:" -# Write-Host "MaxProcessorNumber: $maxCPU, MaxProcessorGroup: $maxGroup" -# } -# } - # WPR CPU profiling helpers $script:WprProfiles = @{} From 6b8df6d971709dafd1bc96551dd164a9673d2c09 Mon Sep 17 00:00:00 2001 From: Alan Jowett Date: Wed, 3 Dec 2025 15:33:15 -0800 Subject: [PATCH 61/70] Add pktmon capture Signed-off-by: Alan Jowett --- .github/scripts/run-echo-test.ps1 | 3 +++ .github/workflows/network-traffic-performance.yml | 7 +++++++ 2 files changed, 10 insertions(+) diff --git a/.github/scripts/run-echo-test.ps1 b/.github/scripts/run-echo-test.ps1 index 979e0bc6..f5b0ea0c 100644 --- a/.github/scripts/run-echo-test.ps1 +++ b/.github/scripts/run-echo-test.ps1 @@ -746,6 +746,9 @@ try { $perfResults = Receive-Job -Job $perfCounterJob -Wait -AutoRemoveJob $perfJsonPath = Join-Path $cwd 'echo_server_perf_counters.json' + # Copy the stats file to the parent folder for GitHub Actions artifact upload + Copy-Item -Path *.json -Destination $cwd\.. -Force + Write-Host "echo tests completed successfully." } catch { diff --git a/.github/workflows/network-traffic-performance.yml b/.github/workflows/network-traffic-performance.yml index b0955a8d..9419c2b5 100644 --- a/.github/workflows/network-traffic-performance.yml +++ b/.github/workflows/network-traffic-performance.yml @@ -341,6 +341,13 @@ jobs: name: cpu_profile_${{ matrix.vec.env }}_${{ matrix.vec.os }}_${{ matrix.vec.arch }} path: ${{ github.workspace }}\ETL\cpu_profile-*.etl + - name: Upload PktMon Drop ETL + if: always() + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 + with: + name: pktmon_drop_trace_${{ matrix.vec.env }}_${{ matrix.vec.os }}_${{ matrix.vec.arch }} + path: ${{ github.workspace }}\ETL\pktmon_drop_trace.etl + - name: Upload Crash Dumps if: always() uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 From 4a8082a56802cddae8f3a36e106cec057d22ac32 Mon Sep 17 00:00:00 2001 From: Alan Jowett Date: Wed, 3 Dec 2025 15:59:10 -0800 Subject: [PATCH 62/70] Add pktmon capture Signed-off-by: Alan Jowett --- .github/scripts/run-echo-test.ps1 | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/.github/scripts/run-echo-test.ps1 b/.github/scripts/run-echo-test.ps1 index f5b0ea0c..3971558b 100644 --- a/.github/scripts/run-echo-test.ps1 +++ b/.github/scripts/run-echo-test.ps1 @@ -719,6 +719,7 @@ try { # Write the performance counter results as a JSON file $perfResults = Receive-Job -Job $perfCounterJob -Wait -AutoRemoveJob $perfJsonPath = Join-Path $cwd 'echo_client_perf_counters.json' + $perfResults | ConvertTo-Json | Out-File -FilePath $perfJsonPath -Encoding utf8 -Force # Launch another per-CPU usage monitor for the recv test $cpuMonitorJob = CaptureIndividualCpuUsagePerformanceMonitorAsJob $Duration @@ -745,6 +746,17 @@ try { # Write the performance counter results as a JSON file $perfResults = Receive-Job -Job $perfCounterJob -Wait -AutoRemoveJob $perfJsonPath = Join-Path $cwd 'echo_server_perf_counters.json' + $perfResults | ConvertTo-Json | Out-File -FilePath $perfJsonPath -Encoding utf8 -Force + + # List json files in cwd + Write-Host "JSON files in $cwd:" + Get-ChildItem -Path $cwd -Filter *.json | ForEach-Object { Write-Host " $($_.FullName)" } + + # Print each JSON file's contents + Get-ChildItem -Path $cwd -Filter *.json | ForEach-Object { + Write-Host "Contents of $($_.FullName):" + Get-Content -Path $_.FullName | ForEach-Object { Write-Host " $_" } + } # Copy the stats file to the parent folder for GitHub Actions artifact upload Copy-Item -Path *.json -Destination $cwd\.. -Force From 1545a2b9ef8f265272512f47303e938080f3d4d7 Mon Sep 17 00:00:00 2001 From: Alan Jowett Date: Wed, 3 Dec 2025 16:01:43 -0800 Subject: [PATCH 63/70] Add pktmon capture Signed-off-by: Alan Jowett --- .github/workflows/network-traffic-performance.yml | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/.github/workflows/network-traffic-performance.yml b/.github/workflows/network-traffic-performance.yml index 9419c2b5..5a86068b 100644 --- a/.github/workflows/network-traffic-performance.yml +++ b/.github/workflows/network-traffic-performance.yml @@ -273,6 +273,10 @@ jobs: Write-Output "Files in echo RelWithDebInfo directory:" Get-ChildItem -Path ${{ github.workspace }}\echo\RelWithDebInfo -Recurse + - name: Log pktmon drop counters before test + run: | + pktmon comp counters --counter-type drops + - name: Start PktMon drop Capture run: | pktmon filter add -p 7 @@ -312,6 +316,10 @@ jobs: pktmon reset pktmon filter remove all + - name: Log pktmon drop counters after test + run: | + pktmon comp counters --counter-type drops + - name: Optional - Stop TCPIP tracing if: ${{ github.event.inputs.tcp_ip_tracing == 'true' }} run: | From 4415ec68fe0181eb716cdac28a6d8591c9b65637 Mon Sep 17 00:00:00 2001 From: Alan Jowett Date: Wed, 3 Dec 2025 16:02:35 -0800 Subject: [PATCH 64/70] Add pktmon capture Signed-off-by: Alan Jowett --- .github/scripts/run-echo-test.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/run-echo-test.ps1 b/.github/scripts/run-echo-test.ps1 index 3971558b..1fa6af43 100644 --- a/.github/scripts/run-echo-test.ps1 +++ b/.github/scripts/run-echo-test.ps1 @@ -754,7 +754,7 @@ try { # Print each JSON file's contents Get-ChildItem -Path $cwd -Filter *.json | ForEach-Object { - Write-Host "Contents of $($_.FullName):" + Write-Host "Contents of $($_.FullName) - " Get-Content -Path $_.FullName | ForEach-Object { Write-Host " $_" } } From 64cd0a0ce852215024a2732989d87ba796167d35 Mon Sep 17 00:00:00 2001 From: Alan Jowett Date: Wed, 3 Dec 2025 16:06:21 -0800 Subject: [PATCH 65/70] Add pktmon capture Signed-off-by: Alan Jowett --- .github/workflows/network-traffic-performance.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/network-traffic-performance.yml b/.github/workflows/network-traffic-performance.yml index 5a86068b..edf336b5 100644 --- a/.github/workflows/network-traffic-performance.yml +++ b/.github/workflows/network-traffic-performance.yml @@ -275,7 +275,7 @@ jobs: - name: Log pktmon drop counters before test run: | - pktmon comp counters --counter-type drops + pktmon comp counters --type drop - name: Start PktMon drop Capture run: | @@ -318,7 +318,7 @@ jobs: - name: Log pktmon drop counters after test run: | - pktmon comp counters --counter-type drops + pktmon comp counters --type drop - name: Optional - Stop TCPIP tracing if: ${{ github.event.inputs.tcp_ip_tracing == 'true' }} From 0b425c23ffbad0d41c979f3209a54350329325f3 Mon Sep 17 00:00:00 2001 From: Alan Jowett Date: Wed, 3 Dec 2025 16:08:54 -0800 Subject: [PATCH 66/70] Add pktmon capture Signed-off-by: Alan Jowett --- .github/scripts/run-echo-test.ps1 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/scripts/run-echo-test.ps1 b/.github/scripts/run-echo-test.ps1 index 1fa6af43..bc5244a3 100644 --- a/.github/scripts/run-echo-test.ps1 +++ b/.github/scripts/run-echo-test.ps1 @@ -749,7 +749,7 @@ try { $perfResults | ConvertTo-Json | Out-File -FilePath $perfJsonPath -Encoding utf8 -Force # List json files in cwd - Write-Host "JSON files in $cwd:" + Write-Host "JSON files in $cwd" Get-ChildItem -Path $cwd -Filter *.json | ForEach-Object { Write-Host " $($_.FullName)" } # Print each JSON file's contents From bdf82f9042842d96b9a159ad0c09b51f6fcfef30 Mon Sep 17 00:00:00 2001 From: Alan Jowett Date: Thu, 4 Dec 2025 09:10:43 -0800 Subject: [PATCH 67/70] Cleanup Signed-off-by: Alan Jowett --- .github/scripts/performance_utlilites.psm1 | 49 ++++++++ .github/scripts/run-echo-test.ps1 | 5 +- test.wprp | 132 --------------------- 3 files changed, 52 insertions(+), 134 deletions(-) delete mode 100644 test.wprp diff --git a/.github/scripts/performance_utlilites.psm1 b/.github/scripts/performance_utlilites.psm1 index fbdcdec9..2f0fa73e 100644 --- a/.github/scripts/performance_utlilites.psm1 +++ b/.github/scripts/performance_utlilites.psm1 @@ -458,4 +458,53 @@ function Set-RssSettings { Write-Host "Set-NetAdapterRss cmdlet not present; cannot modify RSS settings. Displaying reported capabilities instead:" Write-Host "MaxProcessorNumber: $maxCPU, MaxProcessorGroup: $maxGroup" } +} + +function CapturePerformanceMonitorAsJob { + param( + [Parameter(Mandatory=$true)][string]$DurationSeconds, + [Parameter(Mandatory=$false)][string[]]$Counters = @('\Processor Information(*)\% Processor Time') + ) + + # Ensure numeric duration + $intDuration = [int]::Parse($DurationSeconds) + + $perfJob = Start-Job -ScriptBlock { + param($duration, $counters) + + $d = [int]$duration + if (-not $counters -or $counters.Count -eq 0) { + $counters = @('\Processor Information(*)\% Processor Time') + } + + try { + # Collect all requested counters in one Get-Counter call if possible + $samples = Get-Counter -Counter $counters -SampleInterval 1 -MaxSamples $d -ErrorAction Stop + + # Group by counter path and instance name, compute per-instance averages + # CounterSamples contain Path, InstanceName, CounterName, CookedValue + $groupedByCounter = $samples.CounterSamples | Group-Object -Property Path + + $results = @() + foreach ($counterGroup in $groupedByCounter) { + $path = $counterGroup.Name + $innerGroups = $counterGroup.Group | Group-Object -Property InstanceName + foreach ($inst in $innerGroups) { + $instName = $inst.Name + if ([string]::IsNullOrEmpty($instName) -or $instName -eq '_Total') { continue } + $vals = $inst.Group | ForEach-Object { [double]$_.CookedValue } + $avg = ($vals | Measure-Object -Average).Average + $results += [PSCustomObject]@{ Counter = $path; Instance = $instName; Average = $avg } + } + } + + # Emit structured results: an array of PSObjects with Counter, Instance, Average + $results + } + catch { + Write-Output (@([PSCustomObject]@{ Counter = 'error'; Instance = ''; Average = 0; ErrorMessage = $_.Exception.Message })) + } + } -ArgumentList $intDuration, $Counters + + return $perfJob } \ No newline at end of file diff --git a/.github/scripts/run-echo-test.ps1 b/.github/scripts/run-echo-test.ps1 index bc5244a3..e4c756de 100644 --- a/.github/scripts/run-echo-test.ps1 +++ b/.github/scripts/run-echo-test.ps1 @@ -595,7 +595,7 @@ function CapturePerformanceMonitorAsJob { $results } catch { - Write-Output (@([PSCustomObject]@{ Counter = 'error'; Instance = ''; Average = 0 })) + Write-Output (@([PSCustomObject]@{ Counter = 'error'; Instance = ''; Average = 0; ErrorMessage = $_.Exception.Message })) } } -ArgumentList $intDuration, $Counters @@ -668,7 +668,8 @@ $PerformanceCounters = '\IPv6\Datagrams Received Address Errors', '\WFPv4\Packets Discarded/sec', - '\WFPv6\Packets Discarded/sec' + '\WFPv6\Packets Discarded/sec', + '\Processor Information(*)\% Processor Time' ) # ========================= diff --git a/test.wprp b/test.wprp deleted file mode 100644 index b925fe75..00000000 --- a/test.wprp +++ /dev/null @@ -1,132 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file From 07b2f883d3617c024b5ae74f7112f94480919882 Mon Sep 17 00:00:00 2001 From: Alan Jowett Date: Fri, 5 Dec 2025 14:04:03 -0800 Subject: [PATCH 68/70] Skip bad counters Signed-off-by: Alan Jowett --- .github/scripts/performance_utlilites.psm1 | 65 +++++++++++++++------- .github/scripts/run-echo-test.ps1 | 65 +++++++++++++++------- 2 files changed, 90 insertions(+), 40 deletions(-) diff --git a/.github/scripts/performance_utlilites.psm1 b/.github/scripts/performance_utlilites.psm1 index 2f0fa73e..4e1ef034 100644 --- a/.github/scripts/performance_utlilites.psm1 +++ b/.github/scripts/performance_utlilites.psm1 @@ -477,33 +477,58 @@ function CapturePerformanceMonitorAsJob { $counters = @('\Processor Information(*)\% Processor Time') } - try { - # Collect all requested counters in one Get-Counter call if possible - $samples = Get-Counter -Counter $counters -SampleInterval 1 -MaxSamples $d -ErrorAction Stop + # Sample once per second for the requested duration and accumulate per-counter values. + $store = @{} - # Group by counter path and instance name, compute per-instance averages - # CounterSamples contain Path, InstanceName, CounterName, CookedValue - $groupedByCounter = $samples.CounterSamples | Group-Object -Property Path + for ($i = 0; $i -lt $d; $i++) { + $samples = $null + try { + # Try to collect all counters in a single quick sample (returns immediately) + $samples = Get-Counter -Counter $counters -MaxSamples 1 -ErrorAction Stop + } + catch { + # If that fails, collect available counters individually (quick single-sample calls) + $samples = New-Object System.Collections.Generic.List[object] + foreach ($c in $counters) { + try { + $s = Get-Counter -Counter $c -MaxSamples 1 -ErrorAction Stop + if ($s.CounterSamples) { $s.CounterSamples | ForEach-Object { [void]$samples.Add($_) } } + } + catch { + # skip bad counter + continue + } + } + } - $results = @() - foreach ($counterGroup in $groupedByCounter) { - $path = $counterGroup.Name - $innerGroups = $counterGroup.Group | Group-Object -Property InstanceName - foreach ($inst in $innerGroups) { - $instName = $inst.Name - if ([string]::IsNullOrEmpty($instName) -or $instName -eq '_Total') { continue } - $vals = $inst.Group | ForEach-Object { [double]$_.CookedValue } - $avg = ($vals | Measure-Object -Average).Average - $results += [PSCustomObject]@{ Counter = $path; Instance = $instName; Average = $avg } + if ($samples -ne $null) { + $csamples = $samples.CounterSamples + if (-not $csamples -and ($samples -is [System.Collections.IEnumerable])) { $csamples = $samples } + foreach ($cs in $csamples) { + $path = $cs.Path + $inst = $cs.InstanceName + if ([string]::IsNullOrEmpty($inst) -or $inst -eq '_Total') { continue } + if ($cs.Status -ne 'Success') { continue } + $key = "$path`|$inst" + if (-not $store.ContainsKey($key)) { $store[$key] = New-Object System.Collections.ArrayList } + [void]$store[$key].Add([double]$cs.CookedValue) } } - # Emit structured results: an array of PSObjects with Counter, Instance, Average - $results + Start-Sleep -Seconds 1 } - catch { - Write-Output (@([PSCustomObject]@{ Counter = 'error'; Instance = ''; Average = 0; ErrorMessage = $_.Exception.Message })) + + $results = @() + foreach ($k in $store.Keys) { + $parts = $k -split '\|',2 + $path = $parts[0] + $inst = $parts[1] + $avg = ($store[$k] | Measure-Object -Average).Average + $results += [PSCustomObject]@{ Counter = $path; Instance = $inst; Average = $avg } } + + # Emit structured results: an array of PSObjects with Counter, Instance, Average + $results } -ArgumentList $intDuration, $Counters return $perfJob diff --git a/.github/scripts/run-echo-test.ps1 b/.github/scripts/run-echo-test.ps1 index e4c756de..4c27b595 100644 --- a/.github/scripts/run-echo-test.ps1 +++ b/.github/scripts/run-echo-test.ps1 @@ -570,33 +570,58 @@ function CapturePerformanceMonitorAsJob { $counters = @('\Processor Information(*)\% Processor Time') } - try { - # Collect all requested counters in one Get-Counter call if possible - $samples = Get-Counter -Counter $counters -SampleInterval 1 -MaxSamples $d -ErrorAction Stop + # Sample once per second for the requested duration and accumulate per-counter values. + $store = @{} - # Group by counter path and instance name, compute per-instance averages - # CounterSamples contain Path, InstanceName, CounterName, CookedValue - $groupedByCounter = $samples.CounterSamples | Group-Object -Property Path + for ($i = 0; $i -lt $d; $i++) { + $samples = $null + try { + # Try to collect all counters in a single quick sample (returns immediately) + $samples = Get-Counter -Counter $counters -MaxSamples 1 -ErrorAction Stop + } + catch { + # If that fails, collect available counters individually (quick single-sample calls) + $samples = New-Object System.Collections.Generic.List[object] + foreach ($c in $counters) { + try { + $s = Get-Counter -Counter $c -MaxSamples 1 -ErrorAction Stop + if ($s.CounterSamples) { $s.CounterSamples | ForEach-Object { [void]$samples.Add($_) } } + } + catch { + # skip bad counter + continue + } + } + } - $results = @() - foreach ($counterGroup in $groupedByCounter) { - $path = $counterGroup.Name - $innerGroups = $counterGroup.Group | Group-Object -Property InstanceName - foreach ($inst in $innerGroups) { - $instName = $inst.Name - if ([string]::IsNullOrEmpty($instName) -or $instName -eq '_Total') { continue } - $vals = $inst.Group | ForEach-Object { [double]$_.CookedValue } - $avg = ($vals | Measure-Object -Average).Average - $results += [PSCustomObject]@{ Counter = $path; Instance = $instName; Average = $avg } + if ($samples -ne $null) { + $csamples = $samples.CounterSamples + if (-not $csamples -and ($samples -is [System.Collections.IEnumerable])) { $csamples = $samples } + foreach ($cs in $csamples) { + $path = $cs.Path + $inst = $cs.InstanceName + if ([string]::IsNullOrEmpty($inst) -or $inst -eq '_Total') { continue } + if ($cs.Status -ne 'Success') { continue } + $key = "$path`|$inst" + if (-not $store.ContainsKey($key)) { $store[$key] = New-Object System.Collections.ArrayList } + [void]$store[$key].Add([double]$cs.CookedValue) } } - # Emit structured results: an array of PSObjects with Counter, Instance, Average - $results + Start-Sleep -Seconds 1 } - catch { - Write-Output (@([PSCustomObject]@{ Counter = 'error'; Instance = ''; Average = 0; ErrorMessage = $_.Exception.Message })) + + $results = @() + foreach ($k in $store.Keys) { + $parts = $k -split '\|',2 + $path = $parts[0] + $inst = $parts[1] + $avg = ($store[$k] | Measure-Object -Average).Average + $results += [PSCustomObject]@{ Counter = $path; Instance = $inst; Average = $avg } } + + # Emit structured results: an array of PSObjects with Counter, Instance, Average + $results } -ArgumentList $intDuration, $Counters return $perfJob From af4b8395d40bc98c06da3720696bedde94b3c70f Mon Sep 17 00:00:00 2001 From: GitHub Copilot Date: Tue, 6 Jan 2026 17:04:13 -0800 Subject: [PATCH 69/70] Add Linux network traffic workflow using LinuxUDPShardedEcho and Linux pwsh remoting script --- .github/scripts/run-echo-test-linux.ps1 | 95 ++++++++++++ .../network-traffic-performance-linux.yml | 142 ++++++++++++++++++ 2 files changed, 237 insertions(+) create mode 100644 .github/scripts/run-echo-test-linux.ps1 create mode 100644 .github/workflows/network-traffic-performance-linux.yml diff --git a/.github/scripts/run-echo-test-linux.ps1 b/.github/scripts/run-echo-test-linux.ps1 new file mode 100644 index 00000000..c5b191d9 --- /dev/null +++ b/.github/scripts/run-echo-test-linux.ps1 @@ -0,0 +1,95 @@ +param( + [string]$PeerName = "netperf-peer", + [string]$SenderOptions, + [string]$ReceiverOptions, + [string]$Duration = "60" +) + +$ErrorActionPreference = 'Stop' + +function Ensure-Executable($path) { + if (-not (Test-Path $path)) { throw "Missing binary: $path" } + try { chmod +x $path } catch { Write-Host "chmod failed (may already be executable): $path" } +} + +# Resolve local echo binaries in current working directory +$cwd = Get-Location +$serverPath = Join-Path $cwd 'echo_server' +$clientPath = Join-Path $cwd 'echo_client' + +Ensure-Executable $serverPath +Ensure-Executable $clientPath + +# Establish SSH PowerShell remoting to Linux peer +Write-Host "Creating PowerShell SSH session to peer: $PeerName" +$session = $null +try { + $session = New-PSSession -HostName $PeerName +} catch { + throw "Failed to create SSH PSSession to $PeerName. Ensure SSH keys/config are set. Error: $_" +} + +# Copy server binary to peer and ensure executable +Write-Host "Copying server binary to peer" +Copy-Item -Path $serverPath -Destination '~/echo_server' -ToSession $session +Invoke-Command -Session $session -ScriptBlock { chmod +x ~/echo_server } + +# Start server in background on peer, capture PID +Write-Host "Starting server on peer" +$startServer = @" + bash -lc "nohup ~/echo_server $using:ReceiverOptions > ~/server.log 2>&1 & echo \$!" +"@ +$serverPid = Invoke-Command -Session $session -ScriptBlock ([ScriptBlock]::Create($startServer)) +Write-Host "Server PID on peer: $serverPid" +Start-Sleep -Seconds 2 + +# Run client locally and capture output +Write-Host "Running echo client locally" +$clientCmd = "bash -lc \"$clientPath $SenderOptions --duration $Duration\"" +$clientOutput = & pwsh -NoProfile -Command $clientCmd 2>&1 +$clientExit = $LASTEXITCODE + +# Save client output +"$clientOutput" | Out-File -FilePath "echo_client_output.txt" -Encoding utf8 +Write-Host "Client exit: $clientExit" + +# Attempt to parse simple metrics from client output +$sent = 0 +$received = 0 +try { + $sentMatch = ($clientOutput | Select-String -Pattern "Packets sent: (\\d+)") + $recvMatch = ($clientOutput | Select-String -Pattern "Packets received: (\\d+)") + if ($sentMatch) { $sent = [int]($sentMatch.Matches[0].Groups[1].Value) } + if ($recvMatch) { $received = [int]($recvMatch.Matches[0].Groups[1].Value) } +} catch { } + +# Write a minimal CSV summary +$csvLines = @() +$csvLines += "Test,Sent,Received,Duration" +$csvLines += "LinuxEcho,$sent,$received,$Duration" +$csvLines | Out-File -FilePath "echo_summary.csv" -Encoding utf8 + +# Stop server on peer and collect logs +Write-Host "Stopping server on peer" +try { + Invoke-Command -Session $session -ScriptBlock { kill -TERM $using:serverPid } | Out-Null +} catch { + Write-Host "Kill failed: $_" +} +Start-Sleep -Seconds 1 + +Write-Host "Fetching server log from peer" +try { + Copy-Item -FromSession $session -Path '~/server.log' -Destination 'server.log' +} catch { + Write-Host "Failed to copy server.log: $_" +} + +# Close session +if ($session) { Remove-PSSession $session } + +# Return non-zero if client failed +if ($clientExit -ne 0) { + Write-Host "Client reported non-zero exit: $clientExit" + exit $clientExit +} diff --git a/.github/workflows/network-traffic-performance-linux.yml b/.github/workflows/network-traffic-performance-linux.yml new file mode 100644 index 00000000..5a1bfb3f --- /dev/null +++ b/.github/workflows/network-traffic-performance-linux.yml @@ -0,0 +1,142 @@ +name: network-traffic-performance-linux + +on: + workflow_dispatch: + inputs: + ref: + description: 'Test Branch or Commit' + required: false + default: 'main' + type: string + sender_options: + description: 'Command-line options for the sender (quoted string)' + required: true + type: string + receiver_options: + description: 'Command-line options for the receiver (quoted string)' + required: true + type: string + test_tool_ref: + description: 'Branch or ref to build the test tool from' + required: false + default: 'main' + type: string + duration: + description: 'Duration of the test in seconds' + required: false + default: '60' + type: string + +permissions: write-all + +jobs: + build_echo_test_linux: + name: Build echo server/client (Linux) + uses: Alan-Jowett/LinuxUDPShardedEcho/.github/workflows/reusable-build.yml@main + with: + artifact_name: echo_test + repository: 'Alan-Jowett/LinuxUDPShardedEcho' + config: 'RelWithDebInfo' + ref: ${{ inputs.test_tool_ref }} + + test_echo_server_linux: + name: Network Traffic Test (echo, Linux) + needs: [build_echo_test_linux] + strategy: + fail-fast: false + matrix: + vec: + - env: lab + os: "ubuntu-24.04" + arch: x64 + runs-on: + - self-hosted + - ${{ matrix.vec.env }} + - Linux + - ${{ matrix.vec.arch }} + - os-${{ matrix.vec.os }} + + steps: + - name: Checkout repository + uses: actions/checkout@9bb56186c3b09b4f86b1c65136769dd318469633 + + - name: Ensure PowerShell 7 is installed (Linux) + shell: bash + run: | + if ! command -v pwsh >/dev/null 2>&1; then + sudo bash ./install-pwsh.sh + fi + + - name: Setup workspace (Linux) + shell: pwsh + run: | + Get-ChildItem | % { Remove-Item -Recurse $_ -Force -ErrorAction SilentlyContinue } + if (Test-Path "$env:GITHUB_WORKSPACE/echo") { Remove-Item -Recurse -Force "$env:GITHUB_WORKSPACE/echo" } + if (Test-Path "$env:GITHUB_WORKSPACE/ETL") { Remove-Item -Recurse -Force "$env:GITHUB_WORKSPACE/ETL" } + New-Item -ItemType Directory -Path "$env:GITHUB_WORKSPACE/echo/build" | Out-Null + New-Item -ItemType Directory -Path "$env:GITHUB_WORKSPACE/ETL" | Out-Null + + - name: Download echo test tool + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 + with: + name: "echo_test" + path: ${{ github.workspace }}/echo + + - name: Download run-echo-test script + shell: pwsh + run: | + Copy-Item -Path "${{ github.workspace }}/.github/scripts/run-echo-test-linux.ps1" -Destination "${{ github.workspace }}/echo/build/run-echo-test-linux.ps1" + + - name: Log input parameters + shell: pwsh + run: | + Write-Output "Input parameters:" + Write-Output " SenderOptions: ${{ inputs.sender_options }}" + Write-Output " ReceiverOptions: ${{ inputs.receiver_options }}" + Write-Output " ref: ${{ inputs.ref }}" + Write-Output " Duration: ${{ inputs.duration }}" + + - name: Diagnostic - List files in echo build directory + shell: pwsh + run: | + Write-Output "Files in echo build directory:" + Get-ChildItem -Path "$env:GITHUB_WORKSPACE/echo/build" -Recurse + + - name: Run echo traffic tests (Linux) + working-directory: ${{ github.workspace }}/echo/build + env: + SENDER_OPTIONS: ${{ inputs.sender_options }} + RECEIVER_OPTIONS: ${{ inputs.receiver_options }} + shell: pwsh + run: | + $cmd = @( + './run-echo-test-linux.ps1', + '-PeerName', '"netperf-peer"', + '-SenderOptions', '"' + $env:SENDER_OPTIONS + '"', + '-ReceiverOptions', '"' + $env:RECEIVER_OPTIONS + '"', + '-Duration', '"${{ inputs.duration }}"' + ) -join ' ' + + Write-Output "Running: $cmd" + Invoke-Expression $cmd + + - name: Upload echo results (Linux) + if: always() + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 + with: + name: echo_test_linux_${{ matrix.vec.env }}_${{ matrix.vec.os }}_${{ matrix.vec.arch }} + path: | + ${{ github.workspace }}/echo/*.csv + ${{ github.workspace }}/echo/*.log + ${{ github.workspace }}/echo/*.json + ${{ github.workspace }}/echo/build/echo_summary.csv + ${{ github.workspace }}/echo/build/echo_client_output.txt + ${{ github.workspace }}/echo/build/server.log + + attempt-reset-lab-linux: + name: Attempting to reset lab (Linux). Status of this job does not indicate result of lab reset. Look at job details. + needs: [test_echo_server_linux] + if: ${{ always() }} + uses: microsoft/netperf/.github/workflows/schedule-lab-reset.yml@main + with: + workflowId: ${{ github.run_id }} From cc8bc439a1d28b8a93948f796d81ab0290803ad8 Mon Sep 17 00:00:00 2001 From: GitHub Copilot Date: Tue, 6 Jan 2026 17:24:20 -0800 Subject: [PATCH 70/70] Fix bash command quoting in run-echo-test-linux.ps1 --- .github/scripts/run-echo-test-linux.ps1 | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/.github/scripts/run-echo-test-linux.ps1 b/.github/scripts/run-echo-test-linux.ps1 index c5b191d9..328d0c0d 100644 --- a/.github/scripts/run-echo-test-linux.ps1 +++ b/.github/scripts/run-echo-test-linux.ps1 @@ -2,7 +2,9 @@ param( [string]$PeerName = "netperf-peer", [string]$SenderOptions, [string]$ReceiverOptions, - [string]$Duration = "60" + [string]$Duration = "60", + [string]$RemoteServerPath = "/tmp/echo_server", + [string]$RemoteServerLogPath = "/tmp/server.log" ) $ErrorActionPreference = 'Stop' @@ -30,14 +32,14 @@ try { } # Copy server binary to peer and ensure executable -Write-Host "Copying server binary to peer" -Copy-Item -Path $serverPath -Destination '~/echo_server' -ToSession $session -Invoke-Command -Session $session -ScriptBlock { chmod +x ~/echo_server } +Write-Host "Copying server binary to peer at $RemoteServerPath" +Copy-Item -Path $serverPath -Destination $RemoteServerPath -ToSession $session +Invoke-Command -Session $session -ScriptBlock { chmod +x $using:RemoteServerPath } # Start server in background on peer, capture PID Write-Host "Starting server on peer" $startServer = @" - bash -lc "nohup ~/echo_server $using:ReceiverOptions > ~/server.log 2>&1 & echo \$!" + bash -lc "nohup $using:RemoteServerPath $using:ReceiverOptions > $using:RemoteServerLogPath 2>&1 & echo \$!" "@ $serverPid = Invoke-Command -Session $session -ScriptBlock ([ScriptBlock]::Create($startServer)) Write-Host "Server PID on peer: $serverPid" @@ -45,7 +47,9 @@ Start-Sleep -Seconds 2 # Run client locally and capture output Write-Host "Running echo client locally" -$clientCmd = "bash -lc \"$clientPath $SenderOptions --duration $Duration\"" +$clientCmd = @" +bash -lc '$clientPath $SenderOptions --duration $Duration' +"@ $clientOutput = & pwsh -NoProfile -Command $clientCmd 2>&1 $clientExit = $LASTEXITCODE @@ -80,7 +84,7 @@ Start-Sleep -Seconds 1 Write-Host "Fetching server log from peer" try { - Copy-Item -FromSession $session -Path '~/server.log' -Destination 'server.log' + Copy-Item -FromSession $session -Path $RemoteServerLogPath -Destination 'server.log' } catch { Write-Host "Failed to copy server.log: $_" }