diff --git a/Diagnostics/Test-Rdma.ps1 b/Diagnostics/Test-Rdma.ps1 index b8a3fead..d0ed98cb 100644 --- a/Diagnostics/Test-Rdma.ps1 +++ b/Diagnostics/Test-Rdma.ps1 @@ -1,46 +1,39 @@ [CmdletBinding()] -Param( - [Parameter(Mandatory=$True, Position=1, HelpMessage="Interface index of the adapter for which RDMA config is to be verified")] - [string] $IfIndex, - [Parameter(Mandatory=$True, Position=2, HelpMessage="True if underlying fabric type is RoCE. False for iWarp or IB")] - [bool] $IsRoCE, - [Parameter(Mandatory=$True, Position=3, HelpMessage="IP address of the remote RDMA adapter")] - [string] $RemoteIpAddress, - [Parameter(Mandatory=$False, Position=4, HelpMessage="Full path to the folder containing diskspd.exe")] - [string] $PathToDiskspd, - [Parameter(Mandatory=$False, Position=5, HelpMessage="Output level [none|verbose|debug]")] - [string] $OutputLevel, - [Parameter(Mandatory=$False, Position=6, HelpMessage="Interface ID of VF driver in Guest OS (mandatory for Guest RDMA tests only)")] - [string] $VfIndex) - -if ($OutputLevel -eq "none") -{ +param( + [Parameter(Mandatory = $True, Position = 1, HelpMessage = "Interface index of the adapter for which RDMA config is to be verified")] + [string] $IfIndex, + [Parameter(Mandatory = $True, Position = 2, HelpMessage = "True if underlying fabric type is RoCE. False for iWarp or IB")] + [bool] $IsRoCE, + [Parameter(Mandatory = $True, Position = 3, HelpMessage = "IP address of the remote RDMA adapter")] + [string] $RemoteIpAddress, + [Parameter(Mandatory = $False, Position = 4, HelpMessage = "Full path to the folder containing diskspd.exe")] + [string] $PathToDiskspd, + [Parameter(Mandatory = $False, Position = 5, HelpMessage = "Output level [none|verbose|debug]")] + [string] $OutputLevel, + [Parameter(Mandatory = $False, Position = 6, HelpMessage = "Interface ID of VF driver in Guest OS (mandatory for Guest RDMA tests only)")] + [string] $VfIndex) + +if ($OutputLevel -eq "none") { $OutputLevel = 0 } -elseif ($OutputLevel -eq "debug" ) -{ +elseif ($OutputLevel -eq "debug" ) { $OutputLevel = 2 } -else -{ +else { $OutputLevel = 1 # verbose } -if ($RemoteIpAddress -ne $null) -{ - if (($PathToDiskspd -eq $null) -Or ($PathToDiskspd -eq '')) - { +if ($RemoteIpAddress -ne $null) { + if (($PathToDiskspd -eq $null) -or ($PathToDiskspd -eq '')) { $PathToDiskspd = "C:\Windows\System32" } $FullPathToDiskspd = $PathToDiskspd + "\diskspd.exe" - if ((Test-Path $FullPathToDiskspd) -eq $false) - { + if ((Test-Path $FullPathToDiskspd) -eq $false) { Write-Host "ERROR: Diskspd.exe not found at" $FullPathToDiskspd ". Please download diskspd.exe and place it in the specified location. Exiting." -ForegroundColor Red return } - elseif ($OutputLevel -gt 0) - { + elseif ($OutputLevel -gt 0) { Write-Host "VERBOSE: Diskspd.exe found at" $FullPathToDiskspd } } @@ -48,128 +41,105 @@ if ($RemoteIpAddress -ne $null) $rdmaAdapter = Get-NetAdapter -IfIndex $IfIndex -if ($outputLevel -eq 2) -{ +if ($outputLevel -eq 2) { Write-Host "DEBUG: Name is " $rdmaAdapter.Name Write-Host "DEBUG: IfDesc is " $rdmaAdapter.InterfaceDescription } -if ($rdmaAdapter -eq $null) -{ +if ($rdmaAdapter -eq $null) { Write-Host "ERROR: The adapter with interface index $IfIndex not found" -ForegroundColor Red return } $rdmaAdapterName = $rdmaAdapter.Name -if ($rdmaAdapter.InterfaceDescription -Match "Hyper-V Virtual Ethernet Adapter") -{ +if ($rdmaAdapter.InterfaceDescription -match "Hyper-V Virtual Ethernet Adapter") { $rdmaAdapterType = "vNIC" } -elseif ($rdmaAdapter.InterfaceDescription -Match "Microsoft Hyper-V Network Adapter") -{ +elseif ($rdmaAdapter.InterfaceDescription -match "Microsoft Hyper-V Network Adapter") { $rdmaAdapterType = "vmNIC" - if( $VfIndex -eq "" ) - { - Write-Host "ERROR: VF adapter Interface Index missing" -ForegroundColor Red - return + if ( $VfIndex -eq "" ) { + Write-Host "ERROR: VF adapter Interface Index missing" -ForegroundColor Red + return } $VFAdapter = Get-NetAdapter -IfIndex $VfIndex - if ($outputLevel -eq 2) - { + if ($outputLevel -eq 2) { Write-Host "DEBUG: VF Name is " $VFAdapter.Name Write-Host "DEBUG: IfDesc of VF is " $VFAdapter.InterfaceDescription } $VFrdmaCapabilities = Get-NetAdapterRdma -InterfaceDescription $VFAdapter.InterfaceDescription } -else -{ - $rdmaAdapterType = "pNIC" +else { + $rdmaAdapterType = "pNIC" } -if ($outputLevel -gt 0) -{ +if ($outputLevel -gt 0) { Write-Host "VERBOSE: The adapter " $rdmaAdapterName " is a " $rdmaAdapterType } $rdmaCapabilities = Get-NetAdapterRdma -InterfaceDescription $rdmaAdapter.InterfaceDescription -if ($rdmaCapabilities -eq $null -or $rdmaCapabilities.Enabled -eq $false) -{ +if ($rdmaCapabilities -eq $null -or $rdmaCapabilities.Enabled -eq $false) { Write-Host "ERROR: The adapter " $rdmaAdapterName " is not enabled for RDMA" -ForegroundColor Red return } -if ($rdmaAdapterType -eq "vmNIC" -and ( $VFrdmaCapabilities -eq $null -or $VFrdmaCapabilities.Enabled -eq $false )) -{ +if ($rdmaAdapterType -eq "vmNIC" -and ( $VFrdmaCapabilities -eq $null -or $VFrdmaCapabilities.Enabled -eq $false )) { Write-Host "ERROR: The VF adapter " $VFAdapter.Name " is not enabled for RDMA" -ForegroundColor Red return } -if ($rdmaCapabilities.MaxQueuePairCount -eq 0) -{ +if ($rdmaCapabilities.MaxQueuePairCount -eq 0) { Write-Host "ERROR: RDMA capabilities for adapter $rdmaAdapterName are not valid : MaxQueuePairCount is 0" -ForegroundColor Red return } -if ($rdmaCapabilities.MaxCompletionQueueCount -eq 0) -{ +if ($rdmaCapabilities.MaxCompletionQueueCount -eq 0) { Write-Host "ERROR: RDMA capabilities for adapter $rdmaAdapterName are not valid : MaxCompletionQueueCount is 0" -ForegroundColor Red return } $smbClientNetworkInterfaces = Get-SmbClientNetworkInterface -if ($smbClientNetworkInterfaces -eq $null) -{ +if ($smbClientNetworkInterfaces -eq $null) { Write-Host "ERROR: No network interfaces detected by SMB (Get-SmbClientNetworkInterface)" -ForegroundColor Red return } $rdmaAdapterSmbClientNetworkInterface = $null -foreach ($smbClientNetworkInterface in $smbClientNetworkInterfaces) -{ - if ($smbClientNetworkInterface.InterfaceIndex -eq $IfIndex) - { +foreach ($smbClientNetworkInterface in $smbClientNetworkInterfaces) { + if ($smbClientNetworkInterface.InterfaceIndex -eq $IfIndex) { $rdmaAdapterSmbClientNetworkInterface = $smbClientNetworkInterface } } -if ($rdmaAdapterSmbClientNetworkInterface -eq $null) -{ +if ($rdmaAdapterSmbClientNetworkInterface -eq $null) { Write-Host "ERROR: No network interfaces found by SMB for adapter $rdmaAdapterName (Get-SmbClientNetworkInterface)" -ForegroundColor Red return } -if ($rdmaAdapterSmbClientNetworkInterface.RdmaCapable -eq $false) -{ +if ($rdmaAdapterSmbClientNetworkInterface.RdmaCapable -eq $false) { Write-Host "ERROR: SMB did not detect adapter $rdmaAdapterName as RDMA capable. Make sure the adapter is bound to TCP/IP and not to other protocol like vmSwitch." -ForegroundColor Red return } $rdmaAdapters = $rdmaAdapter -if ($RdmaAdapterType -eq "vNIC") -{ - if ($OutputLevel -gt 0) - { +if ($RdmaAdapterType -eq "vNIC") { + if ($OutputLevel -gt 0) { Write-Host "VERBOSE: Retrieving vSwitch bound to the virtual adapter" } - $virtualAdapter = Get-VMNetworkAdapter -ManagementOS | where DeviceId -eq $rdmaAdapter.DeviceID + $virtualAdapter = Get-VMNetworkAdapter -ManagementOS | where DeviceId -EQ $rdmaAdapter.DeviceID $switchName = $virtualAdapter.switchName - if ($OutputLevel -gt 0) - { + if ($OutputLevel -gt 0) { Write-Host "VERBOSE: Found vSwitch: $switchName" } $vSwitch = Get-VMSwitch -Name $switchName $rdmaAdapters = Get-NetAdapter -InterfaceDescription $vSwitch.NetAdapterInterfaceDescriptions - if ($OutputLevel -gt 0) - { + if ($OutputLevel -gt 0) { $vSwitchAdapterMessage = "VERBOSE: Found the following physical adapter(s) bound to vSwitch: " $index = 1 - foreach ($qosAdapter in $rdmaAdapters) - { + foreach ($qosAdapter in $rdmaAdapters) { $qosAdapterName = $qosAdapter.Name $vSwitchAdapterMessage = $vSwitchAdapterMessage + [string]$qosAdapterName - if ($index -lt $rdmaAdapters.Length) - { + if ($index -lt $rdmaAdapters.Length) { $vSwitchAdapterMessage = $vSwitchAdapterMessage + ", " } $index = $index + 1 @@ -179,101 +149,81 @@ if ($RdmaAdapterType -eq "vNIC") } -if ($IsRoCE -eq $true -and $RdmaAdapterType -ne "vmNIC") -{ +if ($IsRoCE -eq $true -and $RdmaAdapterType -ne "vmNIC") { Write-Host "VERBOSE: Underlying adapter is RoCE. Checking if QoS/DCB/PFC is configured on each physical adapter(s)" - foreach ($qosAdapter in $rdmaAdapters) - { + foreach ($qosAdapter in $rdmaAdapters) { $qosAdapterName = $qosAdapter.Name $qos = Get-NetAdapterQos -Name $qosAdapterName - if ($qos.Enabled -eq $false) - { + if ($qos.Enabled -eq $false) { Write-Host "ERROR: QoS is not enabled for adapter $qosAdapterName" -ForegroundColor Red return } - if ($qos.OperationalFlowControl -eq "All Priorities Disabled") - { + if ($qos.OperationalFlowControl -eq "All Priorities Disabled") { Write-Host "ERROR: Flow control is not enabled for adapter $qosAdapterName" -ForegroundColor Red return } } - if ($OutputLevel -gt 0) - { + if ($OutputLevel -gt 0) { Write-Host "VERBOSE: QoS/DCB/PFC configuration is correct." } } -if ($RdmaAdapterType -eq "vmNIC") -{ +if ($RdmaAdapterType -eq "vmNIC") { Write-Host "CAUTION: Guest Virtual NIC being tested, Guest can't check host adapter settings." -ForegroundColor Yellow } -elseif ($OutputLevel -gt 0) -{ +elseif ($OutputLevel -gt 0) { Write-Host "VERBOSE: RDMA configuration is correct." } -if ($RemoteIpAddress -ne '') -{ - if ($OutputLevel -eq 2) - { +if ($RemoteIpAddress -ne '') { + if ($OutputLevel -eq 2) { Write-Host "DEBUG: Checking if remote IP address, $RemoteIpAddress, is reachable." } $canPing = Test-Connection $RemoteIpAddress -Quiet - if ($canPing -eq $false) - { + if ($canPing -eq $false) { Write-Host "ERROR: Cannot reach remote IP $RemoteIpAddress" -ForegroundColor Red return } - elseif ($OutputLevel -gt 0) - { + elseif ($OutputLevel -gt 0) { Write-Host "VERBOSE: Remote IP $RemoteIpAddress is reachable." } } -else -{ +else { Write-Host "ERROR: Remote IP address was not provided." } -if ($OutputLevel -gt 0) -{ +if ($OutputLevel -gt 0) { Write-Host "VERBOSE: Disabling RDMA on adapters that are not part of this test. RDMA will be enabled on them later." } $adapters = Get-NetAdapterRdma $InstanceIds = $rdmaAdapters.InstanceID; $adaptersToEnableRdma = @() -foreach ($adapter in $adapters) -{ - if ($adapter.Enabled -eq $true) - { +foreach ($adapter in $adapters) { + if ($adapter.Enabled -eq $true) { if (($adapter.InstanceID -notin $InstanceIds) -and - ($adapter.InstanceID -ne $rdmaAdapter.InstanceID) -and - ($adapter.InstanceID -ne $VFAdapter.InstanceID)) - { + ($adapter.InstanceID -ne $rdmaAdapter.InstanceID) -and + ($adapter.InstanceID -ne $VFAdapter.InstanceID)) { $adaptersToEnableRdma += $adapter Disable-NetAdapterRdma -Name $adapter.Name - if ($OutputLevel -eq 2) - { + if ($OutputLevel -eq 2) { Write-Host "DEBUG: RDMA disabled on Adapter " $adapter.Name } } - elseif ($OutputLevel -eq 2) - { + elseif ($OutputLevel -eq 2) { Write-Host "DEBUG: RDMA not disabled on Adapter " $adapter.Name - } + } } } -if ($OutputLevel -ne 0) -{ +if ($OutputLevel -ne 0) { Write-Host "VERBOSE: Testing RDMA traffic. Traffic will be sent in a background job. Job details:" } # PseudoRandomize the target file name so two copies of this script can execute at the same time against the same destination $TargetFileName = "\\$RemoteIpAddress\C$\testfile$IfIndex.dat" -if ($OutputLevel -eq 2) -{ +if ($OutputLevel -eq 2) { Write-Host "DEBUG: TargetFileName is $TargetFileName" } @@ -283,62 +233,81 @@ $ScriptBlock = { .\diskspd.exe -b4K -c10G -t4 -o16 -d100000 -L -Sr -d30 $TargetFileName } -$thisJob = Start-Job $ScriptBlock -ArgumentList $RemoteIpAddress,$PathToDiskspd,$TargetFileName +$thisJob = Start-Job $ScriptBlock -ArgumentList $RemoteIpAddress, $PathToDiskspd, $TargetFileName $RdmaTrafficDetected = $false # Check Perfmon counters while the job is running -While ((Get-Job -id $($thisJob).Id).state -eq "Running") -{ - $written = Get-Counter -Counter "\SMB Direct Connection(_Total)\Bytes RDMA Written/sec" -ErrorAction Ignore - $sent = Get-Counter -Counter "\SMB Direct Connection(_Total)\Bytes Sent/sec" -ErrorAction Ignore - if ($written -ne $null) - { +while ((Get-Job -Id $($thisJob).Id).state -eq "Running") { + try { + # WS2025 - attempt to get the counter for "SMB Direct" + $written = Get-Counter -Counter "\\$RemoteIpAddress\SMB Direct\Bytes RDMA Written/sec" -ErrorAction Stop + $sent = Get-Counter -Counter "\SMB Direct\Bytes Sent/sec" -ErrorAction Stop + if ($written -ne $null) { $RdmaWriteBytesPerSecond = [uint64]($written.Readings.split(":")[1]) - if ($RdmaWriteBytesPerSecond -gt 0) - { + if ($RdmaWriteBytesPerSecond -gt 0) { $RdmaTrafficDetected = $true } - if ($OutputLevel -gt 0) - { + if ($OutputLevel -gt 0) { Write-Host "VERBOSE:" $RdmaWriteBytesPerSecond "RDMA bytes written per second" } } - if ($sent -ne $null) - { + if ($sent -ne $null) { $RdmaWriteBytesPerSecond = [uint64]($sent.Readings.split(":")[1]) - if ($RdmaWriteBytesPerSecond -gt 0) - { + if ($RdmaWriteBytesPerSecond -gt 0) { $RdmaTrafficDetected = $true } - if ($OutputLevel -gt 0) - { + if ($OutputLevel -gt 0) { Write-Host "VERBOSE:" $RdmaWriteBytesPerSecond "RDMA bytes sent per second" } } + } + catch { + try { + # WS2022 - attempt to get the counter for "SMB Direct Connection" + $written = Get-Counter -Counter "\\$RemoteIpAddress\SMB Direct Connection(_Total)\Bytes RDMA Written/sec" -ErrorAction Stop + $sent = Get-Counter -Counter "\SMB Direct Connection(_Total)\Bytes Sent/sec" -ErrorAction Stop + if ($written -ne $null) { + $RdmaWriteBytesPerSecond = [uint64]($written.Readings.split(":")[1]) + if ($RdmaWriteBytesPerSecond -gt 0) { + $RdmaTrafficDetected = $true + } + if ($OutputLevel -gt 0) { + Write-Host "VERBOSE:" $RdmaWriteBytesPerSecond "RDMA bytes written per second" + } + } + if ($sent -ne $null) { + $RdmaWriteBytesPerSecond = [uint64]($sent.Readings.split(":")[1]) + if ($RdmaWriteBytesPerSecond -gt 0) { + $RdmaTrafficDetected = $true + } + if ($OutputLevel -gt 0) { + Write-Host "VERBOSE:" $RdmaWriteBytesPerSecond "RDMA bytes sent per second" + } + } + } + catch { + Write-Host "SMB Direct Connection counters not found." + } + } } del $TargetFileName -if ($OutputLevel -gt 0) -{ +if ($OutputLevel -gt 0) { Write-Host "VERBOSE: Enabling RDMA on adapters that are not part of this test. RDMA was disabled on them prior to sending RDMA traffic." } -foreach ($adapter in $adaptersToEnableRdma) -{ +foreach ($adapter in $adaptersToEnableRdma) { Enable-NetAdapterRdma -Name $adapter.Name - if ($OutputLevel -eq 2) - { + if ($OutputLevel -eq 2) { Write-Host "DEBUG: RDMA enabled on Adapter " $adapter.Name } } -if ($RdmaTrafficDetected) -{ +if ($RdmaTrafficDetected) { Write-Host "SUCCESS: RDMA traffic test SUCCESSFUL: RDMA traffic was sent to" $RemoteIpAddress -ForegroundColor Green } -else -{ +else { Write-Host "ERROR: RDMA traffic test FAILED: Please check " -ForegroundColor Yellow Write-Host "ERROR: a) physical switch port configuration for Priorty Flow Control." -ForegroundColor Yellow Write-Host "ERROR: b) job owner has write permission at " $RemoteIpAddress "\C$" -ForegroundColor Yellow -} \ No newline at end of file +}