Skip to content

Commit fa20d81

Browse files
princepereiraprincepereira
committed
Network health script.
1 parent 7045e21 commit fa20d81

File tree

8 files changed

+2428
-0
lines changed

8 files changed

+2428
-0
lines changed

‎scripts/networkhealth/README.md‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,4 +35,30 @@ If the `-OutputMode` is set to `Event` or `All`, the script will register a new
3535
Example:
3636
kubectl logs -l name=networkhealth --all-containers=true >> networkhealth.txt
3737
Provide the generated networkhealth.txt
38+
```
39+
## Command to run startNetworkDiagnostics
40+
```
41+
Normal Execution: .\startNetworkDiagnostics.ps1 -TimeIntervalInSeconds 30 -PrintMatchedRules $true -PodNamePrefixes tcp-server,tcp-client
42+
43+
Execution with DNS Packet Capture: .\startNetworkDiagnostics.ps1 -DnsPktCap $true
44+
45+
Execution with DualStack Test: .\startNetworkDiagnostics.ps1 -DualStack $true
46+
47+
Execution with Vfp Rule Counter Dump for Pods : .\startNetworkDiagnostics.ps1 -PodNamePrefixes tcp-client,tcp-server
48+
49+
Execution with printing matched rule counter : .\startNetworkDiagnostics.ps1 -PodNamePrefixes tcp-client,tcp-server -PrintMatchedRules $true
50+
51+
Execution with validate loadbalancer rules for Service IPS : .\startNetworkDiagnostics.ps1 -ServiceIPS "10.0.0.1,10.0.0.2"
52+
53+
```
54+
55+
## Command to run vfpDropCounterMetrics
56+
```
57+
.\vfpDropCounterMetrics.ps1 -TimeIntervalInSeconds 30 -PrintMatchedRules $true -PodNamePrefixes tcp-server,tcp-client
58+
```
59+
60+
## DNS Health Check
61+
```
62+
Invoke-WebRequest https://raw.githubusercontent.com/microsoft/wcnscripts/2ea829ebaaf523cf58ef8e64120e54849eb4bd51/scripts/networkhealth/startNetworkDiagnostics.ps1 -OutFile startNetworkDiagnostics.ps1
63+
.\startNetworkDiagnostics.ps1
3864
```

‎scripts/networkhealth/RuleDump.ps1‎

Lines changed: 158 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,158 @@
1+
2+
param (
3+
[Parameter(Mandatory=$false, HelpMessage="Rule for Pod")][string]$podIp = "10.224.0.53",
4+
[Parameter(Mandatory=$false, HelpMessage="Rule for Pod")][string]$ipFamily = "IPV4",
5+
[Parameter(Mandatory=$false, HelpMessage="Rule for Pod")][string]$protoFamily = "TCP",
6+
[Parameter(Mandatory=$false, HelpMessage="Rule for Pod")][bool]$PrintMatchedRules = $true
7+
)
8+
9+
10+
$podMac = (Get-HnsEndpoint | Where-Object IPAddress -Eq $podIp).MacAddress
11+
$podMacShortened = $podMac.Replace("-", "")
12+
$PodPortId = ((vfpctrl /list-vmswitch-port /format 1 | ConvertFrom-Json).Ports | Where-Object MacAddress -EQ $podMacShortened).Name
13+
$ExtPortId = ((vfpctrl /list-vmswitch-port /format 1 | ConvertFrom-Json).Ports | Where-Object Id -EQ "ExternalPort").Name
14+
$HostPortId = ((vfpctrl /list-vmswitch-port /format 1 | ConvertFrom-Json).Ports | Where-Object Id -Like "Container NIC*").Name
15+
16+
$podLayers = ((vfpctrl /port $PodPortId /list-layer /format 1 | ConvertFrom-Json).Layers | Sort-Object -Property Priority).Name
17+
$extPortLayers = ((vfpctrl /port $ExtPortId /list-layer /format 1 | ConvertFrom-Json).Layers | Sort-Object -Property Priority).Name
18+
$hostPortLayers = ((vfpctrl /port $HostPortId /list-layer /format 1 | ConvertFrom-Json).Layers | Sort-Object -Property Priority).Name
19+
20+
function RemoveNoise() {
21+
param (
22+
[Parameter(Mandatory=$true)][System.Object]$Rule
23+
)
24+
$Rule.PSObject.Properties.Remove('$type')
25+
$Rule.PSObject.Properties.Remove('Type')
26+
$Rule.PSObject.Properties.Remove('SubType')
27+
$Rule.PSObject.Properties.Remove('MssDelta')
28+
$Rule.PSObject.Properties.Remove('ReverseMssDelta')
29+
$Rule.PSObject.Properties.Remove('RuleFlags')
30+
$Rule.PSObject.Properties.Remove('PaRouteRuleFlags')
31+
$Rule.PSObject.Properties.Remove('CachePruningThreshold')
32+
$Rule.PSObject.Properties.Remove('InformationArray')
33+
$Rule.PSObject.Properties.Remove('NumHeaders')
34+
$Rule.PSObject.Properties.Remove('PartialRewriteTypes')
35+
return $Rule
36+
}
37+
38+
function LogVfpCounter {
39+
param (
40+
[parameter(Mandatory=$false)][string] $value = "",
41+
[parameter(Mandatory=$false)][bool] $error = $false
42+
)
43+
# Add-Content -Path $VfpRuleFile -Value $value
44+
if ($error -eq $true) {
45+
Write-Host $value -ForegroundColor Red
46+
}
47+
else {
48+
Write-Host $value
49+
}
50+
51+
}
52+
53+
function NewLine {
54+
param (
55+
[parameter(Mandatory=$false)][int] $NoOfLines = 1
56+
)
57+
for ($i = 1; $i -le $NoOfLines; $i++) {
58+
LogVfpCounter ""
59+
}
60+
}
61+
62+
function PrintRules {
63+
param (
64+
[Parameter(Mandatory=$true)][string[]]$Layers,
65+
[Parameter(Mandatory=$true)][string]$PortId,
66+
[Parameter(Mandatory=$false)][string]$Dir="OUT"
67+
)
68+
69+
$ruleCounterMap = @{}
70+
71+
foreach($layer in $Layers) {
72+
$groups = ((vfpctrl /port $PortId /layer $layer /list-group /format 1 | ConvertFrom-Json).Groups | Sort-Object -Property Priority).Name
73+
foreach($group in $groups) {
74+
if($group.Contains("_$DIR") -ne $true) {
75+
continue
76+
}
77+
if(($group.Contains("IPV4") -eq $true) -or ($group.Contains("IPV6") -eq $true)) {
78+
if($group.Contains($ipFamily) -ne $true) {
79+
continue
80+
}
81+
}
82+
if(($group.Contains("TCP") -eq $true) -or ($group.Contains("UDP") -eq $true) -or ($group.Contains("ICMP") -eq $true)) {
83+
if($group.Contains($protoFamily) -ne $true) {
84+
continue
85+
}
86+
}
87+
$rules = (vfpctrl /port $PortId /layer $layer /group $group /get-rule-counter /format 1 | ConvertFrom-Json | Sort-Object -Property Priority).Rules
88+
foreach ($rule in $rules) {
89+
90+
$ruleId = $rule.Name
91+
if (($rule.Id).Length -gt 0) {
92+
$ruleId = $rule.Id
93+
}
94+
95+
$ruleKey = "$portId-$layer-$group-$ruleId"
96+
97+
$informationArray = $rule.InformationArray
98+
99+
$rule = RemoveNoise -Rule $rule
100+
101+
if ($informationArray.Count -gt 0) {
102+
$rule | Add-Member -MemberType NoteProperty -Name RuleCounters -Value $informationArray[0].RuleCounters
103+
}
104+
105+
$ruleJson = $rule | ConvertTo-Json -Depth 10
106+
107+
if ($informationArray.Count -gt 0) {
108+
109+
$ruleCounters = $informationArray[0].RuleCounters
110+
$matchedPackets = $ruleCounters.MatchedPackets
111+
$droppedPackets = $ruleCounters.DroppedPackets
112+
$pendingPackets = $ruleCounters.PendingPackets
113+
$droppedFlows = $ruleCounters.DroppedFlows
114+
115+
if (($droppedPackets -gt 0) -or ($pendingPackets -gt 0) -or ($droppedFlows -gt 0)) {
116+
LogVfpCounter " Dropped Rule : " -error $true
117+
LogVfpCounter " ================ " -error $true
118+
NewLine 1
119+
LogVfpCounter " Layer : $layer , Group : $group , Id : $ruleId " -error $true
120+
NewLine 1
121+
LogVfpCounter " $ruleJson " -error $true
122+
NewLine 2
123+
}
124+
elseif (($PrintMatchedRules -eq $true) -and ($matchedPackets -gt 0)) {
125+
LogVfpCounter " Matched Rule : "
126+
LogVfpCounter " ================ "
127+
NewLine 1
128+
LogVfpCounter " Layer : $layer , Group : $group , Id : $ruleId "
129+
NewLine 1
130+
LogVfpCounter " $ruleJson "
131+
132+
NewLine 2
133+
}
134+
135+
$ruleCounterMap[$ruleKey] = $rule
136+
}
137+
}
138+
}
139+
}
140+
141+
return $ruleCounterMap
142+
}
143+
144+
Write-Host "#===================== Pod VFP Port Rules in Outbound Direction ================#"
145+
NewLine 2
146+
$podPortRulesOutbound = PrintRules -Layers $podLayers -portId $PodPortId -Dir "OUT"
147+
NewLine 2
148+
Write-Host "#===================== External VFP Port Rules in Outbound Direction ================#"
149+
NewLine 2
150+
$extPortRulesOutbound = PrintRules -Layers $extPortLayers -portId $ExtPortId -Dir "OUT"
151+
NewLine 2
152+
Write-Host "#===================== External VFP Port Rules in Inbound Direction ================#"
153+
NewLine 2
154+
$extPortRulesIntbound = PrintRules -Layers $extPortLayers -portId $ExtPortId -Dir "IN"
155+
NewLine 2
156+
Write-Host "#===================== Pod VFP Port Rules in Inbound Direction ================#"
157+
NewLine 2
158+
$podPortRulesIutbound = PrintRules -Layers $podLayers -portId $PodPortId -Dir "IN"

‎scripts/networkhealth/checkDupLbDsrRules.ps1‎

Lines changed: 152 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,152 @@
1+
$LogDirPrefix = "C:\k\debug\NetworkHealth\Log-"
2+
$timeNowInUtc = Get-Date -Format u
3+
$LogDirSuffix = $timeNowInUtc.Replace(" ", "-").Replace(":", "-")
4+
$LogsDir = $LogDirPrefix + $LogDirSuffix
5+
$Logfile = "$LogsDir\health.log"
6+
7+
mkdir $LogsDir -ErrorAction Ignore
8+
9+
function LogError {
10+
param (
11+
[parameter(Mandatory=$true)][string] $message
12+
)
13+
Write-Host $message -ForegroundColor Red
14+
Add-content $Logfile -value "[FAILED] $message"
15+
}
16+
17+
function LogSuccess {
18+
param (
19+
[parameter(Mandatory=$true)][string] $message
20+
)
21+
Write-Host $message -ForegroundColor Green
22+
Add-content $Logfile -value "[SUCCESS] $message"
23+
}
24+
25+
function readVfpPortIdList() {
26+
$vfpPortIds = ((vfpctrl /list-vmswitch-port /format 1 | ConvertFrom-Json).Ports).Name
27+
return $vfpPortIds
28+
}
29+
30+
function getDuplicateLbDsrVfpRules() {
31+
param (
32+
[parameter(Mandatory=$true)][string] $portId,
33+
[parameter(Mandatory=$true)][string] $groupId
34+
)
35+
36+
$vfpRules = ((vfpctrl /port $portId /layer LB_DSR /group $groupId /get-rule-counter /format 1 | ConvertFrom-Json).Rules) | Select-Object ID, Name, Conditions
37+
38+
$vfpRuleMap = @{}
39+
40+
foreach($rule in $vfpRules) {
41+
$conditions = $rule.Conditions
42+
$protocolList = ""
43+
$ipList = ""
44+
$portList = ""
45+
foreach($condition in $conditions) {
46+
$type = $condition.ConditionType
47+
if($type -eq 1) {
48+
# Protocol List
49+
$protocols = $condition.ProtocolList
50+
foreach($protocol in $protocols) {
51+
$protocolList += "$protocol" + "-"
52+
}
53+
}
54+
if($type -eq 11) {
55+
# IPV4 Range List
56+
$ips = $condition.DestinationIPv4RangeList
57+
foreach($ip in $ips) {
58+
if($null -ne $ip) {
59+
if($ip.L -eq $ip.H) {
60+
$ipList += $ip.L + "-"
61+
} else {
62+
$ipList += $ip.L + "-" + $ip.H + "-"
63+
}
64+
}
65+
}
66+
}
67+
if($type -eq 13) {
68+
# IPV6 Range List
69+
$ips = $condition.DestinationIPv6RangeList
70+
foreach($ip in $ips) {
71+
if($null -ne $ip) {
72+
if($ip.L -eq $ip.H) {
73+
$ipList += $ip.L + "-"
74+
} else {
75+
$ipList += $ip.L + "-" + $ip.H + "-"
76+
}
77+
}
78+
}
79+
}
80+
if($type -eq 5) {
81+
# Port Range List
82+
$ports = $condition.DestinationPortList
83+
foreach($port in $ports) {
84+
$portList += "$port" + "-"
85+
}
86+
}
87+
}
88+
89+
$key = $ipList + $portList + $protocolList
90+
if($key.Length -gt 5) {
91+
$key = $key.Substring(0, $key.Length-1)
92+
}
93+
94+
$ruleId = $rule.ID
95+
if(($null -eq $ruleId) -OR ("" -EQ $ruleId) -or ($ruleId.Length -lt 5)) {
96+
$ruleId = $rule.Name
97+
}
98+
99+
$existingList = $vfpRuleMap[$key]
100+
if($null -eq $existingList) {
101+
$existingList = @($ruleId)
102+
} else {
103+
$existingList += $ruleId
104+
}
105+
106+
$vfpRuleMap[$key] = $existingList
107+
}
108+
109+
$refinedDuplicateIds = @{}
110+
111+
foreach($e in $vfpRuleMap.GetEnumerator()) {
112+
if(($e.Value).Count -gt 1) {
113+
$refinedDuplicateIds["Condition-" + $e.Key] = $e.Value
114+
}
115+
}
116+
117+
return $refinedDuplicateIds
118+
}
119+
120+
function CheckDuplicate_LB_DSR_VfpRules {
121+
122+
Write-Host "Checking Duplicate LB DSR VFP Rules"
123+
124+
$vfpPortIds = readVfpPortIdList
125+
$duplicateIdAndPorts = @{}
126+
127+
foreach($portId in $vfpPortIds) {
128+
129+
$v4DuplicateIds = getDuplicateLbDsrVfpRules -portId $portId -groupId "LB_DSR_IPv4_OUT"
130+
$v6DuplicateIds = getDuplicateLbDsrVfpRules -portId $portId -groupId "LB_DSR_IPv6_OUT"
131+
132+
$duplicateIds = $v4DuplicateIds + $v6DuplicateIds
133+
134+
if($duplicateIds.Count -gt 0) {
135+
$duplicateIdAndPorts["Port-" + $portId] = $duplicateIds
136+
}
137+
138+
}
139+
140+
if($duplicateIdAndPorts.Count -gt 0){
141+
LogError "Duplicate LB DSR VFP Rules Present. "
142+
$jsonOutput = $duplicateIdAndPorts | ConvertTo-Json -Depth 10
143+
LogError "Info: $jsonOutput "
144+
LogError "Mitigation : Restart-Service -f hns ; Restart-Service -f kubeproxy "
145+
return $true
146+
}
147+
148+
LogSuccess "There is no Duplicate LB DSR VFP Rules."
149+
return $false
150+
}
151+
152+
CheckDuplicate_LB_DSR_VfpRules

0 commit comments

Comments
 (0)