Skip to content

Commit 129d1ef

Browse files
committed
Add collector for SR-IOV network Virtual Function statistics
Add a new netvf collector that exposes SR-IOV network VF statistics and configuration via rtnetlink. The collector queries netlink for interfaces with Virtual Functions and exposes per-VF metrics: - node_net_vf_info: VF configuration (MAC, VLAN, link state, spoof check, trust, PCI address, NUMA node) - node_net_vf_{receive,transmit}_{packets,bytes}_total: traffic counters - node_net_vf_{broadcast,multicast}_packets_total: packet type counters - node_net_vf_{receive,transmit}_dropped_total: drop counters All metrics include a pci_address label resolved from the sysfs virtfn symlink, enabling direct correlation with workloads that reference VFs by PCI BDF address (e.g. OpenStack Nova, libvirt, DPDK). All metrics also include a numa_node label resolved from the PF's PCI device sysfs entry, enabling NUMA alignment verification and cross-NUMA traffic ratio queries in PromQL. The collector is disabled by default and can be enabled with --collector.netvf. PF device filtering is supported via --collector.netvf.device-include/exclude flags. Signed-off-by: Anthony Harivel <aharivel@redhat.com>
1 parent a1cbf81 commit 129d1ef

5 files changed

Lines changed: 649 additions & 2 deletions

File tree

README.md

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -106,6 +106,7 @@ hwmon | chip | --collector.hwmon.chip-include | --collector.hwmon.chip-exclude
106106
hwmon | sensor | --collector.hwmon.sensor-include | --collector.hwmon.sensor-exclude
107107
interrupts | name | --collector.interrupts.name-include | --collector.interrupts.name-exclude
108108
netdev | device | --collector.netdev.device-include | --collector.netdev.device-exclude
109+
netvf | device | --collector.netvf.device-include | --collector.netvf.device-exclude
109110
qdisk | device | --collector.qdisk.device-include | --collector.qdisk.device-exclude
110111
slabinfo | slab-names | --collector.slabinfo.slabs-include | --collector.slabinfo.slabs-exclude
111112
sysctl | all | --collector.sysctl.include | N/A
@@ -202,6 +203,7 @@ logind | Exposes session counts from [logind](http://www.freedesktop.org/wiki/So
202203
meminfo\_numa | Exposes memory statistics from `/sys/devices/system/node/node[0-9]*/meminfo`, `/sys/devices/system/node/node[0-9]*/numastat`. | Linux
203204
mountstats | Exposes filesystem statistics from `/proc/self/mountstats`. Exposes detailed NFS client statistics. | Linux
204205
network_route | Exposes the routing table as metrics | Linux
206+
netvf | Exposes SR-IOV Virtual Function statistics and configuration from netlink. | Linux
205207
pcidevice | Exposes pci devices' information including their link status and parent devices. | Linux
206208
perf | Exposes perf based metrics (Warning: Metrics are dependent on kernel configuration and settings). | Linux
207209
processes | Exposes aggregate process statistics from `/proc`. | Linux

collector/netvf_linux.go

Lines changed: 227 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,227 @@
1+
// Copyright The Prometheus Authors
2+
// Licensed under the Apache License, Version 2.0 (the "License");
3+
// you may not use this file except in compliance with the License.
4+
// You may obtain a copy of the License at
5+
//
6+
// http://www.apache.org/licenses/LICENSE-2.0
7+
//
8+
// Unless required by applicable law or agreed to in writing, software
9+
// distributed under the License is distributed on an "AS IS" BASIS,
10+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
11+
// See the License for the specific language governing permissions and
12+
// limitations under the License.
13+
14+
//go:build !nonetvf
15+
16+
package collector
17+
18+
import (
19+
"errors"
20+
"fmt"
21+
"log/slog"
22+
23+
"github.com/alecthomas/kingpin/v2"
24+
"github.com/jsimonetti/rtnetlink/v2"
25+
"github.com/prometheus/client_golang/prometheus"
26+
"github.com/prometheus/procfs/sysfs"
27+
)
28+
29+
const netvfSubsystem = "net_vf"
30+
31+
var (
32+
netvfDeviceInclude = kingpin.Flag("collector.netvf.device-include", "Regexp of PF devices to include (mutually exclusive to device-exclude).").String()
33+
netvfDeviceExclude = kingpin.Flag("collector.netvf.device-exclude", "Regexp of PF devices to exclude (mutually exclusive to device-include).").String()
34+
)
35+
36+
func init() {
37+
registerCollector("netvf", defaultDisabled, NewNetVFCollector)
38+
}
39+
40+
type netvfCollector struct {
41+
logger *slog.Logger
42+
deviceFilter deviceFilter
43+
44+
info *prometheus.Desc
45+
receivePackets *prometheus.Desc
46+
transmitPackets *prometheus.Desc
47+
receiveBytes *prometheus.Desc
48+
transmitBytes *prometheus.Desc
49+
broadcast *prometheus.Desc
50+
multicast *prometheus.Desc
51+
receiveDropped *prometheus.Desc
52+
transmitDropped *prometheus.Desc
53+
}
54+
55+
func NewNetVFCollector(logger *slog.Logger) (Collector, error) {
56+
if *netvfDeviceExclude != "" && *netvfDeviceInclude != "" {
57+
return nil, errors.New("device-exclude & device-include are mutually exclusive")
58+
}
59+
60+
if *netvfDeviceExclude != "" {
61+
logger.Info("Parsed flag --collector.netvf.device-exclude", "flag", *netvfDeviceExclude)
62+
}
63+
64+
if *netvfDeviceInclude != "" {
65+
logger.Info("Parsed flag --collector.netvf.device-include", "flag", *netvfDeviceInclude)
66+
}
67+
68+
return &netvfCollector{
69+
logger: logger,
70+
deviceFilter: newDeviceFilter(*netvfDeviceExclude, *netvfDeviceInclude),
71+
info: prometheus.NewDesc(
72+
prometheus.BuildFQName(namespace, netvfSubsystem, "info"),
73+
"Virtual Function configuration information.",
74+
[]string{"device", "vf", "mac", "vlan", "link_state", "spoof_check", "trust", "pci_address", "numa_node"}, nil,
75+
),
76+
receivePackets: prometheus.NewDesc(
77+
prometheus.BuildFQName(namespace, netvfSubsystem, "receive_packets_total"),
78+
"Number of received packets by the VF.",
79+
[]string{"device", "vf", "pci_address", "numa_node"}, nil,
80+
),
81+
transmitPackets: prometheus.NewDesc(
82+
prometheus.BuildFQName(namespace, netvfSubsystem, "transmit_packets_total"),
83+
"Number of transmitted packets by the VF.",
84+
[]string{"device", "vf", "pci_address", "numa_node"}, nil,
85+
),
86+
receiveBytes: prometheus.NewDesc(
87+
prometheus.BuildFQName(namespace, netvfSubsystem, "receive_bytes_total"),
88+
"Number of received bytes by the VF.",
89+
[]string{"device", "vf", "pci_address", "numa_node"}, nil,
90+
),
91+
transmitBytes: prometheus.NewDesc(
92+
prometheus.BuildFQName(namespace, netvfSubsystem, "transmit_bytes_total"),
93+
"Number of transmitted bytes by the VF.",
94+
[]string{"device", "vf", "pci_address", "numa_node"}, nil,
95+
),
96+
broadcast: prometheus.NewDesc(
97+
prometheus.BuildFQName(namespace, netvfSubsystem, "broadcast_packets_total"),
98+
"Number of broadcast packets received by the VF.",
99+
[]string{"device", "vf", "pci_address", "numa_node"}, nil,
100+
),
101+
multicast: prometheus.NewDesc(
102+
prometheus.BuildFQName(namespace, netvfSubsystem, "multicast_packets_total"),
103+
"Number of multicast packets received by the VF.",
104+
[]string{"device", "vf", "pci_address", "numa_node"}, nil,
105+
),
106+
receiveDropped: prometheus.NewDesc(
107+
prometheus.BuildFQName(namespace, netvfSubsystem, "receive_dropped_total"),
108+
"Number of dropped received packets by the VF.",
109+
[]string{"device", "vf", "pci_address", "numa_node"}, nil,
110+
),
111+
transmitDropped: prometheus.NewDesc(
112+
prometheus.BuildFQName(namespace, netvfSubsystem, "transmit_dropped_total"),
113+
"Number of dropped transmitted packets by the VF.",
114+
[]string{"device", "vf", "pci_address", "numa_node"}, nil,
115+
),
116+
}, nil
117+
}
118+
119+
func (c *netvfCollector) Update(ch chan<- prometheus.Metric) error {
120+
conn, err := rtnetlink.Dial(nil)
121+
if err != nil {
122+
return fmt.Errorf("failed to connect to rtnetlink: %w", err)
123+
}
124+
defer conn.Close()
125+
126+
links, err := conn.Link.ListWithVFInfo()
127+
if err != nil {
128+
return fmt.Errorf("failed to list interfaces with VF info: %w", err)
129+
}
130+
131+
sysFS, sysErr := sysfs.NewFS(sysFilePath(""))
132+
133+
vfCount := 0
134+
for _, link := range links {
135+
if link.Attributes == nil {
136+
continue
137+
}
138+
139+
// Skip interfaces without VFs
140+
if link.Attributes.NumVF == nil || *link.Attributes.NumVF == 0 {
141+
continue
142+
}
143+
144+
device := link.Attributes.Name
145+
146+
// Apply device filter
147+
if c.deviceFilter.ignored(device) {
148+
c.logger.Debug("Ignoring device", "device", device)
149+
continue
150+
}
151+
152+
// Resolve PCI device once per PF to get NUMA node and VF addresses.
153+
numaNode := "-1"
154+
var pciDev *sysfs.PciDevice
155+
if sysErr == nil {
156+
if dev, err := sysFS.NetClassPCIDevice(device); err == nil {
157+
pciDev = dev
158+
if dev.NumaNode != nil {
159+
numaNode = fmt.Sprintf("%d", *dev.NumaNode)
160+
}
161+
}
162+
}
163+
164+
for _, vf := range link.Attributes.VFInfoList {
165+
vfID := fmt.Sprintf("%d", vf.ID)
166+
167+
// Emit info metric with VF configuration
168+
mac := ""
169+
if vf.MAC != nil {
170+
mac = vf.MAC.String()
171+
}
172+
vlan := fmt.Sprintf("%d", vf.Vlan)
173+
linkState := vfLinkStateString(vf.LinkState)
174+
spoofCheck := fmt.Sprintf("%t", vf.SpoofCheck)
175+
trust := fmt.Sprintf("%t", vf.Trust)
176+
177+
pciAddress := ""
178+
if pciDev != nil {
179+
if addr, err := sysFS.PciDeviceVFAddress(pciDev, vf.ID); err == nil {
180+
pciAddress = addr
181+
}
182+
}
183+
184+
ch <- prometheus.MustNewConstMetric(c.info, prometheus.GaugeValue, 1, device, vfID, mac, vlan, linkState, spoofCheck, trust, pciAddress, numaNode)
185+
186+
// Emit stats metrics if available
187+
if vf.Stats == nil {
188+
c.logger.Debug("VF has no stats", "device", device, "vf", vf.ID)
189+
vfCount++
190+
continue
191+
}
192+
193+
stats := vf.Stats
194+
195+
ch <- prometheus.MustNewConstMetric(c.receivePackets, prometheus.CounterValue, float64(stats.RxPackets), device, vfID, pciAddress, numaNode)
196+
ch <- prometheus.MustNewConstMetric(c.transmitPackets, prometheus.CounterValue, float64(stats.TxPackets), device, vfID, pciAddress, numaNode)
197+
ch <- prometheus.MustNewConstMetric(c.receiveBytes, prometheus.CounterValue, float64(stats.RxBytes), device, vfID, pciAddress, numaNode)
198+
ch <- prometheus.MustNewConstMetric(c.transmitBytes, prometheus.CounterValue, float64(stats.TxBytes), device, vfID, pciAddress, numaNode)
199+
ch <- prometheus.MustNewConstMetric(c.broadcast, prometheus.CounterValue, float64(stats.Broadcast), device, vfID, pciAddress, numaNode)
200+
ch <- prometheus.MustNewConstMetric(c.multicast, prometheus.CounterValue, float64(stats.Multicast), device, vfID, pciAddress, numaNode)
201+
ch <- prometheus.MustNewConstMetric(c.receiveDropped, prometheus.CounterValue, float64(stats.RxDropped), device, vfID, pciAddress, numaNode)
202+
ch <- prometheus.MustNewConstMetric(c.transmitDropped, prometheus.CounterValue, float64(stats.TxDropped), device, vfID, pciAddress, numaNode)
203+
204+
vfCount++
205+
}
206+
}
207+
208+
if vfCount == 0 {
209+
return ErrNoData
210+
}
211+
212+
return nil
213+
}
214+
215+
func vfLinkStateString(state rtnetlink.VFLinkState) string {
216+
switch state {
217+
case rtnetlink.VFLinkStateAuto:
218+
return "auto"
219+
case rtnetlink.VFLinkStateEnable:
220+
return "enable"
221+
case rtnetlink.VFLinkStateDisable:
222+
return "disable"
223+
default:
224+
return "unknown"
225+
}
226+
}
227+

0 commit comments

Comments
 (0)