From 618342bf1755baa21b22bd6f0fd5d1cff2b63dba Mon Sep 17 00:00:00 2001 From: Shirly Radco Date: Sun, 15 Mar 2026 14:26:21 +0200 Subject: [PATCH 1/2] collector: add dmmultipath collector for DM-multipath sysfs metrics MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a new disabled-by-default collector that reads /sys/block/dm-* to discover Device Mapper multipath devices and expose path health metrics. Multipath devices are identified by checking that dm/uuid starts with "mpath-", which distinguishes them from LVM or other DM device types. The path state is reported as-is from /sys/block//device/state, supporting both SCSI devices (running, offline, blocked, etc.) and NVMe devices (live, connecting, dead, etc.) without hardcoding a fixed set of states. All device-level metrics include both the DM friendly name (device) and the kernel block device name (sysfs_name, e.g. dm-0) to enable direct correlation with node_disk_* I/O metrics without recording rules. No special permissions are required — the collector reads only world-readable sysfs attributes. Exposed metrics: - node_dmmultipath_device_info - node_dmmultipath_device_active - node_dmmultipath_device_size_bytes - node_dmmultipath_device_paths - node_dmmultipath_device_paths_active - node_dmmultipath_device_paths_failed - node_dmmultipath_path_state Signed-off-by: Shirly Radco Co-authored-by: AI Assistant --- README.md | 27 +++++ collector/dmmultipath_linux.go | 143 +++++++++++++++++++++++ collector/dmmultipath_linux_test.go | 151 +++++++++++++++++++++++++ collector/fixtures/sys.ttar | 168 ++++++++++++++++++++++++++++ 4 files changed, 489 insertions(+) create mode 100644 collector/dmmultipath_linux.go create mode 100644 collector/dmmultipath_linux_test.go diff --git a/README.md b/README.md index ec5adbf7ac..47aa32e1fc 100644 --- a/README.md +++ b/README.md @@ -191,6 +191,7 @@ buddyinfo | Exposes statistics of memory fragments as reported by /proc/buddyinf cgroups | A summary of the number of active and enabled cgroups | Linux cpu\_vulnerabilities | Exposes CPU vulnerability information from sysfs. | Linux devstat | Exposes device statistics | Dragonfly, FreeBSD +dmmultipath | Exposes DM-multipath device and path metrics from `/sys/block/dm-*`. | Linux drm | Expose GPU metrics using sysfs / DRM, `amdgpu` is the only driver which exposes this information through DRM | Linux drbd | Exposes Distributed Replicated Block Device statistics (to version 8.4) | Linux ethtool | Exposes network interface information and network driver statistics equivalent to `ethtool`, `ethtool -S`, and `ethtool -i`. | Linux @@ -339,6 +340,32 @@ echo 'role{role="application_server"} 1' > /path/to/directory/role.prom.$$ mv /path/to/directory/role.prom.$$ /path/to/directory/role.prom ``` +### DM-Multipath Collector + +The `dmmultipath` collector reads `/sys/block/dm-*` to discover Device Mapper +multipath devices and expose path health metrics. It identifies multipath +devices by checking that `dm/uuid` starts with `mpath-`, which distinguishes +them from LVM or other DM device types. + +No special permissions are required — the collector reads only world-readable +sysfs attributes. + +Enable it with `--collector.dmmultipath`. + +#### Exposed metrics + +| Metric | Type | Description | +|--------|------|-------------| +| `node_dmmultipath_device_info` | Gauge | Info metric with `device`, `sysfs_name`, and `uuid` (contains WWID for PV correlation). | +| `node_dmmultipath_device_active` | Gauge | Whether the DM device is active (1) or suspended (0). Labels: `device`, `sysfs_name`. | +| `node_dmmultipath_device_size_bytes` | Gauge | Size of the DM device in bytes. Labels: `device`, `sysfs_name`. | +| `node_dmmultipath_device_paths` | Gauge | Number of paths. Labels: `device`, `sysfs_name`. | +| `node_dmmultipath_device_paths_active` | Gauge | Number of paths in active state (SCSI `running` or NVMe `live`). Labels: `device`, `sysfs_name`. | +| `node_dmmultipath_device_paths_failed` | Gauge | Number of paths not in active state. Labels: `device`, `sysfs_name`. | +| `node_dmmultipath_path_state` | Gauge | Reports the underlying device state for each path. Labels: `device`, `path`, `state`. | + +The `sysfs_name` label (e.g. `dm-0`) matches the `device` label in `node_disk_*` metrics, enabling direct correlation between multipath health and I/O statistics without recording rules. + ### Filtering enabled collectors The `node_exporter` will expose all metrics from enabled collectors by default. This is the recommended way to collect metrics to avoid errors when comparing metrics of different families. diff --git a/collector/dmmultipath_linux.go b/collector/dmmultipath_linux.go new file mode 100644 index 0000000000..82706ff5ea --- /dev/null +++ b/collector/dmmultipath_linux.go @@ -0,0 +1,143 @@ +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !nodmmultipath + +package collector + +import ( + "errors" + "fmt" + "log/slog" + "os" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/procfs/blockdevice" +) + +// isPathActive returns true for device states that indicate a healthy, +// usable path. This covers SCSI ("running") and NVMe ("live") devices. +func isPathActive(state string) bool { + return state == "running" || state == "live" +} + +type dmMultipathCollector struct { + fs blockdevice.FS + logger *slog.Logger + + deviceInfo *prometheus.Desc + deviceActive *prometheus.Desc + deviceSizeBytes *prometheus.Desc + devicePaths *prometheus.Desc + devicePathsActive *prometheus.Desc + devicePathsFailed *prometheus.Desc + pathState *prometheus.Desc +} + +func init() { + registerCollector("dmmultipath", defaultDisabled, NewDMMultipathCollector) +} + +// NewDMMultipathCollector returns a new Collector exposing Device Mapper +// multipath device metrics from /sys/block/dm-*. +func NewDMMultipathCollector(logger *slog.Logger) (Collector, error) { + const subsystem = "dmmultipath" + + fs, err := blockdevice.NewFS(*procPath, *sysPath) + if err != nil { + return nil, fmt.Errorf("failed to open sysfs: %w", err) + } + + deviceLabels := []string{"device", "sysfs_name"} + + return &dmMultipathCollector{ + fs: fs, + logger: logger, + deviceInfo: prometheus.NewDesc( + prometheus.BuildFQName(namespace, subsystem, "device_info"), + "Non-numeric information about a DM-multipath device.", + []string{"device", "sysfs_name", "uuid"}, nil, + ), + deviceActive: prometheus.NewDesc( + prometheus.BuildFQName(namespace, subsystem, "device_active"), + "Whether the multipath device-mapper device is active (1) or suspended (0).", + deviceLabels, nil, + ), + deviceSizeBytes: prometheus.NewDesc( + prometheus.BuildFQName(namespace, subsystem, "device_size_bytes"), + "Size of the multipath device in bytes, read from /sys/block//size.", + deviceLabels, nil, + ), + devicePaths: prometheus.NewDesc( + prometheus.BuildFQName(namespace, subsystem, "device_paths"), + "Number of paths for a multipath device.", + deviceLabels, nil, + ), + devicePathsActive: prometheus.NewDesc( + prometheus.BuildFQName(namespace, subsystem, "device_paths_active"), + "Number of paths in active state (SCSI running or NVMe live) for a multipath device.", + deviceLabels, nil, + ), + devicePathsFailed: prometheus.NewDesc( + prometheus.BuildFQName(namespace, subsystem, "device_paths_failed"), + "Number of paths not in active state for a multipath device.", + deviceLabels, nil, + ), + pathState: prometheus.NewDesc( + prometheus.BuildFQName(namespace, subsystem, "path_state"), + "Reports the underlying device state for a multipath path, as read from /sys/block//device/state.", + []string{"device", "path", "state"}, nil, + ), + }, nil +} + +func (c *dmMultipathCollector) Update(ch chan<- prometheus.Metric) error { + devices, err := c.fs.DMMultipathDevices() + if err != nil { + if errors.Is(err, os.ErrNotExist) || errors.Is(err, os.ErrPermission) { + c.logger.Debug("Could not read DM-multipath devices", "err", err) + return ErrNoData + } + return fmt.Errorf("failed to scan DM-multipath devices: %w", err) + } + + for _, dev := range devices { + ch <- prometheus.MustNewConstMetric(c.deviceInfo, prometheus.GaugeValue, 1, + dev.Name, dev.SysfsName, dev.UUID) + + active := 0.0 + if !dev.Suspended { + active = 1.0 + } + ch <- prometheus.MustNewConstMetric(c.deviceActive, prometheus.GaugeValue, active, dev.Name, dev.SysfsName) + ch <- prometheus.MustNewConstMetric(c.deviceSizeBytes, prometheus.GaugeValue, float64(dev.SizeBytes), dev.Name, dev.SysfsName) + + var activePaths, failedPaths float64 + for _, p := range dev.Paths { + if isPathActive(p.State) { + activePaths++ + } else { + failedPaths++ + } + + ch <- prometheus.MustNewConstMetric(c.pathState, prometheus.GaugeValue, 1, + dev.Name, p.Device, p.State) + } + + ch <- prometheus.MustNewConstMetric(c.devicePaths, prometheus.GaugeValue, float64(len(dev.Paths)), dev.Name, dev.SysfsName) + ch <- prometheus.MustNewConstMetric(c.devicePathsActive, prometheus.GaugeValue, activePaths, dev.Name, dev.SysfsName) + ch <- prometheus.MustNewConstMetric(c.devicePathsFailed, prometheus.GaugeValue, failedPaths, dev.Name, dev.SysfsName) + } + + return nil +} diff --git a/collector/dmmultipath_linux_test.go b/collector/dmmultipath_linux_test.go new file mode 100644 index 0000000000..454c47c138 --- /dev/null +++ b/collector/dmmultipath_linux_test.go @@ -0,0 +1,151 @@ +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !nodmmultipath + +package collector + +import ( + "io" + "log/slog" + "strings" + "testing" + + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" +) + +func TestDMMultipathMetrics(t *testing.T) { + *procPath = "fixtures/proc" + *sysPath = "fixtures/sys" + + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + coll, err := NewDMMultipathCollector(logger) + if err != nil { + t.Fatal(err) + } + + c := coll.(*dmMultipathCollector) + + ch := make(chan prometheus.Metric, 200) + if err := c.Update(ch); err != nil { + t.Fatal(err) + } + close(ch) + + metrics := make(map[string][]*dto.Metric) + for m := range ch { + d := &dto.Metric{} + if err := m.Write(d); err != nil { + t.Fatal(err) + } + desc := m.Desc().String() + metrics[desc] = append(metrics[desc], d) + } + + assertGaugeValue(t, metrics, "device_active", labelMap{"device": "mpathA", "sysfs_name": "dm-5"}, 1) + assertGaugeValue(t, metrics, "device_active", labelMap{"device": "mpathB", "sysfs_name": "dm-6"}, 1) + assertGaugeValue(t, metrics, "device_size_bytes", labelMap{"device": "mpathA", "sysfs_name": "dm-5"}, 53687091200) + assertGaugeValue(t, metrics, `device_paths"`, labelMap{"device": "mpathA", "sysfs_name": "dm-5"}, 4) + assertGaugeValue(t, metrics, `device_paths"`, labelMap{"device": "mpathB", "sysfs_name": "dm-6"}, 2) + + // mpathA: sdi, sdj, sdk are running; sdl is offline → 3 active, 1 failed. + assertGaugeValue(t, metrics, "device_paths_active", labelMap{"device": "mpathA", "sysfs_name": "dm-5"}, 3) + assertGaugeValue(t, metrics, "device_paths_failed", labelMap{"device": "mpathA", "sysfs_name": "dm-5"}, 1) + + // mpathB: sdm, sdn are both running → 2 active, 0 failed. + assertGaugeValue(t, metrics, "device_paths_active", labelMap{"device": "mpathB", "sysfs_name": "dm-6"}, 2) + assertGaugeValue(t, metrics, "device_paths_failed", labelMap{"device": "mpathB", "sysfs_name": "dm-6"}, 0) + + assertGaugeValue(t, metrics, "path_state", + labelMap{"device": "mpathA", "path": "sdi", "state": "running"}, 1) + assertGaugeValue(t, metrics, "path_state", + labelMap{"device": "mpathA", "path": "sdl", "state": "offline"}, 1) +} + +func TestDMMultipathNoDevices(t *testing.T) { + *procPath = "fixtures/proc" + *sysPath = t.TempDir() + + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + coll, err := NewDMMultipathCollector(logger) + if err != nil { + t.Fatal(err) + } + + c := coll.(*dmMultipathCollector) + + ch := make(chan prometheus.Metric, 200) + err = c.Update(ch) + close(ch) + + if err != ErrNoData { + t.Fatalf("expected ErrNoData, got %v", err) + } +} + +func TestIsPathActive(t *testing.T) { + tests := []struct { + state string + active bool + }{ + {"running", true}, + {"live", true}, + {"offline", false}, + {"blocked", false}, + {"transport-offline", false}, + {"dead", false}, + {"unknown", false}, + {"", false}, + } + for _, tc := range tests { + got := isPathActive(tc.state) + if got != tc.active { + t.Errorf("isPathActive(%q) = %v, want %v", tc.state, got, tc.active) + } + } +} + +type labelMap map[string]string + +func assertGaugeValue(t *testing.T, metrics map[string][]*dto.Metric, metricSubstring string, labels labelMap, expected float64) { + t.Helper() + for desc, ms := range metrics { + if !strings.Contains(desc, metricSubstring) { + continue + } + for _, m := range ms { + if matchLabels(m.GetLabel(), labels) { + got := m.GetGauge().GetValue() + if got != expected { + t.Errorf("%s%v: got %v, want %v", metricSubstring, labels, got, expected) + } + return + } + } + } + t.Errorf("metric %s%v not found", metricSubstring, labels) +} + +func matchLabels(pairs []*dto.LabelPair, want labelMap) bool { + if want == nil { + return len(pairs) == 0 + } + found := 0 + for _, lp := range pairs { + if v, ok := want[lp.GetName()]; ok && v == lp.GetValue() { + found++ + } + } + return found == len(want) +} diff --git a/collector/fixtures/sys.ttar b/collector/fixtures/sys.ttar index bc8744cbe7..a06ab40af6 100644 --- a/collector/fixtures/sys.ttar +++ b/collector/fixtures/sys.ttar @@ -826,6 +826,174 @@ Lines: 1 none Mode: 644 # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Directory: sys/block/dm-5 +Mode: 755 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Directory: sys/block/dm-5/dm +Mode: 755 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/dm-5/dm/name +Lines: 1 +mpathAEOF +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/dm-5/dm/suspended +Lines: 1 +0EOF +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/dm-5/dm/uuid +Lines: 1 +mpath-3600508b1001c1234567890abcdef1234EOF +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/dm-5/size +Lines: 1 +104857600EOF +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Directory: sys/block/dm-5/slaves +Mode: 755 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Directory: sys/block/dm-5/slaves/sdi +Mode: 755 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Directory: sys/block/dm-5/slaves/sdj +Mode: 755 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Directory: sys/block/dm-5/slaves/sdk +Mode: 755 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Directory: sys/block/dm-5/slaves/sdl +Mode: 755 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Directory: sys/block/dm-6 +Mode: 755 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Directory: sys/block/dm-6/dm +Mode: 755 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/dm-6/dm/name +Lines: 1 +mpathBEOF +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/dm-6/dm/suspended +Lines: 1 +0EOF +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/dm-6/dm/uuid +Lines: 1 +mpath-3600508b1001cabcdef4567890123456EOF +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/dm-6/size +Lines: 1 +209715200EOF +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Directory: sys/block/dm-6/slaves +Mode: 755 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Directory: sys/block/dm-6/slaves/sdm +Mode: 755 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Directory: sys/block/dm-6/slaves/sdn +Mode: 755 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Directory: sys/block/dm-7 +Mode: 755 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Directory: sys/block/dm-7/dm +Mode: 755 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/dm-7/dm/name +Lines: 1 +vg0-rootEOF +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/dm-7/dm/suspended +Lines: 1 +0EOF +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/dm-7/dm/uuid +Lines: 1 +LVM-abcdef1234567890abcdef1234567890EOF +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/dm-7/size +Lines: 1 +41943040EOF +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Directory: sys/block/sdi +Mode: 755 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Directory: sys/block/sdi/device +Mode: 755 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/sdi/device/state +Lines: 1 +runningEOF +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Directory: sys/block/sdj +Mode: 755 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Directory: sys/block/sdj/device +Mode: 755 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/sdj/device/state +Lines: 1 +runningEOF +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Directory: sys/block/sdk +Mode: 755 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Directory: sys/block/sdk/device +Mode: 755 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/sdk/device/state +Lines: 1 +runningEOF +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Directory: sys/block/sdl +Mode: 755 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Directory: sys/block/sdl/device +Mode: 755 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/sdl/device/state +Lines: 1 +offlineEOF +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Directory: sys/block/sdm +Mode: 755 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Directory: sys/block/sdm/device +Mode: 755 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/sdm/device/state +Lines: 1 +runningEOF +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Directory: sys/block/sdn +Mode: 755 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Directory: sys/block/sdn/device +Mode: 755 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/block/sdn/device/state +Lines: 1 +runningEOF +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Directory: sys/bus Mode: 755 # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - From 705c86eb607c47acf1e0013c90b4a8286deb85d1 Mon Sep 17 00:00:00 2001 From: Shirly Radco Date: Thu, 12 Mar 2026 11:08:24 +0200 Subject: [PATCH 2/2] collector: add nvmesubsystem collector for NVMe-oF path health MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add a new disabled-by-default collector that reads /sys/class/nvme-subsystem/ to expose NVMe over Fabrics subsystem connectivity metrics. This complements the existing nvme collector (which reports per-controller hardware stats) by monitoring the subsystem-level path redundancy — how many controller paths are live, connecting, or dead for each NVMe subsystem. Exposed metrics: - node_nvmesubsystem_info - node_nvmesubsystem_paths - node_nvmesubsystem_paths_live - node_nvmesubsystem_path_state Signed-off-by: Shirly Radco Co-authored-by: AI Assistant --- README.md | 20 ++++ collector/fixtures/sys.ttar | 98 ++++++++++++++++++ collector/nvmesubsystem_linux.go | 130 +++++++++++++++++++++++ collector/nvmesubsystem_linux_test.go | 143 ++++++++++++++++++++++++++ 4 files changed, 391 insertions(+) create mode 100644 collector/nvmesubsystem_linux.go create mode 100644 collector/nvmesubsystem_linux_test.go diff --git a/README.md b/README.md index 47aa32e1fc..002d38f133 100644 --- a/README.md +++ b/README.md @@ -203,6 +203,7 @@ logind | Exposes session counts from [logind](http://www.freedesktop.org/wiki/So meminfo\_numa | Exposes memory statistics from `/sys/devices/system/node/node[0-9]*/meminfo`, `/sys/devices/system/node/node[0-9]*/numastat`. | Linux mountstats | Exposes filesystem statistics from `/proc/self/mountstats`. Exposes detailed NFS client statistics. | Linux network_route | Exposes the routing table as metrics | Linux +nvmesubsystem | Exposes NVMe-oF subsystem path health from `/sys/class/nvme-subsystem/`. | Linux pcidevice | Exposes pci devices' information including their link status and parent devices. | Linux perf | Exposes perf based metrics (Warning: Metrics are dependent on kernel configuration and settings). | Linux processes | Exposes aggregate process statistics from `/proc`. | Linux @@ -366,6 +367,25 @@ Enable it with `--collector.dmmultipath`. The `sysfs_name` label (e.g. `dm-0`) matches the `device` label in `node_disk_*` metrics, enabling direct correlation between multipath health and I/O statistics without recording rules. +### NVMe Subsystem Collector + +The `nvmesubsystem` collector exposes NVMe-oF (NVMe over Fabrics) subsystem +path health by reading `/sys/class/nvme-subsystem/`. It complements the +existing `nvme` collector (which reports per-controller hardware stats) by +monitoring the **connectivity layer** — how many controller paths are live, +connecting, or dead for each NVMe subsystem. + +Enable it with `--collector.nvmesubsystem`. + +#### Exposed metrics + +| Metric | Description | +|--------|-------------| +| `node_nvmesubsystem_info` | Info metric with subsystem NQN, model, serial and I/O policy as labels. | +| `node_nvmesubsystem_paths` | Number of controller paths for the subsystem. | +| `node_nvmesubsystem_paths_live` | Number of controller paths currently in `live` state. | +| `node_nvmesubsystem_path_state` | Per-controller path state (1 for the current state, 0 for others). | + ### Filtering enabled collectors The `node_exporter` will expose all metrics from enabled collectors by default. This is the recommended way to collect metrics to avoid errors when comparing metrics of different families. diff --git a/collector/fixtures/sys.ttar b/collector/fixtures/sys.ttar index a06ab40af6..455b2cca8f 100644 --- a/collector/fixtures/sys.ttar +++ b/collector/fixtures/sys.ttar @@ -2423,6 +2423,104 @@ Lines: 1 4096 Mode: 644 # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Directory: sys/class/nvme-subsystem +Mode: 755 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Directory: sys/class/nvme-subsystem/nvme-subsys0 +Mode: 755 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/class/nvme-subsystem/nvme-subsys0/iopolicy +Lines: 1 +round-robinEOF +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/class/nvme-subsystem/nvme-subsys0/model +Lines: 1 +Dell PowerStoreEOF +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Directory: sys/class/nvme-subsystem/nvme-subsys0/nvme0 +Mode: 755 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/class/nvme-subsystem/nvme-subsys0/nvme0/address +Lines: 1 +nn-0x200000109b123456:pn-0x100000109b123456EOF +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/class/nvme-subsystem/nvme-subsys0/nvme0/state +Lines: 1 +liveEOF +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/class/nvme-subsystem/nvme-subsys0/nvme0/transport +Lines: 1 +fcEOF +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Directory: sys/class/nvme-subsystem/nvme-subsys0/nvme1 +Mode: 755 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/class/nvme-subsystem/nvme-subsys0/nvme1/address +Lines: 1 +nn-0x200000109b123457:pn-0x100000109b123457EOF +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/class/nvme-subsystem/nvme-subsys0/nvme1/state +Lines: 1 +liveEOF +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/class/nvme-subsystem/nvme-subsys0/nvme1/transport +Lines: 1 +fcEOF +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Directory: sys/class/nvme-subsystem/nvme-subsys0/nvme2 +Mode: 755 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/class/nvme-subsystem/nvme-subsys0/nvme2/address +Lines: 1 +nn-0x200000109b123458:pn-0x100000109b123458EOF +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/class/nvme-subsystem/nvme-subsys0/nvme2/state +Lines: 1 +liveEOF +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/class/nvme-subsystem/nvme-subsys0/nvme2/transport +Lines: 1 +fcEOF +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Directory: sys/class/nvme-subsystem/nvme-subsys0/nvme3 +Mode: 755 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/class/nvme-subsystem/nvme-subsys0/nvme3/address +Lines: 1 +nn-0x200000109b123459:pn-0x100000109b123459EOF +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/class/nvme-subsystem/nvme-subsys0/nvme3/state +Lines: 1 +deadEOF +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/class/nvme-subsystem/nvme-subsys0/nvme3/transport +Lines: 1 +fcEOF +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/class/nvme-subsystem/nvme-subsys0/serial +Lines: 1 +SN12345678EOF +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - +Path: sys/class/nvme-subsystem/nvme-subsys0/subsysnqn +Lines: 1 +nqn.2014-08.org.nvmexpress:uuid:a34c4f3a-0d6f-5cec-dead-beefcafebabeEOF +Mode: 644 +# ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - Directory: sys/class/power_supply Mode: 755 # ttar - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/collector/nvmesubsystem_linux.go b/collector/nvmesubsystem_linux.go new file mode 100644 index 0000000000..71c96b1908 --- /dev/null +++ b/collector/nvmesubsystem_linux.go @@ -0,0 +1,130 @@ +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !nonvmesubsystem + +package collector + +import ( + "errors" + "fmt" + "log/slog" + "os" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/procfs/sysfs" +) + +var nvmeControllerStates = []string{ + "live", "connecting", "resetting", "dead", "unknown", +} + +func normalizeControllerState(raw string) string { + switch raw { + case "live", "connecting", "resetting", "dead": + return raw + case "deleting", "deleting (no IO)", "new": + return raw + default: + return "unknown" + } +} + +type nvmeSubsystemCollector struct { + fs sysfs.FS + logger *slog.Logger + + subsystemInfo *prometheus.Desc + subsystemPaths *prometheus.Desc + subsystemPathsLive *prometheus.Desc + pathState *prometheus.Desc +} + +func init() { + registerCollector("nvmesubsystem", defaultDisabled, NewNVMeSubsystemCollector) +} + +// NewNVMeSubsystemCollector returns a new Collector exposing NVMe-oF subsystem +// path health from /sys/class/nvme-subsystem/. +func NewNVMeSubsystemCollector(logger *slog.Logger) (Collector, error) { + const subsystem = "nvmesubsystem" + + fs, err := sysfs.NewFS(*sysPath) + if err != nil { + return nil, fmt.Errorf("failed to open sysfs: %w", err) + } + + return &nvmeSubsystemCollector{ + fs: fs, + logger: logger, + subsystemInfo: prometheus.NewDesc( + prometheus.BuildFQName(namespace, subsystem, "info"), + "Non-numeric information about an NVMe subsystem.", + []string{"subsystem", "nqn", "model", "serial", "iopolicy"}, nil, + ), + subsystemPaths: prometheus.NewDesc( + prometheus.BuildFQName(namespace, subsystem, "paths"), + "Number of controller paths for an NVMe subsystem.", + []string{"subsystem"}, nil, + ), + subsystemPathsLive: prometheus.NewDesc( + prometheus.BuildFQName(namespace, subsystem, "paths_live"), + "Number of controller paths in live state for an NVMe subsystem.", + []string{"subsystem"}, nil, + ), + pathState: prometheus.NewDesc( + prometheus.BuildFQName(namespace, subsystem, "path_state"), + "Current NVMe controller path state (1 for the current state, 0 for all others).", + []string{"subsystem", "controller", "transport", "state"}, nil, + ), + }, nil +} + +func (c *nvmeSubsystemCollector) Update(ch chan<- prometheus.Metric) error { + subsystems, err := c.fs.NVMeSubsystemClass() + if err != nil { + if errors.Is(err, os.ErrNotExist) || errors.Is(err, os.ErrPermission) { + c.logger.Debug("Could not read NVMe subsystem info", "err", err) + return ErrNoData + } + return fmt.Errorf("failed to scan NVMe subsystems: %w", err) + } + + for _, subsys := range subsystems { + ch <- prometheus.MustNewConstMetric(c.subsystemInfo, prometheus.GaugeValue, 1, + subsys.Name, subsys.NQN, subsys.Model, subsys.Serial, subsys.IOPolicy) + + total := float64(len(subsys.Controllers)) + var live float64 + for _, ctrl := range subsys.Controllers { + state := normalizeControllerState(ctrl.State) + if state == "live" { + live++ + } + + for _, s := range nvmeControllerStates { + val := 0.0 + if s == state { + val = 1.0 + } + ch <- prometheus.MustNewConstMetric(c.pathState, prometheus.GaugeValue, val, + subsys.Name, ctrl.Name, ctrl.Transport, s) + } + } + + ch <- prometheus.MustNewConstMetric(c.subsystemPaths, prometheus.GaugeValue, total, subsys.Name) + ch <- prometheus.MustNewConstMetric(c.subsystemPathsLive, prometheus.GaugeValue, live, subsys.Name) + } + + return nil +} diff --git a/collector/nvmesubsystem_linux_test.go b/collector/nvmesubsystem_linux_test.go new file mode 100644 index 0000000000..0c97530757 --- /dev/null +++ b/collector/nvmesubsystem_linux_test.go @@ -0,0 +1,143 @@ +// Copyright The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !nonvmesubsystem + +package collector + +import ( + "io" + "log/slog" + "strings" + "testing" + + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" +) + +func TestNVMeSubsystemMetrics(t *testing.T) { + *sysPath = "fixtures/sys" + + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + coll, err := NewNVMeSubsystemCollector(logger) + if err != nil { + t.Fatal(err) + } + + c := coll.(*nvmeSubsystemCollector) + + ch := make(chan prometheus.Metric, 200) + if err := c.Update(ch); err != nil { + t.Fatal(err) + } + close(ch) + + metrics := make(map[string][]*dto.Metric) + for m := range ch { + d := &dto.Metric{} + if err := m.Write(d); err != nil { + t.Fatal(err) + } + desc := m.Desc().String() + metrics[desc] = append(metrics[desc], d) + } + + assertGaugeValue(t, metrics, `paths"`, labelMap{"subsystem": "nvme-subsys0"}, 4) + assertGaugeValue(t, metrics, "paths_live", labelMap{"subsystem": "nvme-subsys0"}, 3) + + assertGaugeValue(t, metrics, "path_state", + labelMap{"subsystem": "nvme-subsys0", "controller": "nvme0", "transport": "fc", "state": "live"}, 1) + assertGaugeValue(t, metrics, "path_state", + labelMap{"subsystem": "nvme-subsys0", "controller": "nvme0", "transport": "fc", "state": "dead"}, 0) + assertGaugeValue(t, metrics, "path_state", + labelMap{"subsystem": "nvme-subsys0", "controller": "nvme3", "transport": "fc", "state": "dead"}, 1) + assertGaugeValue(t, metrics, "path_state", + labelMap{"subsystem": "nvme-subsys0", "controller": "nvme3", "transport": "fc", "state": "live"}, 0) +} + +func TestNVMeSubsystemNoDevices(t *testing.T) { + *sysPath = t.TempDir() + + logger := slog.New(slog.NewTextHandler(io.Discard, nil)) + coll, err := NewNVMeSubsystemCollector(logger) + if err != nil { + t.Fatal(err) + } + + c := coll.(*nvmeSubsystemCollector) + + ch := make(chan prometheus.Metric, 200) + err = c.Update(ch) + close(ch) + + if err != ErrNoData { + t.Fatalf("expected ErrNoData, got %v", err) + } +} + +func TestNormalizeControllerState(t *testing.T) { + tests := []struct { + raw string + expected string + }{ + {"live", "live"}, + {"connecting", "connecting"}, + {"resetting", "resetting"}, + {"dead", "dead"}, + {"deleting", "deleting"}, + {"deleting (no IO)", "deleting (no IO)"}, + {"new", "new"}, + {"", "unknown"}, + {"something-else", "unknown"}, + } + for _, tc := range tests { + got := normalizeControllerState(tc.raw) + if got != tc.expected { + t.Errorf("normalizeControllerState(%q) = %q, want %q", tc.raw, got, tc.expected) + } + } +} + +type labelMap map[string]string + +func assertGaugeValue(t *testing.T, metrics map[string][]*dto.Metric, metricSubstring string, labels labelMap, expected float64) { + t.Helper() + for desc, ms := range metrics { + if !strings.Contains(desc, metricSubstring) { + continue + } + for _, m := range ms { + if matchLabels(m.GetLabel(), labels) { + got := m.GetGauge().GetValue() + if got != expected { + t.Errorf("%s%v: got %v, want %v", metricSubstring, labels, got, expected) + } + return + } + } + } + t.Errorf("metric %s%v not found", metricSubstring, labels) +} + +func matchLabels(pairs []*dto.LabelPair, want labelMap) bool { + if want == nil { + return len(pairs) == 0 + } + found := 0 + for _, lp := range pairs { + if v, ok := want[lp.GetName()]; ok && v == lp.GetValue() { + found++ + } + } + return found == len(want) +}