Skip to content

Commit 30bca0d

Browse files
sambhav-jain-16dims
authored andcommitted
feat: add fs io cost metrics
1 parent d3d20e8 commit 30bca0d

File tree

12 files changed

+192
-14
lines changed

12 files changed

+192
-14
lines changed

cmd/go.mod

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,7 @@ require (
5959
github.com/containerd/log v0.1.0 // indirect
6060
github.com/containerd/ttrpc v1.2.7 // indirect
6161
github.com/containerd/typeurl/v2 v2.2.3 // indirect
62-
github.com/coreos/go-systemd/v22 v22.5.0 // indirect
62+
github.com/coreos/go-systemd/v22 v22.6.0 // indirect
6363
github.com/cyphar/filepath-securejoin v0.5.1 // indirect
6464
github.com/davecgh/go-spew v1.1.1 // indirect
6565
github.com/distribution/reference v0.6.0 // indirect
@@ -97,7 +97,7 @@ require (
9797
github.com/moby/sys/sequential v0.6.0 // indirect
9898
github.com/moby/sys/userns v0.1.0 // indirect
9999
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
100-
github.com/opencontainers/cgroups v0.0.4 // indirect
100+
github.com/opencontainers/cgroups v0.0.6 // indirect
101101
github.com/opencontainers/go-digest v1.0.0 // indirect
102102
github.com/opencontainers/image-spec v1.1.1 // indirect
103103
github.com/opencontainers/runc v1.3.3 // indirect

cmd/go.sum

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -68,8 +68,8 @@ github.com/containerd/ttrpc v1.2.7 h1:qIrroQvuOL9HQ1X6KHe2ohc7p+HP/0VE6XPU7elJRq
6868
github.com/containerd/ttrpc v1.2.7/go.mod h1:YCXHsb32f+Sq5/72xHubdiJRQY9inL4a4ZQrAbN1q9o=
6969
github.com/containerd/typeurl/v2 v2.2.3 h1:yNA/94zxWdvYACdYO8zofhrTVuQY73fFU1y++dYSw40=
7070
github.com/containerd/typeurl/v2 v2.2.3/go.mod h1:95ljDnPfD3bAbDJRugOiShd/DlAAsxGtUBhJxIn7SCk=
71-
github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs=
72-
github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
71+
github.com/coreos/go-systemd/v22 v22.6.0 h1:aGVa/v8B7hpb0TKl0MWoAavPDmHvobFe5R5zn0bCJWo=
72+
github.com/coreos/go-systemd/v22 v22.6.0/go.mod h1:iG+pp635Fo7ZmV/j14KUcmEyWF+0X7Lua8rrTWzYgWU=
7373
github.com/cyphar/filepath-securejoin v0.5.1 h1:eYgfMq5yryL4fbWfkLpFFy2ukSELzaJOTaUTuh+oF48=
7474
github.com/cyphar/filepath-securejoin v0.5.1/go.mod h1:Sdj7gXlvMcPZsbhwhQ33GguGLDGQL7h7bg04C/+u9jI=
7575
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -106,7 +106,6 @@ github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre
106106
github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
107107
github.com/goccy/go-json v0.10.3 h1:KZ5WoDbxAIgm2HNbYckL0se1fHD6rz5j4ywS6ebzDqA=
108108
github.com/goccy/go-json v0.10.3/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M=
109-
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
110109
github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk=
111110
github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
112111
github.com/gofrs/uuid v3.3.0+incompatible h1:8K4tyRfvU1CYPgJsveYFQMhpFd/wXNM7iK6rR7UHz84=
@@ -216,8 +215,8 @@ github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7J
216215
github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
217216
github.com/onsi/gomega v1.24.1 h1:KORJXNNTzJXzu4ScJWssJfJMnJ+2QJqhoQSRwNlze9E=
218217
github.com/onsi/gomega v1.24.1/go.mod h1:3AOiACssS3/MajrniINInwbfOOtfZvplPzuRSmvt1jM=
219-
github.com/opencontainers/cgroups v0.0.4 h1:XVj8P/IHVms/j+7eh8ggdkTLAxjz84ZzuFyGoE28DR4=
220-
github.com/opencontainers/cgroups v0.0.4/go.mod h1:s8lktyhlGUqM7OSRL5P7eAW6Wb+kWPNvt4qvVfzA5vs=
218+
github.com/opencontainers/cgroups v0.0.6 h1:tfZFWTIIGaUUFImTyuTg+Mr5x8XRiSdZESgEBW7UxuI=
219+
github.com/opencontainers/cgroups v0.0.6/go.mod h1:oWVzJsKK0gG9SCRBfTpnn16WcGEqDI8PAcpMGbqWxcs=
221220
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
222221
github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
223222
github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040=

container/common/helpers.go

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -387,6 +387,10 @@ func AssignDeviceNamesToDiskStats(namer DeviceNamer, stats *info.DiskIoStats) {
387387
stats.IoTime,
388388
stats.IoWaitTime,
389389
stats.Sectors,
390+
stats.IoCostUsage,
391+
stats.IoCostWait,
392+
stats.IoCostIndebt,
393+
stats.IoCostIndelay,
390394
)
391395
}
392396

container/libcontainer/handler.go

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -793,6 +793,10 @@ func setDiskIoStats(s *cgroups.Stats, ret *info.ContainerStats) {
793793
ret.DiskIo.IoWaitTime = diskStatsCopy(s.BlkioStats.IoWaitTimeRecursive)
794794
ret.DiskIo.IoMerged = diskStatsCopy(s.BlkioStats.IoMergedRecursive)
795795
ret.DiskIo.IoTime = diskStatsCopy(s.BlkioStats.IoTimeRecursive)
796+
ret.DiskIo.IoCostUsage = diskStatsCopy(s.BlkioStats.IoCostUsage)
797+
ret.DiskIo.IoCostWait = diskStatsCopy(s.BlkioStats.IoCostWait)
798+
ret.DiskIo.IoCostIndebt = diskStatsCopy(s.BlkioStats.IoCostIndebt)
799+
ret.DiskIo.IoCostIndelay = diskStatsCopy(s.BlkioStats.IoCostIndelay)
796800
setPSIStats(s.BlkioStats.PSI, &ret.DiskIo.PSI)
797801
}
798802

go.mod

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ require (
2222
github.com/karrick/godirwalk v1.17.0
2323
github.com/mistifyio/go-zfs v2.1.1+incompatible
2424
github.com/moby/sys/mountinfo v0.7.2
25-
github.com/opencontainers/cgroups v0.0.4
25+
github.com/opencontainers/cgroups v0.0.6
2626
github.com/opencontainers/runc v1.3.3
2727
github.com/opencontainers/runtime-spec v1.2.1
2828
github.com/prometheus/client_golang v1.22.0
@@ -51,7 +51,7 @@ require (
5151
github.com/beorn7/perks v1.0.1 // indirect
5252
github.com/cespare/xxhash/v2 v2.3.0 // indirect
5353
github.com/containerd/log v0.1.0 // indirect
54-
github.com/coreos/go-systemd/v22 v22.5.0 // indirect
54+
github.com/coreos/go-systemd/v22 v22.6.0 // indirect
5555
github.com/cyphar/filepath-securejoin v0.5.1 // indirect
5656
github.com/davecgh/go-spew v1.1.1 // indirect
5757
github.com/distribution/reference v0.6.0 // indirect

go.sum

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -50,8 +50,8 @@ github.com/containerd/ttrpc v1.2.7 h1:qIrroQvuOL9HQ1X6KHe2ohc7p+HP/0VE6XPU7elJRq
5050
github.com/containerd/ttrpc v1.2.7/go.mod h1:YCXHsb32f+Sq5/72xHubdiJRQY9inL4a4ZQrAbN1q9o=
5151
github.com/containerd/typeurl/v2 v2.2.3 h1:yNA/94zxWdvYACdYO8zofhrTVuQY73fFU1y++dYSw40=
5252
github.com/containerd/typeurl/v2 v2.2.3/go.mod h1:95ljDnPfD3bAbDJRugOiShd/DlAAsxGtUBhJxIn7SCk=
53-
github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs=
54-
github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
53+
github.com/coreos/go-systemd/v22 v22.6.0 h1:aGVa/v8B7hpb0TKl0MWoAavPDmHvobFe5R5zn0bCJWo=
54+
github.com/coreos/go-systemd/v22 v22.6.0/go.mod h1:iG+pp635Fo7ZmV/j14KUcmEyWF+0X7Lua8rrTWzYgWU=
5555
github.com/cyphar/filepath-securejoin v0.5.1 h1:eYgfMq5yryL4fbWfkLpFFy2ukSELzaJOTaUTuh+oF48=
5656
github.com/cyphar/filepath-securejoin v0.5.1/go.mod h1:Sdj7gXlvMcPZsbhwhQ33GguGLDGQL7h7bg04C/+u9jI=
5757
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -74,7 +74,6 @@ github.com/go-logr/logr v1.4.3 h1:CjnDlHq8ikf6E492q6eKboGOC0T8CDaOvkHCIg8idEI=
7474
github.com/go-logr/logr v1.4.3/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY=
7575
github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
7676
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
77-
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
7877
github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk=
7978
github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
8079
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=
@@ -116,8 +115,8 @@ github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A=
116115
github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc=
117116
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA=
118117
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ=
119-
github.com/opencontainers/cgroups v0.0.4 h1:XVj8P/IHVms/j+7eh8ggdkTLAxjz84ZzuFyGoE28DR4=
120-
github.com/opencontainers/cgroups v0.0.4/go.mod h1:s8lktyhlGUqM7OSRL5P7eAW6Wb+kWPNvt4qvVfzA5vs=
118+
github.com/opencontainers/cgroups v0.0.6 h1:tfZFWTIIGaUUFImTyuTg+Mr5x8XRiSdZESgEBW7UxuI=
119+
github.com/opencontainers/cgroups v0.0.6/go.mod h1:oWVzJsKK0gG9SCRBfTpnn16WcGEqDI8PAcpMGbqWxcs=
121120
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
122121
github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
123122
github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040=

info/v1/container.go

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -374,6 +374,10 @@ type DiskIoStats struct {
374374
IoWaitTime []PerDiskStats `json:"io_wait_time,omitempty"`
375375
IoMerged []PerDiskStats `json:"io_merged,omitempty"`
376376
IoTime []PerDiskStats `json:"io_time,omitempty"`
377+
IoCostUsage []PerDiskStats `json:"io_cost_usage,omitempty"`
378+
IoCostWait []PerDiskStats `json:"io_cost_wait,omitempty"`
379+
IoCostIndebt []PerDiskStats `json:"io_cost_indebt,omitempty"`
380+
IoCostIndelay []PerDiskStats `json:"io_cost_indelay,omitempty"`
377381
PSI PSIStats `json:"psi"`
378382
}
379383

metrics/prometheus.go

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -755,6 +755,54 @@ func NewPrometheusCollector(i infoProvider, f ContainerLabelsFunc, includedMetri
755755
return float64(fs.WeightedIoTime) / float64(time.Second)
756756
}, s.Timestamp)
757757
},
758+
}, {
759+
name: "container_fs_io_cost_usage_seconds_total",
760+
help: "Cumulative IOCost usage in seconds",
761+
valueType: prometheus.CounterValue,
762+
extraLabels: []string{"device"},
763+
getValues: func(s *info.ContainerStats) metricValues {
764+
return ioValues(
765+
s.DiskIo.IoCostUsage, "Count", asMicrosecondsToSeconds,
766+
[]info.FsStats{}, nil,
767+
s.Timestamp,
768+
)
769+
},
770+
}, {
771+
name: "container_fs_io_cost_wait_seconds_total",
772+
help: "Cumulative IOCost wait in seconds",
773+
valueType: prometheus.CounterValue,
774+
extraLabels: []string{"device"},
775+
getValues: func(s *info.ContainerStats) metricValues {
776+
return ioValues(
777+
s.DiskIo.IoCostWait, "Count", asMicrosecondsToSeconds,
778+
[]info.FsStats{}, nil,
779+
s.Timestamp,
780+
)
781+
},
782+
}, {
783+
name: "container_fs_io_cost_indebt_seconds_total",
784+
help: "Cumulative IOCost debt in seconds",
785+
valueType: prometheus.CounterValue,
786+
extraLabels: []string{"device"},
787+
getValues: func(s *info.ContainerStats) metricValues {
788+
return ioValues(
789+
s.DiskIo.IoCostIndebt, "Count", asMicrosecondsToSeconds,
790+
[]info.FsStats{}, nil,
791+
s.Timestamp,
792+
)
793+
},
794+
}, {
795+
name: "container_fs_io_cost_indelay_seconds_total",
796+
help: "Cumulative IOCost delay in seconds",
797+
valueType: prometheus.CounterValue,
798+
extraLabels: []string{"device"},
799+
getValues: func(s *info.ContainerStats) metricValues {
800+
return ioValues(
801+
s.DiskIo.IoCostIndelay, "Count", asMicrosecondsToSeconds,
802+
[]info.FsStats{}, nil,
803+
s.Timestamp,
804+
)
805+
},
758806
},
759807
{
760808
name: "container_blkio_device_usage_total",

metrics/prometheus_fake.go

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -578,6 +578,30 @@ func (p testSubcontainersInfoProvider) GetRequestedContainersInfo(string, v2.Req
578578
"Write": 6,
579579
},
580580
}},
581+
IoCostUsage: []info.PerDiskStats{{
582+
Device: "sda1",
583+
Major: 8,
584+
Minor: 1,
585+
Stats: map[string]uint64{"Count": 1500000},
586+
}},
587+
IoCostWait: []info.PerDiskStats{{
588+
Device: "sda1",
589+
Major: 8,
590+
Minor: 1,
591+
Stats: map[string]uint64{"Count": 2500000},
592+
}},
593+
IoCostIndebt: []info.PerDiskStats{{
594+
Device: "sda1",
595+
Major: 8,
596+
Minor: 1,
597+
Stats: map[string]uint64{"Count": 500000},
598+
}},
599+
IoCostIndelay: []info.PerDiskStats{{
600+
Device: "sda1",
601+
Major: 8,
602+
Minor: 1,
603+
Stats: map[string]uint64{"Count": 750000},
604+
}},
581605
PSI: info.PSIStats{
582606
Full: info.PSIData{
583607
Avg10: 0.3,

metrics/prometheus_test.go

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -356,3 +356,75 @@ func TestGetContainerHealthState(t *testing.T) {
356356
})
357357
}
358358
}
359+
360+
func TestIOCostMetrics(t *testing.T) {
361+
containerStats := &info.ContainerStats{
362+
Timestamp: time.Unix(1395066363, 0),
363+
DiskIo: info.DiskIoStats{
364+
IoCostUsage: []info.PerDiskStats{{
365+
Device: "sda1",
366+
Major: 8,
367+
Minor: 1,
368+
Stats: map[string]uint64{"Count": 1500000},
369+
}},
370+
IoCostWait: []info.PerDiskStats{{
371+
Device: "sda1",
372+
Major: 8,
373+
Minor: 1,
374+
Stats: map[string]uint64{"Count": 2500000},
375+
}},
376+
IoCostIndebt: []info.PerDiskStats{{
377+
Device: "sda1",
378+
Major: 8,
379+
Minor: 1,
380+
Stats: map[string]uint64{"Count": 500000},
381+
}},
382+
IoCostIndelay: []info.PerDiskStats{{
383+
Device: "sda1",
384+
Major: 8,
385+
Minor: 1,
386+
Stats: map[string]uint64{"Count": 750000},
387+
}},
388+
},
389+
}
390+
391+
testCases := []struct {
392+
name string
393+
stats []info.PerDiskStats
394+
expectedValue float64
395+
}{
396+
{
397+
name: "IoCostUsage",
398+
stats: containerStats.DiskIo.IoCostUsage,
399+
expectedValue: 1.5,
400+
},
401+
{
402+
name: "IoCostWait",
403+
stats: containerStats.DiskIo.IoCostWait,
404+
expectedValue: 2.5,
405+
},
406+
{
407+
name: "IoCostIndebt",
408+
stats: containerStats.DiskIo.IoCostIndebt,
409+
expectedValue: 0.5,
410+
},
411+
{
412+
name: "IoCostIndelay",
413+
stats: containerStats.DiskIo.IoCostIndelay,
414+
expectedValue: 0.75,
415+
},
416+
}
417+
418+
for _, tc := range testCases {
419+
t.Run(tc.name, func(t *testing.T) {
420+
values := ioValues(
421+
tc.stats, "Count", asMicrosecondsToSeconds,
422+
[]info.FsStats{}, nil,
423+
containerStats.Timestamp,
424+
)
425+
assert.Equal(t, 1, len(values))
426+
assert.Equal(t, tc.expectedValue, values[0].value)
427+
assert.Equal(t, []string{"sda1"}, values[0].labels)
428+
})
429+
}
430+
}

0 commit comments

Comments
 (0)