From 101a730836242a859106b42123609c4496e70600 Mon Sep 17 00:00:00 2001 From: ScarletBlizzard Date: Thu, 6 Nov 2025 00:48:17 +0300 Subject: [PATCH 1/4] Add collector for hung_task_detect_count Signed-off-by: ScarletBlizzard --- README.md | 1 + collector/kernel_hung_linux.go | 62 ++++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+) create mode 100644 collector/kernel_hung_linux.go diff --git a/README.md b/README.md index 3761bcc93c..ec5adbf7ac 100644 --- a/README.md +++ b/README.md @@ -195,6 +195,7 @@ drm | Expose GPU metrics using sysfs / DRM, `amdgpu` is the only driver which ex drbd | Exposes Distributed Replicated Block Device statistics (to version 8.4) | Linux ethtool | Exposes network interface information and network driver statistics equivalent to `ethtool`, `ethtool -S`, and `ethtool -i`. | Linux interrupts | Exposes detailed interrupts statistics. | Linux, OpenBSD +kernel_hung | Exposes number of tasks that have been detected as hung from `/proc/sys/kernel/hung_task_detect_count`. | Linux ksmd | Exposes kernel and system statistics from `/sys/kernel/mm/ksm`. | Linux lnstat | Exposes stats from `/proc/net/stat/`. | Linux logind | Exposes session counts from [logind](http://www.freedesktop.org/wiki/Software/systemd/logind/). | Linux diff --git a/collector/kernel_hung_linux.go b/collector/kernel_hung_linux.go new file mode 100644 index 0000000000..606c48567f --- /dev/null +++ b/collector/kernel_hung_linux.go @@ -0,0 +1,62 @@ +// Copyright 2018 The Prometheus Authors +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +//go:build !noprocesses +// +build !noprocesses + +package collector + +import ( + "fmt" + "log/slog" + + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/procfs" +) + +type kernelHungCollector struct { + fs procfs.FS + taskDetectCount *prometheus.Desc + logger *slog.Logger +} + +func init() { + registerCollector("kernel_hung", defaultDisabled, NewKernelHungCollector) +} + +func NewKernelHungCollector(logger *slog.Logger) (Collector, error) { + fs, err := procfs.NewFS(*procPath) + if err != nil { + return nil, fmt.Errorf("failed to open procfs: %w", err) + } + return &kernelHungCollector{ + fs: fs, + taskDetectCount: prometheus.NewDesc( + prometheus.BuildFQName(namespace, "kernel_hung", "task_detect_count"), + "Total number of interrupts serviced.", + nil, nil, + ), + logger: logger, + }, nil +} + +func (c *kernelHungCollector) Update(ch chan<- prometheus.Metric) error { + kernelHung, err := c.fs.KernelHung() + if err != nil { + return err + } + + ch <- prometheus.MustNewConstMetric(c.taskDetectCount, prometheus.CounterValue, float64(*kernelHung.HungTaskDetectCount)) + + return nil +} From c87cc88b06b4311abd8f2a24faa425880803f448 Mon Sep 17 00:00:00 2001 From: ScarletBlizzard Date: Wed, 12 Nov 2025 22:47:56 +0300 Subject: [PATCH 2/4] Remove prometheus.Desc from collector struct Signed-off-by: ScarletBlizzard --- collector/kernel_hung_linux.go | 18 ++++++++---------- 1 file changed, 8 insertions(+), 10 deletions(-) diff --git a/collector/kernel_hung_linux.go b/collector/kernel_hung_linux.go index 606c48567f..2c5721a89f 100644 --- a/collector/kernel_hung_linux.go +++ b/collector/kernel_hung_linux.go @@ -25,9 +25,8 @@ import ( ) type kernelHungCollector struct { - fs procfs.FS - taskDetectCount *prometheus.Desc - logger *slog.Logger + fs procfs.FS + logger *slog.Logger } func init() { @@ -40,12 +39,7 @@ func NewKernelHungCollector(logger *slog.Logger) (Collector, error) { return nil, fmt.Errorf("failed to open procfs: %w", err) } return &kernelHungCollector{ - fs: fs, - taskDetectCount: prometheus.NewDesc( - prometheus.BuildFQName(namespace, "kernel_hung", "task_detect_count"), - "Total number of interrupts serviced.", - nil, nil, - ), + fs: fs, logger: logger, }, nil } @@ -56,7 +50,11 @@ func (c *kernelHungCollector) Update(ch chan<- prometheus.Metric) error { return err } - ch <- prometheus.MustNewConstMetric(c.taskDetectCount, prometheus.CounterValue, float64(*kernelHung.HungTaskDetectCount)) + ch <- prometheus.MustNewConstMetric(prometheus.NewDesc( + prometheus.BuildFQName(namespace, "kernel_hung", "task_detect_count"), + "Total number of interrupts serviced.", + nil, nil, + ), prometheus.CounterValue, float64(*kernelHung.HungTaskDetectCount)) return nil } From e80ddc6cbfbbaaf82f1bc91d869f5392c12b5e67 Mon Sep 17 00:00:00 2001 From: ScarletBlizzard Date: Wed, 12 Nov 2025 23:00:10 +0300 Subject: [PATCH 3/4] Make taskDetectCount a package var Signed-off-by: ScarletBlizzard --- collector/kernel_hung_linux.go | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/collector/kernel_hung_linux.go b/collector/kernel_hung_linux.go index 2c5721a89f..0bf618d3ac 100644 --- a/collector/kernel_hung_linux.go +++ b/collector/kernel_hung_linux.go @@ -44,17 +44,21 @@ func NewKernelHungCollector(logger *slog.Logger) (Collector, error) { }, nil } +var ( + taskDetectCount = prometheus.NewDesc( + prometheus.BuildFQName(namespace, "kernel_hung", "task_detect_count"), + "Total number of interrupts serviced.", + nil, nil, + ) +) + func (c *kernelHungCollector) Update(ch chan<- prometheus.Metric) error { kernelHung, err := c.fs.KernelHung() if err != nil { return err } - ch <- prometheus.MustNewConstMetric(prometheus.NewDesc( - prometheus.BuildFQName(namespace, "kernel_hung", "task_detect_count"), - "Total number of interrupts serviced.", - nil, nil, - ), prometheus.CounterValue, float64(*kernelHung.HungTaskDetectCount)) + ch <- prometheus.MustNewConstMetric(taskDetectCount, prometheus.CounterValue, float64(*kernelHung.HungTaskDetectCount)) return nil } From 88000364f0f46b20296780f1299f754d26f98e7d Mon Sep 17 00:00:00 2001 From: ScarletBlizzard Date: Wed, 12 Nov 2025 23:01:02 +0300 Subject: [PATCH 4/4] Remove '+ build' line Signed-off-by: ScarletBlizzard --- collector/kernel_hung_linux.go | 1 - 1 file changed, 1 deletion(-) diff --git a/collector/kernel_hung_linux.go b/collector/kernel_hung_linux.go index 0bf618d3ac..cf842d0b1e 100644 --- a/collector/kernel_hung_linux.go +++ b/collector/kernel_hung_linux.go @@ -12,7 +12,6 @@ // limitations under the License. //go:build !noprocesses -// +build !noprocesses package collector