diff --git a/fdbkubernetesmonitor/metrics.go b/fdbkubernetesmonitor/metrics.go new file mode 100644 index 00000000000..d652c23f958 --- /dev/null +++ b/fdbkubernetesmonitor/metrics.go @@ -0,0 +1,140 @@ +// metrics.go +// +// This source file is part of the FoundationDB open source project +// +// Copyright 2021-2024 Apple Inc. and the FoundationDB project authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "strconv" + + "github.com/prometheus/client_golang/prometheus" +) + +const ( + // versionLabel represents the version label for the prometheus metrics. + versionLabel = "version" + // processLabel represents the process label for the prometheus metrics. + processLabel = "process" + // namespace is the prometheus namespace for the metrics + namespace = "fdbkubernetesmonitor" + // restartCountMetricName represents the name of the restart metric. + restartCountMetricName = "restart_count" + // configurationChangeCountMetricName represents the configuration_change_count metric. + configurationChangeCountMetricName = "configuration_change_count" + // lastAppliedConfigurationTimestampMetricName represents the last_applied_configuration_timestamp metric. + lastAppliedConfigurationTimestampMetricName = "last_applied_configuration_timestamp" + // startTimestampMetricName represents the start_timestamp metric. + startTimestampMetricName = "start_timestamp" + // runningVersionMetricName represents the running_version metric. + runningVersionMetricName = "running_version" + // desiredVersionMetricName represents the desired_version metric. + desiredVersionMetricName = "desired_version" +) + +// metrics represents the custom prometheus metrics for the Monitor. +type metrics struct { + // restartCount represents the total number of fdbserver process restarts. + restartCount *prometheus.CounterVec + // configurationChangeCount represents the total number of observed configuration changes. + configurationChangeCount prometheus.Counter + // lastAppliedConfigurationTimestamp provides a unix timestamp when the last configuration was applied. + lastAppliedConfigurationTimestamp prometheus.Gauge + // startTimestamp represents the timestamp when a specific process was started. + startTimestamp *prometheus.GaugeVec + // runningVersion represents the current running version of the binaries. + runningVersion *prometheus.GaugeVec + // desiredVersion represents the desired running version of the binaries. + desiredVersion *prometheus.GaugeVec + // previousDesiredVersion keeps the previous seen desired version. + previousDesiredVersion string + // previousRunningVersion keeps the prvious seen running version. + previousRunningVersion string +} + +// registerConfigurationChange will update the current prometheus metrics related to configuration changes. +// This method doesn't use latch as it will only be called from acceptConfiguration which already uses a mutex. +func (metrics *metrics) registerConfigurationChange(version string) { + // Update the prometheus metrics. + metrics.lastAppliedConfigurationTimestamp.SetToCurrentTime() + metrics.configurationChangeCount.Inc() + // Update the desired version for the fdbserver process. + metrics.desiredVersion.With(prometheus.Labels{versionLabel: version}).Set(1.0) + // If we had a different desired version before we want to make sure to reset this value. + if metrics.previousDesiredVersion != "" && metrics.previousDesiredVersion != version { + metrics.desiredVersion.With(prometheus.Labels{versionLabel: metrics.previousDesiredVersion}).Set(0.0) + } + metrics.previousDesiredVersion = version +} + +// registerProcessStartup will update the current prometheus metrics related to process startup and running versions. +func (metrics *metrics) registerProcessStartup(processNumber int, version string) { + castedProcessNumber := strconv.Itoa(processNumber) + metrics.restartCount.With(prometheus.Labels{processLabel: castedProcessNumber}).Inc() + metrics.startTimestamp.With(prometheus.Labels{processLabel: castedProcessNumber}).SetToCurrentTime() + metrics.runningVersion.With(prometheus.Labels{versionLabel: version}).Set(1.0) + // If we had a different desired version before we want to make sure to reset this value. + if metrics.previousRunningVersion != "" && metrics.previousRunningVersion != version { + metrics.runningVersion.With(prometheus.Labels{versionLabel: metrics.previousRunningVersion}).Set(0.0) + } + metrics.previousRunningVersion = version +} + +// registerMetrics will register the monitor metrics and returns a metrics struct to update the current metrics. +func registerMetrics(reg prometheus.Registerer) *metrics { + monitorMetrics := &metrics{ + restartCount: prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: namespace, + Name: restartCountMetricName, + Help: "Number of fdbserver process restarts in total.", + }, []string{processLabel}), + configurationChangeCount: prometheus.NewCounter(prometheus.CounterOpts{ + Namespace: namespace, + Name: configurationChangeCountMetricName, + Help: "Number of observed configuration changes.", + }), + lastAppliedConfigurationTimestamp: prometheus.NewGauge(prometheus.GaugeOpts{ + Namespace: namespace, + Name: lastAppliedConfigurationTimestampMetricName, + Help: "Timestamp when the last time the configuration was applied.", + }), + startTimestamp: prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: namespace, + Name: startTimestampMetricName, + Help: "Timestamp when the last time the configuration was applied.", + }, []string{processLabel}), + runningVersion: prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: namespace, + Name: runningVersionMetricName, + Help: "The current running version of the fdbserver processes started by this monitor.", + }, []string{versionLabel}), + desiredVersion: prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Namespace: namespace, + Name: desiredVersionMetricName, + Help: "The desired running version of the fdbserver processes started by this monitor.", + }, []string{versionLabel}), + } + + reg.MustRegister(monitorMetrics.restartCount) + reg.MustRegister(monitorMetrics.configurationChangeCount) + reg.MustRegister(monitorMetrics.lastAppliedConfigurationTimestamp) + reg.MustRegister(monitorMetrics.startTimestamp) + reg.MustRegister(monitorMetrics.runningVersion) + reg.MustRegister(monitorMetrics.desiredVersion) + + return monitorMetrics +} diff --git a/fdbkubernetesmonitor/metrics_test.go b/fdbkubernetesmonitor/metrics_test.go new file mode 100644 index 00000000000..8ccb58d2d88 --- /dev/null +++ b/fdbkubernetesmonitor/metrics_test.go @@ -0,0 +1,271 @@ +// metrics_test.go +// +// This source file is part of the FoundationDB open source project +// +// Copyright 2021-2024 Apple Inc. and the FoundationDB project authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "strings" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + "github.com/prometheus/client_golang/prometheus" + "k8s.io/utils/pointer" +) + +var _ = Describe("Testing Monitor metrics", func() { + When("getting the copy details", func() { + sevenOneVersion := "7.1.57" + sevenThreeVersion := "7.3.37" + var registry *prometheus.Registry + var monitorMetrics *metrics + + BeforeEach(func() { + registry = prometheus.NewRegistry() + monitorMetrics = registerMetrics(registry) + }) + + It("shouldn't throw any error", func() { + Expect(monitorMetrics).NotTo(BeNil()) + }) + + When("no metrics are added", func() { + It("only the counters should be setup", func() { + metrics, err := registry.Gather() + Expect(err).NotTo(HaveOccurred()) + Expect(metrics).To(HaveLen(2)) + }) + }) + + When("a configuration change is registered", func() { + BeforeEach(func() { + monitorMetrics.registerConfigurationChange(sevenOneVersion) + }) + + It("should update the metrics", func() { + metrics, err := registry.Gather() + Expect(err).NotTo(HaveOccurred()) + Expect(metrics).To(HaveLen(3)) + + for _, metric := range metrics { + Expect(metric.Metric).To(HaveLen(1)) + name := pointer.StringDeref(metric.Name, "") + if strings.HasSuffix(name, configurationChangeCountMetricName) { + Expect(*metric.Metric[0].Counter.Value).To(BeNumerically("==", 1)) + continue + } + if strings.HasSuffix(name, desiredVersionMetricName) { + Expect(*metric.Metric[0].Gauge.Value).To(BeNumerically("==", 1)) + Expect(metric.Metric[0].Label).To(HaveLen(1)) + Expect(*metric.Metric[0].Label[0].Name).To(Equal(versionLabel)) + Expect(*metric.Metric[0].Label[0].Value).To(Equal(sevenOneVersion)) + continue + } + } + }) + + When("a new configuration change is registered for the same version", func() { + BeforeEach(func() { + monitorMetrics.registerConfigurationChange(sevenOneVersion) + }) + + It("should update the metrics", func() { + metrics, err := registry.Gather() + Expect(err).NotTo(HaveOccurred()) + Expect(metrics).To(HaveLen(3)) + + for _, metric := range metrics { + Expect(metric.Metric).To(HaveLen(1)) + name := pointer.StringDeref(metric.Name, "") + if strings.HasSuffix(name, configurationChangeCountMetricName) { + Expect(*metric.Metric[0].Counter.Value).To(BeNumerically("==", 2)) + continue + } + if strings.HasSuffix(name, desiredVersionMetricName) { + Expect(*metric.Metric[0].Gauge.Value).To(BeNumerically("==", 1)) + Expect(metric.Metric[0].Label).To(HaveLen(1)) + Expect(*metric.Metric[0].Label[0].Name).To(Equal(versionLabel)) + Expect(*metric.Metric[0].Label[0].Value).To(Equal(sevenOneVersion)) + continue + } + } + }) + }) + + When("a new configuration change is registered for a different version", func() { + BeforeEach(func() { + monitorMetrics.registerConfigurationChange(sevenThreeVersion) + }) + + It("should update the metrics", func() { + metrics, err := registry.Gather() + Expect(err).NotTo(HaveOccurred()) + Expect(metrics).To(HaveLen(3)) + + for _, metric := range metrics { + name := pointer.StringDeref(metric.Name, "") + if strings.HasSuffix(name, desiredVersionMetricName) { + Expect(metric.Metric).To(HaveLen(2)) + for _, versionMetric := range metric.Metric { + Expect(versionMetric.Label).To(HaveLen(1)) + Expect(*versionMetric.Label[0].Name).To(Equal(versionLabel)) + expected := 0 + if *versionMetric.Label[0].Value == sevenThreeVersion { + expected = 1 + } + Expect(*versionMetric.Gauge.Value).To(BeNumerically("==", expected)) + } + continue + } + + Expect(metric.Metric).To(HaveLen(1)) + if strings.HasSuffix(name, configurationChangeCountMetricName) { + Expect(*metric.Metric[0].Counter.Value).To(BeNumerically("==", 2)) + continue + } + } + }) + }) + }) + + When("a process startup registered", func() { + BeforeEach(func() { + monitorMetrics.registerProcessStartup(1, sevenOneVersion) + }) + + It("should update the metrics", func() { + metrics, err := registry.Gather() + Expect(err).NotTo(HaveOccurred()) + Expect(metrics).To(HaveLen(5)) + + for _, metric := range metrics { + Expect(metric.Metric).To(HaveLen(1)) + name := pointer.StringDeref(metric.Name, "") + if strings.HasSuffix(name, restartCountMetricName) { + Expect(*metric.Metric[0].Counter.Value).To(BeNumerically("==", 1)) + Expect(metric.Metric[0].Label).To(HaveLen(1)) + Expect(*metric.Metric[0].Label[0].Name).To(Equal(processLabel)) + Expect(*metric.Metric[0].Label[0].Value).To(Equal("1")) + continue + } + if strings.HasSuffix(name, runningVersionMetricName) { + Expect(*metric.Metric[0].Gauge.Value).To(BeNumerically("==", 1)) + Expect(metric.Metric[0].Label).To(HaveLen(1)) + Expect(*metric.Metric[0].Label[0].Name).To(Equal(versionLabel)) + Expect(*metric.Metric[0].Label[0].Value).To(Equal(sevenOneVersion)) + continue + } + } + }) + + When("the same process is restarted with the same version", func() { + BeforeEach(func() { + monitorMetrics.registerProcessStartup(1, sevenOneVersion) + }) + + It("should update the metrics", func() { + metrics, err := registry.Gather() + Expect(err).NotTo(HaveOccurred()) + Expect(metrics).To(HaveLen(5)) + + for _, metric := range metrics { + Expect(metric.Metric).To(HaveLen(1)) + name := pointer.StringDeref(metric.Name, "") + if strings.HasSuffix(name, restartCountMetricName) { + Expect(*metric.Metric[0].Counter.Value).To(BeNumerically("==", 2)) + Expect(metric.Metric[0].Label).To(HaveLen(1)) + Expect(*metric.Metric[0].Label[0].Name).To(Equal(processLabel)) + Expect(*metric.Metric[0].Label[0].Value).To(Equal("1")) + continue + } + if strings.HasSuffix(name, runningVersionMetricName) { + Expect(*metric.Metric[0].Gauge.Value).To(BeNumerically("==", 1)) + Expect(metric.Metric[0].Label).To(HaveLen(1)) + Expect(*metric.Metric[0].Label[0].Name).To(Equal(versionLabel)) + Expect(*metric.Metric[0].Label[0].Value).To(Equal(sevenOneVersion)) + continue + } + } + }) + }) + + When("the same process is restarted with a different version", func() { + BeforeEach(func() { + monitorMetrics.registerProcessStartup(1, sevenThreeVersion) + }) + + It("should update the metrics", func() { + metrics, err := registry.Gather() + Expect(err).NotTo(HaveOccurred()) + Expect(metrics).To(HaveLen(5)) + + for _, metric := range metrics { + name := pointer.StringDeref(metric.Name, "") + if strings.HasSuffix(name, runningVersionMetricName) { + Expect(metric.Metric).To(HaveLen(2)) + for _, versionMetric := range metric.Metric { + Expect(versionMetric.Label).To(HaveLen(1)) + Expect(*versionMetric.Label[0].Name).To(Equal(versionLabel)) + expected := 0 + if *versionMetric.Label[0].Value == sevenThreeVersion { + expected = 1 + } + Expect(*versionMetric.Gauge.Value).To(BeNumerically("==", expected)) + } + continue + } + + Expect(metric.Metric).To(HaveLen(1)) + if strings.HasSuffix(name, restartCountMetricName) { + Expect(*metric.Metric[0].Counter.Value).To(BeNumerically("==", 2)) + Expect(metric.Metric[0].Label).To(HaveLen(1)) + Expect(*metric.Metric[0].Label[0].Name).To(Equal(processLabel)) + Expect(*metric.Metric[0].Label[0].Value).To(Equal("1")) + continue + } + } + }) + }) + + When("a new process is restarted with a the same version", func() { + BeforeEach(func() { + monitorMetrics.registerProcessStartup(2, sevenOneVersion) + }) + + It("should update the metrics", func() { + metrics, err := registry.Gather() + Expect(err).NotTo(HaveOccurred()) + Expect(metrics).To(HaveLen(5)) + + for _, metric := range metrics { + name := pointer.StringDeref(metric.Name, "") + + if strings.HasSuffix(name, restartCountMetricName) { + Expect(metric.Metric).To(HaveLen(2)) + for _, versionMetric := range metric.Metric { + Expect(versionMetric.Label).To(HaveLen(1)) + Expect(*versionMetric.Label[0].Name).To(Equal(processLabel)) + Expect(*versionMetric.Counter.Value).To(BeNumerically("==", 1)) + } + continue + } + } + }) + }) + }) + }) +}) diff --git a/fdbkubernetesmonitor/monitor.go b/fdbkubernetesmonitor/monitor.go index 1662a2fc48e..e7ffeb4280c 100644 --- a/fdbkubernetesmonitor/monitor.go +++ b/fdbkubernetesmonitor/monitor.go @@ -38,6 +38,8 @@ import ( "github.com/apple/foundationdb/fdbkubernetesmonitor/api" "github.com/fsnotify/fsnotify" "github.com/go-logr/logr" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/collectors" "github.com/prometheus/client_golang/prometheus/promhttp" "k8s.io/apimachinery/pkg/api/equality" "k8s.io/utils/pointer" @@ -90,6 +92,9 @@ type Monitor struct { // Logger is the logger instance for this monitor. Logger logr.Logger + + // metrics represents the prometheus monitor metrics. + metrics *metrics } // StartMonitor starts the monitor loop. @@ -128,8 +133,15 @@ func StartMonitor(ctx context.Context, logger logr.Logger, configFile string, cu mux.HandleFunc("/debug/pprof/trace", pprof.Trace) } + reg := prometheus.NewRegistry() + // Enable the default go metrics. + reg.MustRegister(collectors.NewGoCollector()) + monitorMetrics := registerMetrics(reg) + monitor.metrics = monitorMetrics + promHandler := promhttp.HandlerFor(reg, promhttp.HandlerOpts{}) + // Add Prometheus support - mux.Handle("/metrics", promhttp.Handler()) + mux.Handle("/metrics", promHandler) go func() { err := http.ListenAndServe(listenAddr, mux) if err != nil { @@ -209,6 +221,8 @@ func (monitor *Monitor) acceptConfiguration(configuration *api.ProcessConfigurat monitor.ActiveConfiguration = configuration monitor.ActiveConfigurationBytes = configurationBytes monitor.LastConfigurationTime = time.Now() + // Update the prometheus metrics. + monitor.metrics.registerConfigurationChange(configuration.Version) for processNumber := 1; processNumber <= monitor.ProcessCount; processNumber++ { if monitor.ProcessIDs[processNumber] == 0 { @@ -289,6 +303,9 @@ func (monitor *Monitor) RunProcess(processNumber int) { continue } + // Update the prometheus metrics for the process. + monitor.metrics.registerProcessStartup(processNumber, monitor.ActiveConfiguration.Version) + if cmd.Process != nil { pid = cmd.Process.Pid } else {