diff --git a/Cargo.lock b/Cargo.lock index ec1a6d886e9f..5d91bdc729e7 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -9558,6 +9558,14 @@ version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b5c7628eac357aecda461130f8074468be5aa4d258a002032d82d817f79f1f8" +[[package]] +name = "ic-metrics-tool" +version = "0.1.0" +dependencies = [ + "anyhow", + "clap 4.5.20", +] + [[package]] name = "ic-nervous-system-agent" version = "0.0.1" diff --git a/Cargo.toml b/Cargo.toml index 6fed6e894fee..71e1486b89f8 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -126,6 +126,7 @@ members = [ "rs/ic_os/build_tools/diroid", "rs/ic_os/config", "rs/ic_os/fstrim_tool", + "rs/ic_os/metrics_tool", "rs/ic_os/os_tools/guestos_tool", "rs/ic_os/os_tools/hostos_tool", "rs/ic_os/build_tools/inject_files", diff --git a/ic-os/components/guestos.bzl b/ic-os/components/guestos.bzl index 5bf638bafc72..c21dcbf4976c 100644 --- a/ic-os/components/guestos.bzl +++ b/ic-os/components/guestos.bzl @@ -85,6 +85,8 @@ component_files = { Label("monitoring/journald.conf"): "/etc/systemd/journald.conf", Label("monitoring/nft-exporter/nft-exporter.service"): "/etc/systemd/system/nft-exporter.service", Label("monitoring/nft-exporter/nft-exporter.timer"): "/etc/systemd/system/nft-exporter.timer", + Label("monitoring/custom-metrics/metrics_tool.service"): "/etc/systemd/system/metrics_tool.service", + Label("monitoring/custom-metrics/metrics_tool.timer"): "/etc/systemd/system/metrics_tool.timer", # networking Label("networking/generate-network-config/guestos/generate-network-config.service"): "/etc/systemd/system/generate-network-config.service", diff --git a/ic-os/components/monitoring/custom-metrics/metrics_tool.service b/ic-os/components/monitoring/custom-metrics/metrics_tool.service new file mode 100644 index 000000000000..c03e1125ab83 --- /dev/null +++ b/ic-os/components/monitoring/custom-metrics/metrics_tool.service @@ -0,0 +1,33 @@ +[Unit] +Description=Report custom metrics once per minute + +[Service] +Type=oneshot +ExecStart=/opt/ic/bin/metrics_tool --metrics /run/node_exporter/collector_textfile/custom_metrics.prom +DeviceAllow=/dev/vda +IPAddressDeny=any +LockPersonality=yes +MemoryDenyWriteExecute=yes +NoNewPrivileges=yes +PrivateDevices=no +PrivateNetwork=yes +PrivateTmp=yes +PrivateUsers=no +ProtectClock=yes +ProtectControlGroups=yes +ProtectHome=yes +ProtectHostname=yes +ProtectKernelModules=yes +ProtectKernelTunables=yes +ProtectSystem=strict +ReadOnlyPaths=/proc/interrupts +ReadWritePaths=/run/node_exporter/collector_textfile +RestrictAddressFamilies=AF_UNIX +RestrictAddressFamilies=~AF_UNIX +RestrictNamespaces=yes +RestrictRealtime=yes +RestrictSUIDSGID=yes +SystemCallArchitectures=native +SystemCallErrorNumber=EPERM +SystemCallFilter=@system-service +UMask=022 diff --git a/ic-os/components/monitoring/custom-metrics/metrics_tool.timer b/ic-os/components/monitoring/custom-metrics/metrics_tool.timer new file mode 100644 index 000000000000..7015869a5f09 --- /dev/null +++ b/ic-os/components/monitoring/custom-metrics/metrics_tool.timer @@ -0,0 +1,10 @@ +[Unit] +Description=Collect custom metrics every minute + +[Timer] +OnBootSec=60s +OnUnitActiveSec=60s +Unit=metrics_tool.service + +[Install] +WantedBy=timers.target diff --git a/ic-os/guestos/defs.bzl b/ic-os/guestos/defs.bzl index 30e3d49787be..177302685d2c 100644 --- a/ic-os/guestos/defs.bzl +++ b/ic-os/guestos/defs.bzl @@ -50,6 +50,7 @@ def image_deps(mode, malicious = False): "//cpp:infogetty": "/opt/ic/bin/infogetty:0755", # Terminal manager that replaces the login shell. "//cpp:prestorecon": "/opt/ic/bin/prestorecon:0755", # Parallel restorecon replacement for filesystem relabeling. "//rs/ic_os/release:metrics-proxy": "/opt/ic/bin/metrics-proxy:0755", # Proxies, filters, and serves public node metrics. + "//rs/ic_os/release:metrics_tool": "/opt/ic/bin/metrics_tool:0755", # Collects and reports custom metrics. # additional libraries to install "//rs/ic_os/release:nss_icos": "/usr/lib/x86_64-linux-gnu/libnss_icos.so.2:0644", # Allows referring to the guest IPv6 by name guestos from host, and host as hostos from guest. diff --git a/rs/ic_os/metrics_tool/BUILD.bazel b/rs/ic_os/metrics_tool/BUILD.bazel new file mode 100644 index 000000000000..b182a7a8a935 --- /dev/null +++ b/rs/ic_os/metrics_tool/BUILD.bazel @@ -0,0 +1,54 @@ +load("@rules_rust//rust:defs.bzl", "rust_binary", "rust_library", "rust_test", "rust_test_suite") + +package(default_visibility = ["//rs:ic-os-pkg"]) + +DEPENDENCIES = [ + # Keep sorted. + "//rs/sys", + "@crate_index//:anyhow", + "@crate_index//:clap", +] + +DEV_DEPENDENCIES = [ + # Keep sorted. +] + +MACRO_DEPENDENCIES = [] + +ALIASES = {} + +rust_library( + name = "metrics_tool", + srcs = glob( + ["src/**/*.rs"], + exclude = ["src/main.rs"], + ), + aliases = ALIASES, + crate_name = "ic_metrics_tool", + proc_macro_deps = MACRO_DEPENDENCIES, + visibility = ["//rs:system-tests-pkg"], + deps = DEPENDENCIES, +) + +rust_binary( + name = "metrics_tool_bin", + srcs = ["src/main.rs"], + aliases = ALIASES, + proc_macro_deps = MACRO_DEPENDENCIES, + deps = DEPENDENCIES + [":metrics_tool"], +) + +rust_test( + name = "metrics_tool_test", + crate = ":metrics_tool", + deps = DEPENDENCIES + DEV_DEPENDENCIES, +) + +rust_test_suite( + name = "metrics_tool_integration", + srcs = glob(["tests/**/*.rs"]), + target_compatible_with = [ + "@platforms//os:linux", + ], + deps = [":metrics_tool_bin"] + DEPENDENCIES + DEV_DEPENDENCIES, +) diff --git a/rs/ic_os/metrics_tool/Cargo.toml b/rs/ic_os/metrics_tool/Cargo.toml new file mode 100644 index 000000000000..4e6b604e829f --- /dev/null +++ b/rs/ic_os/metrics_tool/Cargo.toml @@ -0,0 +1,12 @@ +[package] +name = "ic-metrics-tool" +version = "0.1.0" +edition = "2021" + +[[bin]] +name = "metrics_tool" +path = "src/main.rs" + +[dependencies] +anyhow = { workspace = true } +clap = { workspace = true } \ No newline at end of file diff --git a/rs/ic_os/metrics_tool/src/lib.rs b/rs/ic_os/metrics_tool/src/lib.rs new file mode 100644 index 000000000000..02bb129c55dd --- /dev/null +++ b/rs/ic_os/metrics_tool/src/lib.rs @@ -0,0 +1,187 @@ +// TODO: refactor/merge this with fstrim_tool and guestos_tool metrics functionality +use std::fs::File; +use std::io::{self, Write}; +use std::path::Path; + +// TODO: everything is floating point for now +pub struct Metric { + name: String, + value: f64, + annotation: String, + labels: Vec<(String, String)>, +} + +impl Metric { + pub fn new(name: &str, value: f64) -> Self { + Self { + name: name.to_string(), + value, + annotation: "Custom metric".to_string(), + labels: Vec::new(), + } + } + pub fn with_annotation(name: &str, value: f64, annotation: &str) -> Self { + Self { + name: name.to_string(), + value, + annotation: annotation.to_string(), + labels: Vec::new(), + } + } + + pub fn add_annotation(mut self, annotation: &str) -> Self { + self.annotation = annotation.to_string(); + self + } + + pub fn add_label(mut self, key: &str, value: &str) -> Self { + self.labels.push((key.to_string(), value.to_string())); + self + } + + // TODO: formatting of floats + // Convert to prometheus exposition format + pub fn to_prom_string(&self) -> String { + let labels_str = if self.labels.is_empty() { + String::new() + } else { + let labels: Vec = self + .labels + .iter() + .map(|(k, v)| format!("{}=\"{}\"", k, v)) + .collect(); + format!("{{{}}}", labels.join(",")) + }; + format!( + "# HELP {} {}\n\ + # TYPE {} counter\n\ + {}{} {}", + self.name, self.annotation, self.name, self.name, labels_str, self.value + ) + } +} + +pub struct MetricsWriter { + file_path: String, +} + +impl MetricsWriter { + pub fn new(file_path: &str) -> Self { + Self { + file_path: file_path.to_string(), + } + } + + pub fn write_metrics(&self, metrics: &[Metric]) -> io::Result<()> { + let path = Path::new(&self.file_path); + let mut file = File::create(path)?; + for metric in metrics { + writeln!(file, "{}", metric.to_prom_string())?; + } + Ok(()) + } +} +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_metric_to_string() { + let metric = Metric::new("test_metric", 123.45) + .add_label("label1", "value1") + .add_label("label2", "value2"); + assert_eq!( + metric.to_prom_string(), + "# HELP test_metric Custom metric\n\ + # TYPE test_metric counter\n\ + test_metric{label1=\"value1\",label2=\"value2\"} 123.45" + ); + } + + #[test] + fn test_write_metrics() { + let metrics = vec![ + Metric::new("metric1", 1.0), + Metric::new("metric2", 2.0).add_label("label", "value"), + ]; + let writer = MetricsWriter::new("/tmp/test_metrics.prom"); + writer.write_metrics(&metrics).unwrap(); + let content = std::fs::read_to_string("/tmp/test_metrics.prom").unwrap(); + assert!(content.contains( + "# HELP metric1 Custom metric\n\ + # TYPE metric1 counter\n\ + metric1 1" + )); + assert!(content.contains( + "# HELP metric2 Custom metric\n\ + # TYPE metric2 counter\n\ + metric2{label=\"value\"} 2" + )); + } + + #[test] + fn test_metric_large_value() { + let metric = Metric::new("large_value_metric", 1.0e64); + assert_eq!( + metric.to_prom_string(), + "# HELP large_value_metric Custom metric\n\ + # TYPE large_value_metric counter\n\ + large_value_metric 10000000000000000000000000000000000000000000000000000000000000000" + ); + } + + #[test] + fn test_metric_without_labels() { + let metric = Metric::new("no_label_metric", 42.0); + assert_eq!( + metric.to_prom_string(), + "# HELP no_label_metric Custom metric\n\ + # TYPE no_label_metric counter\n\ + no_label_metric 42" + ); + } + + #[test] + fn test_metric_with_annotation() { + let metric = Metric::with_annotation("annotated_metric", 99.9, "This is a test metric"); + assert_eq!( + metric.to_prom_string(), + "# HELP annotated_metric This is a test metric\n\ + # TYPE annotated_metric counter\n\ + annotated_metric 99.9" + ); + } + + #[test] + fn test_write_empty_metrics() { + let metrics: Vec = Vec::new(); + let writer = MetricsWriter::new("/tmp/test_empty_metrics.prom"); + writer.write_metrics(&metrics).unwrap(); + let content = std::fs::read_to_string("/tmp/test_empty_metrics.prom").unwrap(); + assert!(content.is_empty()); + } + + #[test] + fn test_metric_with_multiple_labels() { + let metric = Metric::new("multi_label_metric", 10.0) + .add_label("foo", "bar") + .add_label("version", "1.0.0"); + assert_eq!( + metric.to_prom_string(), + "# HELP multi_label_metric Custom metric\n\ + # TYPE multi_label_metric counter\n\ + multi_label_metric{foo=\"bar\",version=\"1.0.0\"} 10" + ); + } + + #[test] + fn test_metric_with_empty_annotation() { + let metric = Metric::with_annotation("empty_annotation_metric", 5.5, ""); + assert_eq!( + metric.to_prom_string(), + "# HELP empty_annotation_metric \n\ + # TYPE empty_annotation_metric counter\n\ + empty_annotation_metric 5.5" + ); + } +} diff --git a/rs/ic_os/metrics_tool/src/main.rs b/rs/ic_os/metrics_tool/src/main.rs new file mode 100644 index 000000000000..cfd4a295c3fd --- /dev/null +++ b/rs/ic_os/metrics_tool/src/main.rs @@ -0,0 +1,62 @@ +use anyhow::Result; +use clap::Parser; + +use std::fs::File; +use std::io::{self, BufRead}; +use std::path::Path; + +use ic_metrics_tool::{Metric, MetricsWriter}; + +const INTERRUPT_FILTER: &str = "TLB shootdowns"; +const INTERRUPT_SOURCE: &str = "/proc/interrupts"; +const CUSTOM_METRICS_PROM: &str = "/run/node_exporter/collector_textfile/custom_metrics.prom"; +const TLB_SHOOTDOWN_METRIC_NAME: &str = "sum_tlb_shootdowns"; +const TLB_SHOOTDOWN_METRIC_ANNOTATION: &str = "Total TLB shootdowns"; + +#[derive(Parser)] +struct MetricToolArgs { + #[arg( + short = 'm', + long = "metrics", + default_value = CUSTOM_METRICS_PROM + )] + /// Filename to write the prometheus metrics for node_exporter generation. + /// Fails badly if the directory doesn't exist. + metrics_filename: String, +} + +fn get_sum_tlb_shootdowns() -> Result { + let path = Path::new(INTERRUPT_SOURCE); + let file = File::open(path)?; + let reader = io::BufReader::new(file); + + let mut total_tlb_shootdowns = 0; + + for line in reader.lines() { + let line = line?; + if line.contains(INTERRUPT_FILTER) { + for part in line.split_whitespace().skip(1) { + if let Ok(value) = part.parse::() { + total_tlb_shootdowns += value; + } + } + } + } + + Ok(total_tlb_shootdowns) +} + +pub fn main() -> Result<()> { + let opts = MetricToolArgs::parse(); + let mpath = Path::new(&opts.metrics_filename); + let tlb_shootdowns = get_sum_tlb_shootdowns()?; + + let metrics = vec![ + Metric::new(TLB_SHOOTDOWN_METRIC_NAME, tlb_shootdowns as f64) + .add_annotation(TLB_SHOOTDOWN_METRIC_ANNOTATION), + ]; + let writer = MetricsWriter::new(mpath.to_str().unwrap()); + writer.write_metrics(&metrics).unwrap(); + + Ok(()) +} diff --git a/rs/ic_os/release/BUILD.bazel b/rs/ic_os/release/BUILD.bazel index d5a5eb405988..902e58a30828 100644 --- a/rs/ic_os/release/BUILD.bazel +++ b/rs/ic_os/release/BUILD.bazel @@ -13,6 +13,7 @@ OBJECTS = { "vsock_host": "//rs/ic_os/vsock/host:vsock_host", "metrics-proxy": "@crate_index//:metrics-proxy__metrics-proxy", "nss_icos": "//rs/ic_os/nss_icos", + "metrics_tool": "//rs/ic_os/metrics_tool:metrics_tool_bin", } [release_strip_binary(