diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index b60ce477f..76c7c5c97 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -22,7 +22,7 @@ jobs: # Service containers to run with `build` (Required for end-to-end testing) services: influxdb: - image: influxdb:2.2 + image: influxdb:2.9 env: DOCKER_INFLUXDB_INIT_MODE: setup DOCKER_INFLUXDB_INIT_USERNAME: admin @@ -30,6 +30,7 @@ jobs: DOCKER_INFLUXDB_INIT_ORG: scrutiny DOCKER_INFLUXDB_INIT_BUCKET: metrics DOCKER_INFLUXDB_INIT_ADMIN_TOKEN: my-super-secret-auth-token + INFLUXD_USE_HASHED_TOKENS: "false" ports: - 8086:8086 env: diff --git a/.github/workflows/docker-build.yaml b/.github/workflows/docker-build.yaml index 3e847f771..d888b0c51 100644 --- a/.github/workflows/docker-build.yaml +++ b/.github/workflows/docker-build.yaml @@ -92,6 +92,66 @@ jobs: cache-from: type=gha,scope=docker-collector cache-to: type=gha,mode=max,scope=docker-collector + collector-omnibus: + runs-on: ubuntu-latest + timeout-minutes: 30 + permissions: + contents: read + packages: write + + steps: + - name: Checkout repository + uses: actions/checkout@v7 + with: + fetch-depth: 0 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v4 + with: + platforms: 'arm64,arm' + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v4 + + - name: Log into registry ${{ env.REGISTRY }} + if: github.event_name != 'pull_request' + uses: docker/login-action@v4 + with: + registry: ${{ env.REGISTRY }} + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Extract Docker metadata + id: meta + uses: docker/metadata-action@v6 + with: + images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} + flavor: | + latest=false + tags: | + # Manual trigger + type=raw,value=${{ inputs.tag_suffix }}-collector-omnibus,enable=${{ github.event_name == 'workflow_dispatch' }} + # Branch builds + type=raw,value=latest-collector-omnibus,enable=${{ (github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags/v')) && github.event_name != 'workflow_dispatch' }} + type=raw,value=beta-collector-omnibus,enable=${{ github.ref == 'refs/heads/beta' && github.event_name != 'workflow_dispatch' }} + type=raw,value=develop-collector-omnibus,enable=${{ github.ref == 'refs/heads/develop' && github.event_name != 'workflow_dispatch' }} + # Version tags + type=semver,pattern={{version}}-collector-omnibus + type=semver,pattern={{major}}.{{minor}}-collector-omnibus + type=semver,pattern={{major}}-collector-omnibus,enable=${{ !startsWith(github.ref, 'refs/tags/v0.') }} + + - name: Build and push Docker image + uses: docker/build-push-action@v7 + with: + platforms: linux/amd64,linux/arm64 + context: . + file: docker/Dockerfile.collector-omnibus + push: true + tags: ${{ steps.meta.outputs.tags }} + labels: ${{ steps.meta.outputs.labels }} + cache-from: type=gha,scope=docker-collector-omnibus + cache-to: type=gha,mode=max,scope=docker-collector-omnibus + collector-zfs: runs-on: ubuntu-latest timeout-minutes: 30 diff --git a/.github/workflows/sonarqube.yaml b/.github/workflows/sonarqube.yaml index d7ae759ce..ad750a761 100644 --- a/.github/workflows/sonarqube.yaml +++ b/.github/workflows/sonarqube.yaml @@ -16,7 +16,7 @@ jobs: runs-on: ubuntu-latest services: influxdb: - image: influxdb:2.2 + image: influxdb:2.9 env: DOCKER_INFLUXDB_INIT_MODE: setup DOCKER_INFLUXDB_INIT_USERNAME: admin @@ -24,6 +24,7 @@ jobs: DOCKER_INFLUXDB_INIT_ORG: scrutiny DOCKER_INFLUXDB_INIT_BUCKET: metrics DOCKER_INFLUXDB_INIT_ADMIN_TOKEN: my-super-secret-auth-token + INFLUXD_USE_HASHED_TOKENS: "false" ports: - 8086:8086 env: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index fc45cbcc9..c1da966c6 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -87,7 +87,7 @@ unless the task specifically requires host-level device access from Zeus. ``` 4. if you need a Dockerized InfluxDB dependency, start it on Zeus rather than on this Mac. ```bash - docker run -p 8086:8086 --rm influxdb:2.2 + docker run -p 8086:8086 -e INFLUXD_USE_HASHED_TOKENS=false --rm influxdb:2.9 ``` 5. start the scrutiny web server ```bash @@ -151,7 +151,7 @@ you'll need to follow the steps below: ``` 4. start a InfluxDB docker container. ```bash - docker run -p 8086:8086 --rm influxdb:2.2 + docker run -p 8086:8086 -e INFLUXD_USE_HASHED_TOKENS=false --rm influxdb:2.9 ``` 5. build the Angular Frontend Application ```bash @@ -175,7 +175,7 @@ If you'd like to populate the database with some test data, you can run the fol > This is done automatically by the `webapp/backend/pkg/models/testdata/helper.go` script ``` -docker run -p 8086:8086 --rm influxdb:2.2 +docker run -p 8086:8086 -e INFLUXD_USE_HASHED_TOKENS=false --rm influxdb:2.9 # curl -X POST -H "Content-Type: application/json" -d @webapp/backend/pkg/web/testdata/register-devices-req.json localhost:8080/api/devices/register @@ -249,7 +249,8 @@ docker run -p 8086:8086 -d --rm \ -e DOCKER_INFLUXDB_INIT_ORG=scrutiny \ -e DOCKER_INFLUXDB_INIT_BUCKET=metrics \ -e DOCKER_INFLUXDB_INIT_ADMIN_TOKEN=my-super-secret-auth-token \ -influxdb:2.2 +-e INFLUXD_USE_HASHED_TOKENS=false \ +influxdb:2.9 go test ./... # Run all tests go test -v ./webapp/backend/pkg/web/... # Run tests in specific package diff --git a/Makefile b/Makefile index a5350405e..7682fea8a 100644 --- a/Makefile +++ b/Makefile @@ -212,6 +212,11 @@ docker-collector: @echo "building collector docker image" docker build $(DOCKER_TARGETARCH_BUILD_ARG) -f docker/Dockerfile.collector -t ghcr.io/starosdev/scrutiny-dev:collector . +.PHONY: docker-collector-omnibus +docker-collector-omnibus: + @echo "building collector omnibus docker image" + docker build $(DOCKER_TARGETARCH_BUILD_ARG) -f docker/Dockerfile.collector-omnibus -t ghcr.io/starosdev/scrutiny-dev:collector-omnibus . + .PHONY: docker-collector-zfs docker-collector-zfs: @echo "building ZFS collector docker image" diff --git a/README.md b/README.md index b6c1837a6..1337aff78 100644 --- a/README.md +++ b/README.md @@ -77,12 +77,13 @@ This repository also owns the testing and production deployment definitions for - Beta images publish from the `beta` branch through [`Publish Beta Image`](./.github/workflows/deploy-beta.yml) - Production deploys from the `master` branch through [`Automated Release and Deploy`](./.github/workflows/release-and-deploy.yml) - `beta` is an optional pre-release channel for features that need validation before going to `master` -- Zeus production and testing are separate host appdata trees: +- Zeus currently runs separate develop, beta, and production host appdata trees: + - develop: `/mnt/user/appdata/scrutiny-develop` + - beta: `/mnt/user/appdata/scrutiny-beta` - production: `/mnt/user/appdata/scrutiny` - - testing: `/mnt/user/appdata/scrutiny-dev` - Zeus host-side deploy helpers target the live appdata-root compose files, not the repo `deploy/*` example compose files: + - develop helper: `/mnt/user/appdata/scrutiny-develop/docker-compose.yml` - production helper: `/mnt/user/appdata/scrutiny/docker-compose.yml` - - testing helper: `/mnt/user/appdata/scrutiny-dev/docker-compose.yml` - Host smoke checks require `/api/health` to return `200`, while the root path may legitimately redirect depending on auth or proxy behavior. - Deployment compose files, env templates, and host expectations live in [docs/DEPLOYMENTS.md](./docs/DEPLOYMENTS.md) @@ -247,6 +248,9 @@ the following Docker images: - `ghcr.io/starosdev/scrutiny:latest-collector` - Contains the Scrutiny data collector, `smartctl` binary and cron-like scheduler. You can run one collector on each server. +- `ghcr.io/starosdev/scrutiny:latest-collector-omnibus` - Recommended single-spoke image for hub/spoke deployments. + Bundles the SMART, ZFS, MDADM, Btrfs, filesystem, and performance collectors in one container while keeping each + optional collector disabled until you enable its existing schedule or run-on-startup env vars. - `ghcr.io/starosdev/scrutiny:latest-collector-zfs` - ZFS pool collector for monitoring ZFS health. Run alongside or instead of the standard collector if you use ZFS. See [docs/ZFS_POOL_MONITORING.md](./docs/ZFS_POOL_MONITORING.md) for setup instructions. - `ghcr.io/starosdev/scrutiny:latest-collector-mdadm` - MDADM collector for Linux software RAID monitoring. @@ -256,7 +260,7 @@ the following Docker images: - `ghcr.io/starosdev/scrutiny:latest-collector-performance` - Performance benchmark collector using fio. Runs periodic benchmarks and tracks throughput, IOPS, and latency over time. See [Performance Benchmarking](#performance-benchmarking) for details. - `ghcr.io/starosdev/scrutiny:latest-web` - Contains the Web UI and API. Only one container necessary -- `influxdb:2.2` - InfluxDB image, used by the Web container to persist SMART data. Only one container necessary. +- `influxdb:2.9` - InfluxDB image, used by the Web container to persist SMART data. Only one container necessary. See [docs/TROUBLESHOOTING_INFLUXDB.md](./docs/TROUBLESHOOTING_INFLUXDB.md) Branch channel tags follow the same pattern across images: @@ -268,15 +272,16 @@ Branch channel tags follow the same pattern across images: Default CI image publishing currently builds: - `collector` for `linux/amd64`, `linux/arm64`, and `linux/arm/v7` -- `web`, `collector-zfs`, `collector-mdadm`, `collector-btrfs`, and `collector-performance` for `linux/amd64` and `linux/arm64` +- `collector-omnibus`, `web`, `collector-zfs`, `collector-mdadm`, `collector-btrfs`, and `collector-performance` for `linux/amd64` and `linux/arm64` > See [docker/example.hubspoke.docker-compose.yml](docker/example.hubspoke.docker-compose.yml) for a docker-compose file. ```bash docker run -p 8086:8086 --restart unless-stopped \ -v `pwd`/influxdb2:/var/lib/influxdb2 \ + -e INFLUXD_USE_HASHED_TOKENS=false \ --name scrutiny-influxdb \ - influxdb:2.2 + influxdb:2.9 docker run -p 8080:8080 --restart unless-stopped \ -v `pwd`/scrutiny:/opt/scrutiny/config \ @@ -286,13 +291,17 @@ docker run -p 8080:8080 --restart unless-stopped \ docker run --restart unless-stopped \ -v /run/udev:/run/udev:ro \ --cap-add SYS_RAWIO \ + --cap-add SYS_ADMIN \ --device=/dev/sda \ --device=/dev/sdb \ -e COLLECTOR_API_ENDPOINT=http://SCRUTINY_WEB_IPADDRESS:8080 \ --name scrutiny-collector \ - ghcr.io/starosdev/scrutiny:latest-collector + ghcr.io/starosdev/scrutiny:latest-collector-omnibus ``` +Use `ghcr.io/starosdev/scrutiny:latest-collector` instead if you only want the SMART collector in that spoke and do not +need the extra host tooling for ZFS, MDADM, Btrfs, filesystem, or fio workloads. + ## Manual Installation (without-Docker) While the easiest way to get started with Scrutiny is using Docker (see above), @@ -335,7 +344,8 @@ None of these files are required, however if provided, they allow you to configu ## Cron Schedule Unfortunately the Cron schedule cannot be configured via the `collector.yaml` (as the collector binary needs to be triggered by a scheduler/cron). -However, if you are using the official `ghcr.io/starosdev/scrutiny:latest-collector` or `ghcr.io/starosdev/scrutiny:latest-omnibus` docker images, +However, if you are using the official `ghcr.io/starosdev/scrutiny:latest-collector`, `ghcr.io/starosdev/scrutiny:latest-collector-omnibus`, +or `ghcr.io/starosdev/scrutiny:latest-omnibus` docker images, you can use the `COLLECTOR_CRON_SCHEDULE` environmental variable to override the default cron schedule (daily @ midnight - `0 0 * * *`). `docker run -e COLLECTOR_CRON_SCHEDULE="0 0 * * *" ...` @@ -695,6 +705,12 @@ The test route sends through the same configured targets as normal notifications Scrutiny provides various methods to change the log level and generate log files. The web server and collector have **independent** log configurations and can be set separately. +## Shared Startup Banner Suppression + +Set `SCRUTINY_NO_LOGO=true` to suppress the ASCII startup banner in the web binary and all container-facing collector binaries. + +By default, the banner is still printed during startup. This env var is evaluated at process startup, so it works alongside `SCRUTINY_LOG_LEVEL` and `COLLECTOR_LOG_LEVEL` when you want quiet `WARN` or `ERROR` startup behavior without changing the default banner behavior for other runs. + ## Valid Log Levels The following log levels are supported (case-insensitive), listed from highest to lowest severity: @@ -718,6 +734,7 @@ You can use environmental variables to enable debug logging and/or log files for ```bash DEBUG=true +SCRUTINY_NO_LOGO=true SCRUTINY_LOG_FILE=/tmp/web.log ``` @@ -799,6 +816,7 @@ Example: ```bash docker run -e SCRUTINY_WEB_LISTEN_PORT=9090 \ -e SCRUTINY_WEB_INFLUXDB_HOST=influxdb.local \ + -e SCRUTINY_NO_LOGO=true \ -e SCRUTINY_LOG_LEVEL=DEBUG \ ghcr.io/starosdev/scrutiny:latest-web ``` @@ -809,6 +827,7 @@ You can use environmental variables to enable debug logging and/or log files for ```bash DEBUG=true +SCRUTINY_NO_LOGO=true COLLECTOR_LOG_FILE=/tmp/collector.log ``` @@ -846,6 +865,7 @@ Example: ```bash docker run -e COLLECTOR_COMMANDS_METRICS_SMART_ARGS="--xall --json -T permissive" \ -e COLLECTOR_API_ENDPOINT=http://scrutiny-web:8080 \ + -e SCRUTINY_NO_LOGO=true \ ghcr.io/starosdev/scrutiny:latest-collector ``` @@ -865,6 +885,7 @@ The performance collector is a separate binary (`scrutiny-collector-performance` ```bash DEBUG=true +SCRUTINY_NO_LOGO=true COLLECTOR_PERF_LOG_FILE=/tmp/performance.log ``` @@ -937,6 +958,8 @@ The MDADM collector prefers its own config file, `collector-mdadm.yaml`, and fal For mounts, capabilities, compose examples, and troubleshooting, see [docs/MDADM_MONITORING.md](docs/MDADM_MONITORING.md). +If you upgrade from a build that registered MDADM arrays before `host_id` was persisted on re-registration, run one fresh MDADM collection on each affected host so grouped host headings can backfill correctly in the UI. + # Supported Architectures | Architecture Name | Binaries | Docker | diff --git a/TESTING.md b/TESTING.md index b8fe39d9d..7efe31f84 100644 --- a/TESTING.md +++ b/TESTING.md @@ -55,7 +55,7 @@ main repository. ```bash # Run all backend tests -docker run -p 8086:8086 -d --rm influxdb:2.2 # InfluxDB required +docker run -p 8086:8086 -d -e INFLUXD_USE_HASHED_TOKENS=false --rm influxdb:2.9 # InfluxDB required go test ./... # Run all frontend tests @@ -96,7 +96,8 @@ docker run -p 8086:8086 -d --rm \ -e DOCKER_INFLUXDB_INIT_ORG=scrutiny \ -e DOCKER_INFLUXDB_INIT_BUCKET=metrics \ -e DOCKER_INFLUXDB_INIT_ADMIN_TOKEN=my-super-secret-auth-token \ - influxdb:2.2 + -e INFLUXD_USE_HASHED_TOKENS=false \ + influxdb:2.9 ``` Then run tests: @@ -189,7 +190,7 @@ npm run lint:fix # Auto-fix what it can ## 3. Building Docker Images Locally -The project produces five Docker images. Build whichever you need to test: +Build whichever image matches the path you changed: ```bash # Omnibus (web + all collectors + InfluxDB, all-in-one) @@ -204,10 +205,22 @@ make docker-web make docker-collector # -> ghcr.io/starosdev/scrutiny-dev:collector +# Single-container hub/spoke collector bundle +make docker-collector-omnibus +# -> ghcr.io/starosdev/scrutiny-dev:collector-omnibus + # ZFS collector only make docker-collector-zfs # -> ghcr.io/starosdev/scrutiny-dev:collector-zfs +# MDADM collector only +make docker-collector-mdadm +# -> ghcr.io/starosdev/scrutiny-dev:collector-mdadm + +# Btrfs collector only +make docker-collector-btrfs +# -> ghcr.io/starosdev/scrutiny-dev:collector-btrfs + # Performance collector only make docker-collector-performance # -> ghcr.io/starosdev/scrutiny-dev:collector-performance @@ -272,7 +285,8 @@ For testing changes to individual components in isolation: # 1. Start InfluxDB docker run -d --name influxdb \ -p 8086:8086 \ - influxdb:2.2 + -e INFLUXD_USE_HASHED_TOKENS=false \ + influxdb:2.9 # 2. Build and start the web server docker build -f docker/Dockerfile.web -t scrutiny-web:local . @@ -307,9 +321,11 @@ For a more production-like test, use the provided compose files: cat > docker-compose.test.yaml << 'EOF' services: influxdb: - image: influxdb:2.2 + image: influxdb:2.9 ports: - '8086:8086' + environment: + INFLUXD_USE_HASHED_TOKENS: "false" healthcheck: test: ["CMD", "curl", "-f", "http://localhost:8086/health"] interval: 5s @@ -759,7 +775,8 @@ docker run -p 8086:8086 -d --rm \ -e DOCKER_INFLUXDB_INIT_ORG=scrutiny \ -e DOCKER_INFLUXDB_INIT_BUCKET=metrics \ -e DOCKER_INFLUXDB_INIT_ADMIN_TOKEN=my-super-secret-auth-token \ - influxdb:2.2 + -e INFLUXD_USE_HASHED_TOKENS=false \ + influxdb:2.9 go mod vendor go test ./... diff --git a/collector/cmd/collector-btrfs/collector-btrfs.go b/collector/cmd/collector-btrfs/collector-btrfs.go index 80abe7a8b..1d061e7ce 100644 --- a/collector/cmd/collector-btrfs/collector-btrfs.go +++ b/collector/cmd/collector-btrfs/collector-btrfs.go @@ -9,12 +9,11 @@ import ( "strings" "time" - _ "go.uber.org/automaxprocs" - utils "github.com/analogj/go-util/utils" "github.com/analogj/scrutiny/collector/pkg/btrfs" "github.com/analogj/scrutiny/collector/pkg/config" "github.com/analogj/scrutiny/collector/pkg/errors" + "github.com/analogj/scrutiny/pkg/startup" "github.com/analogj/scrutiny/webapp/backend/pkg/version" "github.com/fatih/color" "github.com/sirupsen/logrus" @@ -39,8 +38,8 @@ func main() { configFilePath := resolveConfigPath() - bootstrapLogger := logrus.WithFields(logrus.Fields{"type": "btrfs"}) - bootstrapLogger.Logger.SetLevel(logrus.InfoLevel) + bootstrapLogger := startup.NewBootstrapLogger("btrfs", appConfig) + startup.ConfigureMaxProcs(bootstrapLogger) err = appConfig.ReadConfig(configFilePath, bootstrapLogger) if _, missing := err.(errors.ConfigFileMissingError); err != nil && !missing { @@ -53,6 +52,9 @@ func main() { Version: version.VERSION, Compiled: time.Now(), Before: func(c *cli.Context) error { + if !startup.ShouldPrintBanner() { + return nil + } collectorName := "Starosdev/scrutiny/btrfs" var versionInfo string diff --git a/collector/cmd/collector-filesystem/collector-filesystem.go b/collector/cmd/collector-filesystem/collector-filesystem.go index 4c6b43b01..4069379ea 100644 --- a/collector/cmd/collector-filesystem/collector-filesystem.go +++ b/collector/cmd/collector-filesystem/collector-filesystem.go @@ -9,12 +9,11 @@ import ( "strings" "time" - _ "go.uber.org/automaxprocs" - utils "github.com/analogj/go-util/utils" "github.com/analogj/scrutiny/collector/pkg/config" "github.com/analogj/scrutiny/collector/pkg/errors" "github.com/analogj/scrutiny/collector/pkg/filesystem" + "github.com/analogj/scrutiny/pkg/startup" "github.com/analogj/scrutiny/webapp/backend/pkg/version" "github.com/fatih/color" "github.com/sirupsen/logrus" @@ -39,8 +38,8 @@ func main() { configFilePath := resolveConfigPath() - bootstrapLogger := logrus.WithFields(logrus.Fields{"type": "filesystem"}) - bootstrapLogger.Logger.SetLevel(logrus.InfoLevel) + bootstrapLogger := startup.NewBootstrapLogger("filesystem", appConfig) + startup.ConfigureMaxProcs(bootstrapLogger) err = appConfig.ReadConfig(configFilePath, bootstrapLogger) if _, missing := err.(errors.ConfigFileMissingError); err != nil && !missing { @@ -53,6 +52,9 @@ func main() { Version: version.VERSION, Compiled: time.Now(), Before: func(c *cli.Context) error { + if !startup.ShouldPrintBanner() { + return nil + } collectorName := "Starosdev/scrutiny/filesystem" var versionInfo string diff --git a/collector/cmd/collector-mdadm/collector-mdadm.go b/collector/cmd/collector-mdadm/collector-mdadm.go index d0e415515..ae041cd82 100644 --- a/collector/cmd/collector-mdadm/collector-mdadm.go +++ b/collector/cmd/collector-mdadm/collector-mdadm.go @@ -9,12 +9,11 @@ import ( "strings" "time" - _ "go.uber.org/automaxprocs" - utils "github.com/analogj/go-util/utils" "github.com/analogj/scrutiny/collector/pkg/config" "github.com/analogj/scrutiny/collector/pkg/errors" "github.com/analogj/scrutiny/collector/pkg/mdadm" + "github.com/analogj/scrutiny/pkg/startup" "github.com/analogj/scrutiny/webapp/backend/pkg/version" "github.com/fatih/color" "github.com/sirupsen/logrus" @@ -39,8 +38,8 @@ func main() { } // Create a bootstrap logger for config loading - bootstrapLogger := logrus.WithFields(logrus.Fields{"type": "mdadm"}) - bootstrapLogger.Logger.SetLevel(logrus.InfoLevel) + bootstrapLogger := startup.NewBootstrapLogger("mdadm", cfg) + startup.ConfigureMaxProcs(bootstrapLogger) if err := readOptionalCollectorConfig(cfg, resolveCollectorConfigPath("mdadm"), bootstrapLogger); err != nil { os.Exit(1) @@ -58,7 +57,9 @@ func main() { }, }, Before: func(c *cli.Context) error { - color.New(color.FgGreen).Fprintf(c.App.Writer, "%s", collectorBanner("Starosdev/scrutiny/mdadm")) + if startup.ShouldPrintBanner() { + color.New(color.FgGreen).Fprintf(c.App.Writer, "%s", collectorBanner("Starosdev/scrutiny/mdadm")) + } return nil }, diff --git a/collector/cmd/collector-metrics/collector-metrics.go b/collector/cmd/collector-metrics/collector-metrics.go index 6dea30851..39dc4d996 100644 --- a/collector/cmd/collector-metrics/collector-metrics.go +++ b/collector/cmd/collector-metrics/collector-metrics.go @@ -11,12 +11,11 @@ import ( "syscall" "time" - _ "go.uber.org/automaxprocs" - utils "github.com/analogj/go-util/utils" "github.com/analogj/scrutiny/collector/pkg/collector" "github.com/analogj/scrutiny/collector/pkg/config" "github.com/analogj/scrutiny/collector/pkg/errors" + "github.com/analogj/scrutiny/pkg/startup" "github.com/analogj/scrutiny/webapp/backend/pkg/version" "github.com/fatih/color" "github.com/robfig/cron/v3" @@ -41,9 +40,8 @@ func main() { configFilePath = configFilePathAlternative } - // Create a bootstrap logger for config loading (before config-based logger is available) - bootstrapLogger := logrus.WithFields(logrus.Fields{"type": "metrics"}) - bootstrapLogger.Logger.SetLevel(logrus.InfoLevel) + bootstrapLogger := startup.NewBootstrapLogger("metrics", config) + startup.ConfigureMaxProcs(bootstrapLogger) //we're going to load the config file manually, since we need to validate it. err = config.ReadConfig(configFilePath, bootstrapLogger) // Find and read the config file @@ -78,6 +76,9 @@ OPTIONS: }, }, Before: func(c *cli.Context) error { + if !startup.ShouldPrintBanner() { + return nil + } collectorMetrics := "AnalogJ/scrutiny/metrics" diff --git a/collector/cmd/collector-performance/collector-performance.go b/collector/cmd/collector-performance/collector-performance.go index b834a4a30..da9c97e68 100644 --- a/collector/cmd/collector-performance/collector-performance.go +++ b/collector/cmd/collector-performance/collector-performance.go @@ -9,12 +9,11 @@ import ( "strings" "time" - _ "go.uber.org/automaxprocs" - utils "github.com/analogj/go-util/utils" "github.com/analogj/scrutiny/collector/pkg/config" "github.com/analogj/scrutiny/collector/pkg/errors" "github.com/analogj/scrutiny/collector/pkg/performance" + "github.com/analogj/scrutiny/pkg/startup" "github.com/analogj/scrutiny/webapp/backend/pkg/version" "github.com/fatih/color" "github.com/sirupsen/logrus" @@ -42,8 +41,8 @@ func main() { configFilePath := resolveConfigPath() // Create a bootstrap logger for config loading - bootstrapLogger := logrus.WithFields(logrus.Fields{"type": "performance"}) - bootstrapLogger.Logger.SetLevel(logrus.InfoLevel) + bootstrapLogger := startup.NewBootstrapLogger("performance", config) + startup.ConfigureMaxProcs(bootstrapLogger) // Load the config file (ignore "could not find config file") err = config.ReadConfig(configFilePath, bootstrapLogger) @@ -76,6 +75,9 @@ OPTIONS: }, }, Before: func(c *cli.Context) error { + if !startup.ShouldPrintBanner() { + return nil + } collectorPerf := "Starosdev/scrutiny/performance" var versionInfo string diff --git a/collector/cmd/collector-zfs/collector-zfs.go b/collector/cmd/collector-zfs/collector-zfs.go index f14488c37..df68fd436 100644 --- a/collector/cmd/collector-zfs/collector-zfs.go +++ b/collector/cmd/collector-zfs/collector-zfs.go @@ -9,12 +9,11 @@ import ( "strings" "time" - _ "go.uber.org/automaxprocs" - utils "github.com/analogj/go-util/utils" "github.com/analogj/scrutiny/collector/pkg/config" "github.com/analogj/scrutiny/collector/pkg/errors" "github.com/analogj/scrutiny/collector/pkg/zfs" + "github.com/analogj/scrutiny/pkg/startup" "github.com/analogj/scrutiny/webapp/backend/pkg/version" "github.com/fatih/color" "github.com/sirupsen/logrus" @@ -39,8 +38,8 @@ func main() { } // Create a bootstrap logger for config loading - bootstrapLogger := logrus.WithFields(logrus.Fields{"type": "zfs"}) - bootstrapLogger.Logger.SetLevel(logrus.InfoLevel) + bootstrapLogger := startup.NewBootstrapLogger("zfs", cfg) + startup.ConfigureMaxProcs(bootstrapLogger) if err := readOptionalCollectorConfig(cfg, resolveCollectorConfigPath("zfs"), bootstrapLogger); err != nil { os.Exit(1) @@ -71,7 +70,9 @@ OPTIONS: }, }, Before: func(c *cli.Context) error { - color.New(color.FgGreen).Fprintf(c.App.Writer, "%s", collectorBanner("Starosdev/scrutiny/zfs")) + if startup.ShouldPrintBanner() { + color.New(color.FgGreen).Fprintf(c.App.Writer, "%s", collectorBanner("Starosdev/scrutiny/zfs")) + } return nil }, diff --git a/collector/pkg/mdadm/detect/detect.go b/collector/pkg/mdadm/detect/detect.go index dbc6f74bd..b73c0412d 100644 --- a/collector/pkg/mdadm/detect/detect.go +++ b/collector/pkg/mdadm/detect/detect.go @@ -121,7 +121,7 @@ func (d *Detect) getArrayDetail(name string) (models.MDADMArray, models.MDADMMet } d.resolveArrayUUID(&array, devicePath, name) - d.enrichArrayMetrics(&metrics, name, devicePath) + d.enrichArrayMetrics(&array, &metrics, name, devicePath) return array, metrics, nil } @@ -146,11 +146,14 @@ func (d *Detect) resolveArrayUUID(array *models.MDADMArray, devicePath, name str d.Logger.Warnf("Using synthetic MDADM identifier for %s because mdadm did not expose a UUID", devicePath) } -// enrichArrayMetrics augments parsed metrics with /proc/mdstat data (raw block, sync progress) -// and filesystem usage when the array is mounted in the container. -func (d *Detect) enrichArrayMetrics(metrics *models.MDADMMetrics, name, devicePath string) { +// enrichArrayMetrics augments parsed array data with /proc/mdstat member devices and metrics, +// plus filesystem usage when the array is mounted in the container. +func (d *Detect) enrichArrayMetrics(array *models.MDADMArray, metrics *models.MDADMMetrics, name, devicePath string) { rawMdstat, _ := d.getRawMdstat(name) metrics.RawMdstat = rawMdstat + if len(array.Devices) == 0 { + array.Devices = parseMdstatDevices(rawMdstat) + } // Parse sync/check/rebuild/recovery progress from /proc/mdstat if not already set // by mdadm --detail. The "check = X%" line only appears in /proc/mdstat. @@ -171,6 +174,29 @@ func (d *Detect) enrichArrayMetrics(metrics *models.MDADMMetrics, name, devicePa // mdstatProgressPattern matches the "check|resync|recovery|rebuild = X%" progress line in /proc/mdstat. var mdstatProgressPattern = regexp.MustCompile(`(?:check|resync|recovery|rebuild)\s*=\s*(\d+(?:\.\d+)?)%`) +var mdstatDevicePattern = regexp.MustCompile(`(\S+)\[\d+\]`) + +// parseMdstatDevices extracts member device names from an array's first /proc/mdstat line. +// mdstat omits the /dev prefix, so normalize the result to match mdadm --detail output. +func parseMdstatDevices(rawMdstat string) []string { + devices := []string{} + firstLine, _, _ := strings.Cut(rawMdstat, "\n") + _, members, found := strings.Cut(firstLine, ":") + if !found { + return devices + } + + for _, match := range mdstatDevicePattern.FindAllStringSubmatch(members, -1) { + device := match[1] + if !strings.HasPrefix(device, "/") { + device = "/dev/" + device + } + devices = append(devices, device) + } + + return devices +} + func (d *Detect) getArrayUUIDFromExport(devicePath string) (string, error) { var cmd *exec.Cmd if os.Getuid() == 0 { @@ -234,7 +260,7 @@ func (d *Detect) parseMdadmOutput(name string, output string) (models.MDADMArray // Device list starts after the header inDeviceList := false - devicePattern := regexp.MustCompile(`\s+\d+\s+\d+\s+\d+\s+\d+\s+.+\s+(/dev/\S+)`) + devicePattern := regexp.MustCompile(`(/dev/\S+)\s*$`) for scanner.Scan() { line := scanner.Text() diff --git a/collector/pkg/mdadm/detect/detect_test.go b/collector/pkg/mdadm/detect/detect_test.go index a232316c6..d98bdd6ae 100644 --- a/collector/pkg/mdadm/detect/detect_test.go +++ b/collector/pkg/mdadm/detect/detect_test.go @@ -18,7 +18,7 @@ md0 : active raid1 sdb[1] sda[0] 1048512 blocks super 1.2 [2/2] [UU] unused devices: ` - + tmpfile, err := ioutil.TempFile("", "mdstat") require.NoError(t, err) defer os.Remove(tmpfile.Name()) @@ -31,7 +31,7 @@ unused devices: ` // We need to override the file path in the implementation or use a trick // Since I can't easily override os.Open("/proc/mdstat"), I'll modify the implementation // to take a path or I'll just test the parsing logic directly if I split it. - + // For now, I'll test the parseMdadmOutput logic which is more complex. } @@ -133,6 +133,39 @@ Consistency Policy : resync // Note: UsedBytes is populated by getMountUsage (statfs) in getArrayDetail, not by parseMdadmOutput. } +func TestDetect_ParseMdadmOutput_DeviceWithNoRaidSlot(t *testing.T) { + d := &Detect{ + Logger: logrus.NewEntry(logrus.New()), + } + + output := `/dev/md0: + Raid Level : raid1 + UUID : 12345678:12345678:12345678:12345678 + + Number Major Minor RaidDevice State + 0 8 2 0 active sync /dev/sda2 + 2 8 34 - spare /dev/sdc2` + + array, _, err := d.parseMdadmOutput("md0", output) + + require.NoError(t, err) + assert.Equal(t, []string{"/dev/sda2", "/dev/sdc2"}, array.Devices) +} + +func TestParseMdstatDevices(t *testing.T) { + rawMdstat := `md1 : active raid6 sde2[0] sdc2[1](F) sdg2[2] sdf2[3] + 15627534336 blocks super 1.2 level 6, 512k chunk, algorithm 2 [4/3] [U_UU]` + + assert.Equal(t, []string{"/dev/sde2", "/dev/sdc2", "/dev/sdg2", "/dev/sdf2"}, parseMdstatDevices(rawMdstat)) +} + +func TestParseMdstatDevicesReturnsEmptySliceForMissingHeader(t *testing.T) { + devices := parseMdstatDevices("") + + assert.NotNil(t, devices) + assert.Empty(t, devices) +} + func TestParseMdadmExportUUID(t *testing.T) { output := `MD_LEVEL=raid1 MD_DEVICES=2 diff --git a/docker/Dockerfile b/docker/Dockerfile index 0f4666318..1c191a902 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -31,7 +31,8 @@ WORKDIR /opt/scrutiny ENV PATH="/opt/scrutiny/bin:${PATH}" ENV INFLUXD_CONFIG_PATH=/opt/scrutiny/influxdb ENV S6VER="3.1.6.2" -ENV INFLUXVER="2.2.0" +ENV INFLUXVER="2.9.1-1" +ENV INFLUXD_USE_HASHED_TOKENS=false ENV S6_SERVICES_READYTIME=1000 SHELL ["/usr/bin/sh", "-c"] @@ -83,7 +84,7 @@ RUN sed -i 's/Components: main/Components: main contrib/' /etc/apt/sources.list. && download_and_extract_xz \ "https://github.com/just-containers/s6-overlay/releases/download/v${S6VER}/s6-overlay-${S6_ARCH}.tar.xz" \ /tmp/s6-overlay-${S6_ARCH}.tar.xz -RUN curl -L https://dl.influxdata.com/influxdb/releases/influxdb2-${INFLUXVER}-${TARGETARCH}.deb --output /tmp/influxdb2-${INFLUXVER}-${TARGETARCH}.deb \ +RUN curl -L https://dl.influxdata.com/influxdb/releases/influxdb2_${INFLUXVER}_${TARGETARCH}.deb --output /tmp/influxdb2-${INFLUXVER}-${TARGETARCH}.deb \ && dpkg -i --force-all /tmp/influxdb2-${INFLUXVER}-${TARGETARCH}.deb \ && rm -rf /tmp/influxdb2-${INFLUXVER}-${TARGETARCH}.deb diff --git a/docker/Dockerfile.collector-omnibus b/docker/Dockerfile.collector-omnibus new file mode 100644 index 000000000..ccdab6df8 --- /dev/null +++ b/docker/Dockerfile.collector-omnibus @@ -0,0 +1,105 @@ +# syntax=docker/dockerfile:1.4 +######################################################################################################################## +# Collector Omnibus Image +######################################################################################################################## + +FROM golang:1.26-trixie AS backendbuild + +WORKDIR /go/src/github.com/analogj/scrutiny +COPY --link . /go/src/github.com/analogj/scrutiny +RUN apt-get update && DEBIAN_FRONTEND=noninteractive \ + apt-get install -y --no-install-recommends \ + file \ + && rm -rf /var/lib/apt/lists/* +RUN make binary-clean \ + binary-collector \ + binary-collector-zfs \ + binary-collector-performance \ + binary-collector-mdadm \ + binary-collector-filesystem \ + binary-collector-btrfs + +FROM debian:trixie-slim AS runtime +ARG TARGETARCH +WORKDIR /opt/scrutiny +ENV PATH="/opt/scrutiny/bin:${PATH}" +ENV S6VER="3.1.6.2" +SHELL ["/usr/bin/sh", "-c"] + +RUN sed -i 's/Components: main/Components: main contrib/' /etc/apt/sources.list.d/debian.sources \ + && echo "deb http://deb.debian.org/debian trixie-backports main" > /etc/apt/sources.list.d/backports.list \ + && apt-get update && DEBIAN_FRONTEND=noninteractive \ + apt-get install -y --no-install-recommends \ + ca-certificates \ + cron \ + curl \ + tzdata \ + procps \ + xz-utils \ + btrfs-progs \ + zfsutils-linux \ + mdadm \ + fio \ + && apt-get install -y --no-install-recommends -t trixie-backports smartmontools \ + && rm -rf /var/lib/apt/lists/* \ + && update-ca-certificates \ + && download_and_extract_xz() { \ + url="$1"; \ + archive="$2"; \ + checksum_file="${archive}.sha256"; \ + curl --fail --location --show-error --silent \ + --retry 5 --retry-all-errors --retry-delay 2 --max-time 300 \ + "$url" --output "$archive" \ + && curl --fail --location --show-error --silent \ + --retry 5 --retry-all-errors --retry-delay 2 --max-time 60 \ + "${url}.sha256" --output "$checksum_file" \ + && awk -v archive="$archive" '{print $1 " " archive}' "$checksum_file" | sha256sum -c - \ + && xz -t "$archive" \ + && tar -Jxpf "$archive" -C / \ + && rm -f "$archive" "$checksum_file"; \ + } \ + && case ${TARGETARCH} in \ + "amd64") S6_ARCH=x86_64 ;; \ + "arm64") S6_ARCH=aarch64 ;; \ + esac \ + && mkdir -p /opt/scrutiny/bin \ + && download_and_extract_xz \ + "https://github.com/just-containers/s6-overlay/releases/download/v${S6VER}/s6-overlay-noarch.tar.xz" \ + /tmp/s6-overlay-noarch.tar.xz \ + && download_and_extract_xz \ + "https://github.com/just-containers/s6-overlay/releases/download/v${S6VER}/s6-overlay-${S6_ARCH}.tar.xz" \ + /tmp/s6-overlay-${S6_ARCH}.tar.xz + +COPY /docker/apparmor-profile /opt/scrutiny/apparmor-profile +COPY /docker/entrypoint-collector-omnibus-startup.sh /etc/cont-init.d/60-collector-startup +COPY /rootfs/etc/cont-init.d/01-timezone /etc/cont-init.d/01-timezone +COPY /rootfs/etc/cont-init.d/50-cron-config /etc/cont-init.d/50-cron-config +COPY /rootfs/etc/cron.d/scrutiny /etc/cron.d/scrutiny +COPY /rootfs/etc/cron.d/scrutiny-zfs /etc/cron.d/scrutiny-zfs +COPY /rootfs/etc/cron.d/scrutiny-performance /etc/cron.d/scrutiny-performance +COPY /rootfs/etc/cron.d/scrutiny-mdadm /etc/cron.d/scrutiny-mdadm +COPY /rootfs/etc/cron.d/scrutiny-filesystem /etc/cron.d/scrutiny-filesystem +COPY /rootfs/etc/cron.d/scrutiny-btrfs /etc/cron.d/scrutiny-btrfs +COPY /rootfs/etc/services.d/cron /etc/services.d/cron + +COPY --link --from=backendbuild --chmod=755 /go/src/github.com/analogj/scrutiny/scrutiny-collector-metrics /opt/scrutiny/bin/ +COPY --link --from=backendbuild --chmod=755 /go/src/github.com/analogj/scrutiny/scrutiny-collector-zfs /opt/scrutiny/bin/ +COPY --link --from=backendbuild --chmod=755 /go/src/github.com/analogj/scrutiny/scrutiny-collector-performance /opt/scrutiny/bin/ +COPY --link --from=backendbuild --chmod=755 /go/src/github.com/analogj/scrutiny/scrutiny-collector-mdadm /opt/scrutiny/bin/ +COPY --link --from=backendbuild --chmod=755 /go/src/github.com/analogj/scrutiny/scrutiny-collector-filesystem /opt/scrutiny/bin/ +COPY --link --from=backendbuild --chmod=755 /go/src/github.com/analogj/scrutiny/scrutiny-collector-btrfs /opt/scrutiny/bin/ + +RUN chmod 0644 /etc/cron.d/scrutiny \ + && chmod 0644 /etc/cron.d/scrutiny-zfs \ + && chmod 0644 /etc/cron.d/scrutiny-performance \ + && chmod 0644 /etc/cron.d/scrutiny-mdadm \ + && chmod 0644 /etc/cron.d/scrutiny-filesystem \ + && chmod 0644 /etc/cron.d/scrutiny-btrfs \ + && rm -f /etc/cron.daily/* \ + && mkdir -p /opt/scrutiny/config \ + && chmod -R ugo+rwx /opt/scrutiny/config \ + && chmod +x /etc/cont-init.d/* \ + && chmod +x /etc/services.d/*/run \ + && chmod +x /etc/services.d/*/finish + +ENTRYPOINT ["/init"] diff --git a/docker/README.md b/docker/README.md index 0e1127305..12a9fa322 100644 --- a/docker/README.md +++ b/docker/README.md @@ -1 +1 @@ -`rootfs` is only used by Dockerfile and Dockerfile.collector +`rootfs` is used by `docker/Dockerfile`, `docker/Dockerfile.collector`, and `docker/Dockerfile.collector-omnibus`. diff --git a/docker/entrypoint-collector-btrfs.sh b/docker/entrypoint-collector-btrfs.sh index ef3728bb1..553aebc82 100644 --- a/docker/entrypoint-collector-btrfs.sh +++ b/docker/entrypoint-collector-btrfs.sh @@ -5,6 +5,10 @@ # will then source into the crontab file (/etc/cron.d/scrutiny-btrfs) (set -o posix; export -p) > /env.sh +log_info() { + printf 'time="%s" level=info msg="%s" type=btrfs\n' "$(date -u +"%Y-%m-%dT%H:%M:%SZ")" "$1" +} + # adding ability to customize the cron schedule. COLLECTOR_BTRFS_CRON_SCHEDULE=${COLLECTOR_BTRFS_CRON_SCHEDULE:-"*/15 * * * *"} COLLECTOR_BTRFS_RUN_STARTUP=${COLLECTOR_BTRFS_RUN_STARTUP:-"false"} @@ -18,11 +22,11 @@ sed -i 's|{COLLECTOR_BTRFS_CRON_SCHEDULE}|'"${COLLECTOR_BTRFS_CRON_SCHEDULE}"'|g if [[ "${COLLECTOR_BTRFS_RUN_STARTUP}" == "true" ]]; then sleep ${COLLECTOR_BTRFS_RUN_STARTUP_SLEEP} - echo "starting scrutiny Btrfs collector (run-once mode. subsequent calls will be triggered via cron service)" + log_info "starting scrutiny Btrfs collector (run-once mode. subsequent calls will be triggered via cron service)" COLLECTOR_CRON_SCHEDULE= COLLECTOR_BTRFS_RUN_STARTUP= /opt/scrutiny/bin/scrutiny-collector-btrfs run fi # now that we have the env start cron in the foreground -echo "starting cron" +log_info "starting cron" exec su -c "cron -f -L 15" root diff --git a/docker/entrypoint-collector-mdadm.sh b/docker/entrypoint-collector-mdadm.sh index 7a0385d11..9daa87eb6 100644 --- a/docker/entrypoint-collector-mdadm.sh +++ b/docker/entrypoint-collector-mdadm.sh @@ -5,6 +5,10 @@ # will then source into the crontab file (/etc/cron.d/scrutiny-mdadm) (set -o posix; export -p) > /env.sh +log_info() { + printf 'time="%s" level=info msg="%s" type=mdadm\n' "$(date -u +"%Y-%m-%dT%H:%M:%SZ")" "$1" +} + # adding ability to customize the cron schedule. COLLECTOR_MDADM_CRON_SCHEDULE=${COLLECTOR_MDADM_CRON_SCHEDULE:-"*/15 * * * *"} COLLECTOR_MDADM_RUN_STARTUP=${COLLECTOR_MDADM_RUN_STARTUP:-"false"} @@ -18,11 +22,11 @@ sed -i 's|{COLLECTOR_MDADM_CRON_SCHEDULE}|'"${COLLECTOR_MDADM_CRON_SCHEDULE}"'|g if [[ "${COLLECTOR_MDADM_RUN_STARTUP}" == "true" ]]; then sleep ${COLLECTOR_MDADM_RUN_STARTUP_SLEEP} - echo "starting scrutiny MDADM collector (run-once mode. subsequent calls will be triggered via cron service)" + log_info "starting scrutiny MDADM collector (run-once mode. subsequent calls will be triggered via cron service)" COLLECTOR_CRON_SCHEDULE= COLLECTOR_MDADM_RUN_STARTUP= /opt/scrutiny/bin/scrutiny-collector-mdadm run fi # now that we have the env start cron in the foreground -echo "starting cron" +log_info "starting cron" exec su -c "cron -f -L 15" root diff --git a/docker/entrypoint-collector-omnibus-startup.sh b/docker/entrypoint-collector-omnibus-startup.sh new file mode 100644 index 000000000..95b4037b5 --- /dev/null +++ b/docker/entrypoint-collector-omnibus-startup.sh @@ -0,0 +1,38 @@ +#!/command/with-contenv bash + +log_info() { + local component="$1" + local message="$2" + printf 'time="%s" level=info msg="%s" type=%s\n' "$(date -u +"%Y-%m-%dT%H:%M:%SZ")" "$message" "$component" +} + +run_startup_collector() { + local startup_var="$1" + local sleep_var="$2" + local binary="$3" + local label="$4" + local component="$5" + local startup="${!startup_var:-false}" + local delay="${!sleep_var:-1}" + + if [ "${startup}" != "true" ]; then + return + fi + + log_info "${component}" "starting ${label} collector (run-once mode)" + sleep "${delay}" + "${binary}" run +} + +run_startup_collector "COLLECTOR_RUN_STARTUP" "COLLECTOR_RUN_STARTUP_SLEEP" \ + "/opt/scrutiny/bin/scrutiny-collector-metrics" "scrutiny metrics" "metrics" +run_startup_collector "COLLECTOR_ZFS_RUN_STARTUP" "COLLECTOR_ZFS_RUN_STARTUP_SLEEP" \ + "/opt/scrutiny/bin/scrutiny-collector-zfs" "scrutiny ZFS" "zfs" +run_startup_collector "COLLECTOR_MDADM_RUN_STARTUP" "COLLECTOR_MDADM_RUN_STARTUP_SLEEP" \ + "/opt/scrutiny/bin/scrutiny-collector-mdadm" "scrutiny MDADM" "mdadm" +run_startup_collector "COLLECTOR_BTRFS_RUN_STARTUP" "COLLECTOR_BTRFS_RUN_STARTUP_SLEEP" \ + "/opt/scrutiny/bin/scrutiny-collector-btrfs" "scrutiny Btrfs" "btrfs" +run_startup_collector "COLLECTOR_FILESYSTEM_RUN_STARTUP" "COLLECTOR_FILESYSTEM_RUN_STARTUP_SLEEP" \ + "/opt/scrutiny/bin/scrutiny-collector-filesystem" "scrutiny filesystem" "filesystem" +run_startup_collector "COLLECTOR_PERF_RUN_STARTUP" "COLLECTOR_PERF_RUN_STARTUP_SLEEP" \ + "/opt/scrutiny/bin/scrutiny-collector-performance" "scrutiny performance" "performance" diff --git a/docker/entrypoint-collector-performance.sh b/docker/entrypoint-collector-performance.sh index 589895419..d9f83e3d0 100644 --- a/docker/entrypoint-collector-performance.sh +++ b/docker/entrypoint-collector-performance.sh @@ -5,6 +5,10 @@ # will then source into the crontab file (/etc/cron.d/scrutiny-performance) (set -o posix; export -p) > /env.sh +log_info() { + printf 'time="%s" level=info msg="%s" type=performance\n' "$(date -u +"%Y-%m-%dT%H:%M:%SZ")" "$1" +} + # adding ability to customize the cron schedule. COLLECTOR_PERF_CRON_SCHEDULE=${COLLECTOR_PERF_CRON_SCHEDULE:-"0 2 * * 0"} COLLECTOR_PERF_RUN_STARTUP=${COLLECTOR_PERF_RUN_STARTUP:-"false"} @@ -18,11 +22,11 @@ sed -i 's|{COLLECTOR_PERF_CRON_SCHEDULE}|'"${COLLECTOR_PERF_CRON_SCHEDULE}"'|g' if [[ "${COLLECTOR_PERF_RUN_STARTUP}" == "true" ]]; then sleep ${COLLECTOR_PERF_RUN_STARTUP_SLEEP} - echo "starting scrutiny performance collector (run-once mode. subsequent calls will be triggered via cron service)" + log_info "starting scrutiny performance collector (run-once mode. subsequent calls will be triggered via cron service)" COLLECTOR_CRON_SCHEDULE= COLLECTOR_PERF_RUN_STARTUP= /opt/scrutiny/bin/scrutiny-collector-performance run fi # now that we have the env start cron in the foreground -echo "starting cron" +log_info "starting cron" exec su -c "cron -f -L 15" root diff --git a/docker/entrypoint-collector-zfs.sh b/docker/entrypoint-collector-zfs.sh index 5ee93f2a8..9f2a13059 100644 --- a/docker/entrypoint-collector-zfs.sh +++ b/docker/entrypoint-collector-zfs.sh @@ -5,6 +5,10 @@ # will then source into the crontab file (/etc/cron.d/scrutiny-zfs) (set -o posix; export -p) > /env.sh +log_info() { + printf 'time="%s" level=info msg="%s" type=zfs\n' "$(date -u +"%Y-%m-%dT%H:%M:%SZ")" "$1" +} + # adding ability to customize the cron schedule. COLLECTOR_ZFS_CRON_SCHEDULE=${COLLECTOR_ZFS_CRON_SCHEDULE:-"*/15 * * * *"} COLLECTOR_ZFS_RUN_STARTUP=${COLLECTOR_ZFS_RUN_STARTUP:-"false"} @@ -18,11 +22,11 @@ sed -i 's|{COLLECTOR_ZFS_CRON_SCHEDULE}|'"${COLLECTOR_ZFS_CRON_SCHEDULE}"'|g' /e if [[ "${COLLECTOR_ZFS_RUN_STARTUP}" == "true" ]]; then sleep ${COLLECTOR_ZFS_RUN_STARTUP_SLEEP} - echo "starting scrutiny ZFS collector (run-once mode. subsequent calls will be triggered via cron service)" + log_info "starting scrutiny ZFS collector (run-once mode. subsequent calls will be triggered via cron service)" COLLECTOR_CRON_SCHEDULE= COLLECTOR_ZFS_RUN_STARTUP= /opt/scrutiny/bin/scrutiny-collector-zfs run fi # now that we have the env start cron in the foreground -echo "starting cron" +log_info "starting cron" exec su -c "cron -f -L 15" root diff --git a/docker/entrypoint-collector.sh b/docker/entrypoint-collector.sh index 390628b04..4306c16eb 100755 --- a/docker/entrypoint-collector.sh +++ b/docker/entrypoint-collector.sh @@ -5,6 +5,10 @@ # will then source into the crontab file (/etc/cron.d/scrutiny) (set -o posix; export -p) > /env.sh +log_info() { + printf 'time="%s" level=info msg="%s" type=metrics\n' "$(date -u +"%Y-%m-%dT%H:%M:%SZ")" "$1" +} + # adding ability to customize the cron schedule. COLLECTOR_CRON_SCHEDULE=${COLLECTOR_CRON_SCHEDULE:-"0 0 * * *"} COLLECTOR_RUN_STARTUP=${COLLECTOR_RUN_STARTUP:-"false"} @@ -18,11 +22,11 @@ sed -i 's|{COLLECTOR_CRON_SCHEDULE}|'"${COLLECTOR_CRON_SCHEDULE}"'|g' /etc/cron. if [[ "${COLLECTOR_RUN_STARTUP}" == "true" ]]; then sleep ${COLLECTOR_RUN_STARTUP_SLEEP} - echo "starting scrutiny collector (run-once mode. subsequent calls will be triggered via cron service)" + log_info "starting scrutiny collector (run-once mode. subsequent calls will be triggered via cron service)" COLLECTOR_CRON_SCHEDULE= COLLECTOR_RUN_STARTUP= /opt/scrutiny/bin/scrutiny-collector-metrics run fi # now that we have the env start cron in the foreground -echo "starting cron" +log_info "starting cron" exec su -c "cron -f -L 15" root diff --git a/docker/example.hubspoke.docker-compose.yml b/docker/example.hubspoke.docker-compose.yml index 51a16cab4..ba5f30c4c 100644 --- a/docker/example.hubspoke.docker-compose.yml +++ b/docker/example.hubspoke.docker-compose.yml @@ -3,9 +3,11 @@ version: '2.4' services: influxdb: restart: unless-stopped - image: influxdb:2.2 + image: influxdb:2.9 ports: - '8086:8086' + environment: + INFLUXD_USE_HASHED_TOKENS: "false" volumes: - './influxdb:/var/lib/influxdb2' healthcheck: @@ -47,10 +49,11 @@ services: collector: restart: unless-stopped - image: 'ghcr.io/starosdev/scrutiny:latest-collector' + image: 'ghcr.io/starosdev/scrutiny:latest-collector-omnibus' cap_add: - SYS_RAWIO - SYS_ADMIN # Required for NVMe devices + # Use latest-collector instead if you only want SMART metrics in this spoke. # AppArmor: on systems with AppArmor enforced (TrueNAS SCALE, Ubuntu, Debian), # the default Docker profile blocks raw device I/O even with SYS_RAWIO/SYS_ADMIN. # Option 1 (recommended): load the custom profile on the host, then uncomment: @@ -63,12 +66,35 @@ services: # - apparmor=unconfined volumes: - '/run/udev:/run/udev:ro' + # Filesystem/Btrfs collectors need host mount visibility: + # - '/mnt:/mnt:ro' + # - '/var/lib/btrfs:/var/lib/btrfs:ro' + # MDADM collector needs host mdstat and broad device visibility: + # - '/dev:/dev' + # - '/proc/mdstat:/host/proc/mdstat:ro' + # Performance collector can use mounted filesystem paths for fio targets: + # - '/mnt/data:/mnt/data' + # - '/mnt/backup:/mnt/backup' environment: COLLECTOR_API_ENDPOINT: 'http://web:8080' COLLECTOR_HOST_ID: 'scrutiny-collector-hostname' # If true forces the collector to run on startup (cron will be started after the collector completes) # see: https://github.com/Starosdev/scrutiny/blob/master/docs/TROUBLESHOOTING_DEVICE_COLLECTOR.md#collector-trigger-on-startup COLLECTOR_RUN_STARTUP: false + # Optional collectors stay disabled until their own schedules or run-on-startup flags are set. + # COLLECTOR_ZFS_CRON_SCHEDULE: '*/15 * * * *' + # COLLECTOR_ZFS_RUN_STARTUP: 'true' + # COLLECTOR_MDADM_CRON_SCHEDULE: '*/15 * * * *' + # COLLECTOR_MDADM_RUN_STARTUP: 'true' + # COLLECTOR_BTRFS_CRON_SCHEDULE: '*/15 * * * *' + # COLLECTOR_BTRFS_RUN_STARTUP: 'true' + # COLLECTOR_FILESYSTEM_CRON_SCHEDULE: '*/15 * * * *' + # COLLECTOR_FILESYSTEM_RUN_STARTUP: 'true' + # COLLECTOR_PERF_CRON_SCHEDULE: '0 2 * * 0' + # COLLECTOR_PERF_RUN_STARTUP: 'false' + # COLLECTOR_PERF_PROFILE: 'quick' + # COLLECTOR_PERFORMANCE_MOUNT_POINTS_SDA: '/mnt/data' + # COLLECTOR_PERFORMANCE_MOUNT_POINTS_SDB: '/mnt/backup' # Required when server auth is enabled (must match SCRUTINY_WEB_AUTH_TOKEN) # COLLECTOR_API_TOKEN: 'your-secret-api-token' depends_on: @@ -81,7 +107,7 @@ services: # - "/dev/nvme0" # - "/dev/nvme0n1" - # ZFS Pool Collector (optional - only needed if you use ZFS) + # Smaller dedicated collector images remain available when you do not want the collector omnibus image: # See docs/ZFS_POOL_MONITORING.md for details # collector-zfs: # restart: unless-stopped diff --git a/docker/unraid/scrutiny-collector-omnibus.xml b/docker/unraid/scrutiny-collector-omnibus.xml new file mode 100644 index 000000000..8fcdb336b --- /dev/null +++ b/docker/unraid/scrutiny-collector-omnibus.xml @@ -0,0 +1,41 @@ + + + scrutiny-collector-omnibus + ghcr.io/starosdev/scrutiny:latest-collector-omnibus + https://github.com/Starosdev/scrutiny/pkgs/container/scrutiny + + latest-collector-omnibus + Stable collector omnibus spoke release + + bridge + false + https://github.com/Starosdev/scrutiny/issues + https://github.com/Starosdev/scrutiny + Scrutiny Collector Omnibus (Hub/Spoke mode). Bundles the SMART, ZFS, MDADM, Btrfs, filesystem, and performance collectors in one spoke container. Requires a running Scrutiny web server instance. Optional collectors stay disabled until their schedule or startup variables are set and the required host mounts are provided. + Tools: Productivity: + https://raw.githubusercontent.com/Starosdev/scrutiny/master/webapp/frontend/src/assets/images/logo/scrutiny-logo-dark.png + --cap-add SYS_RAWIO --cap-add SYS_ADMIN + bash + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 32e417c0e..af4293203 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -33,13 +33,13 @@ flowchart LR ## Hub/Spoke -The Hub/Spoke deployment separates the collector from the central web/API service. One or more `latest-collector` containers run on the systems that have access to disks, then send their results to a hub running `latest-web` plus its databases. +The Hub/Spoke deployment separates the collector from the central web/API service. One or more collector containers run on the systems that have access to disks, then send their results to a hub running `latest-web` plus its databases. Use `latest-collector` for SMART-only spokes or `latest-collector-omnibus` when you want one spoke container that can also run the ZFS, MDADM, Btrfs, filesystem, and performance collectors. ```mermaid flowchart LR Browser["Frontend / Browser"] CollectorA["Collector (`latest-collector`)"] - CollectorB["Collector (`latest-collector`)"] + CollectorB["Collector Omnibus (`latest-collector-omnibus`)"] subgraph Hub["Hub services"] Web["Web/API server (`latest-web`)"] @@ -56,6 +56,7 @@ flowchart LR ``` - Collectors run close to the disks they monitor and communicate with the hub over the network. +- The collector-omnibus spoke image bundles the non-web collector binaries and host tools, but still uses the same per-collector env vars and schedules to decide which optional collectors run. - The hub's web/API service aggregates data from all spokes and serves the frontend. - InfluxDB remains the time-series store for SMART history. - SQLite remains the metadata store for device records, settings, and other app state managed by the web/API layer. diff --git a/docs/AUTH.md b/docs/AUTH.md index ae245a43b..7807e9056 100644 --- a/docs/AUTH.md +++ b/docs/AUTH.md @@ -317,7 +317,7 @@ version: '3.5' services: influxdb: - image: influxdb:2.2-alpine + image: influxdb:2.9-alpine ports: - '8086:8086' volumes: @@ -329,6 +329,7 @@ services: DOCKER_INFLUXDB_INIT_ORG: scrutiny DOCKER_INFLUXDB_INIT_BUCKET: metrics DOCKER_INFLUXDB_INIT_ADMIN_TOKEN: my-super-secret-auth-token + INFLUXD_USE_HASHED_TOKENS: "false" web: image: ghcr.io/starosdev/scrutiny:latest-web diff --git a/docs/DEPLOYMENTS.md b/docs/DEPLOYMENTS.md index 3c2f1ac4b..14de3f425 100644 --- a/docs/DEPLOYMENTS.md +++ b/docs/DEPLOYMENTS.md @@ -17,7 +17,7 @@ For release-version verification details, see [RELEASE_VERSION_VERIFICATION.md]( - Check out the repo - Normalize the GHCR image name to lowercase - Build the omnibus image for `linux/amd64` and `linux/arm64` -- Build the default `web`, `collector-zfs`, `collector-mdadm`, `collector-btrfs`, and `collector-performance` images for `linux/amd64` and `linux/arm64` +- Build the default `collector-omnibus`, `web`, `collector-zfs`, `collector-mdadm`, `collector-btrfs`, and `collector-performance` images for `linux/amd64` and `linux/arm64` - Build the base `collector` image for `linux/amd64`, `linux/arm64`, and `linux/arm/v7` - Exclude `webapp/backend/pkg/version/version.go` from the Docker workflow path trigger so release-version sync commits do not rebuild images on their own - Push the published tags to GHCR @@ -55,25 +55,31 @@ The workflows use the built-in GitHub token to push images to `ghcr.io`. Environment rollout is outside GitHub Actions. -If Zeus should move to a new image, do that from the host by pulling the published tags and restarting the compose project there. The current Zeus mapping is still: +If Zeus should move to a new image, do that from the host by pulling the published tags and restarting the compose project there. The current Zeus mapping is: - develop image path: `ghcr.io/starosdev/scrutiny:develop-omnibus` - beta image path: `ghcr.io/starosdev/scrutiny:beta-omnibus` - production image path: `ghcr.io/starosdev/scrutiny:latest` -- develop port: `8680` -- beta port: choose a separate host port for side-by-side validation if you run it alongside testing or production +- develop compose project: `scrutiny-develop` +- beta compose project: `scrutiny-beta` +- production compose project: `scrutiny` +- develop port: `8780` +- beta port: `8680` - production port: `8580` +- develop appdata root: `/mnt/user/appdata/scrutiny-develop` +- beta appdata root: `/mnt/user/appdata/scrutiny-beta` - production appdata root: `/mnt/user/appdata/scrutiny` -- Zeus testing appdata root: `/mnt/user/appdata/scrutiny-dev` +- develop compose file: `/mnt/user/appdata/scrutiny-develop/docker-compose.yml` +- beta compose file: `/mnt/user/appdata/scrutiny-beta/docker-compose.yml` - production compose file: `/mnt/user/appdata/scrutiny/docker-compose.yml` -- testing compose file: `/mnt/user/appdata/scrutiny-dev/docker-compose.yml` ## Current Zeus Host Layout -Zeus does not currently run testing and production from the same appdata tree. +Zeus currently runs three separate appdata trees and compose projects side by side. +- Develop uses `/mnt/user/appdata/scrutiny-develop` +- Beta uses `/mnt/user/appdata/scrutiny-beta` - Production uses `/mnt/user/appdata/scrutiny` -- Testing uses `/mnt/user/appdata/scrutiny-dev` This repo now treats `beta` as an optional pre-release channel for changes that need validation before `master`. @@ -84,12 +90,41 @@ This repo now treats `beta` as an optional pre-release channel for changes that That distinction matters for both manual host rollouts and the helper scripts in `ops/`: - `ops/deploy-production.sh` should target `/mnt/user/appdata/scrutiny/docker-compose.yml` with compose project `scrutiny` -- `ops/deploy-testing.sh` should target `/mnt/user/appdata/scrutiny-dev/docker-compose.yml` with compose project `scrutiny-dev` +- `ops/deploy-testing.sh` should target `/mnt/user/appdata/scrutiny-develop/docker-compose.yml` with compose project `scrutiny-develop` -If you point the testing deploy helper at `/mnt/user/appdata/scrutiny`, you will be operating on the production environment instead of Zeus testing. +There is no repo-owned beta deploy helper today. Beta rollouts on Zeus are still host-side operations against `/mnt/user/appdata/scrutiny-beta/docker-compose.yml`. + +If you point the develop deploy helper at `/mnt/user/appdata/scrutiny`, you will be operating on the production environment instead of the development environment. The `deploy/` compose files in this repo remain available as repo-owned examples, but the Zeus helpers default to the live appdata-root compose files because those are what the host actually runs today. +### Zeus Develop Rebuild Preflight + +- **Problem:** Unraid can renumber `/dev/sd*` devices. A stale compose `devices:` source prevents Docker from recreating the container and leaves it exited with an error such as `error gathering device information ... no such file or directory`. +- **Approach:** Compare every configured device source with the current `lsblk` output. Map a missing path to the same physical disk by model and serial before updating the develop compose file. +- **Dead end:** Re-running `docker compose up` without correcting a missing device path repeats the exit-128 failure; restarting the old container does not repair the compose configuration. +- **Rule:** Every device source in `/mnt/user/appdata/scrutiny-develop/docker-compose.yml` must exist before a forced recreate. + +The deploy helper requires a checkout at `/mnt/user/appdata/scrutiny-develop/repo`. If that checkout is absent, use the live compose project directly: + +```bash +ROOT=/mnt/user/appdata/scrutiny-develop + +docker compose \ + -p scrutiny-develop \ + -f "$ROOT/docker-compose.yml" \ + --env-file "$ROOT/testing.env" \ + pull + +docker compose \ + -p scrutiny-develop \ + -f "$ROOT/docker-compose.yml" \ + --env-file "$ROOT/testing.env" \ + up -d --force-recreate --remove-orphans +``` + +Before an embedded InfluxDB upgrade, back up the complete `influxdb` directory and follow the rollback guidance in [TROUBLESHOOTING_INFLUXDB.md](./TROUBLESHOOTING_INFLUXDB.md). + ## Zeus MDADM Testing Notes For actual deployment and troubleshooting steps, see [MDADM_MONITORING.md](./MDADM_MONITORING.md). diff --git a/docs/INSTALL_HUB_SPOKE.md b/docs/INSTALL_HUB_SPOKE.md index 89a756ea1..c26605bd0 100644 --- a/docs/INSTALL_HUB_SPOKE.md +++ b/docs/INSTALL_HUB_SPOKE.md @@ -61,7 +61,7 @@ services: influxdb: restart: unless-stopped container_name: influxdb - image: influxdb:2.1-alpine + image: influxdb:2.9-alpine ports: - 8086:8086 volumes: @@ -74,6 +74,7 @@ services: - DOCKER_INFLUXDB_INIT_ORG=homelab - DOCKER_INFLUXDB_INIT_BUCKET=scrutiny - DOCKER_INFLUXDB_INIT_ADMIN_TOKEN=your-very-secret-token + - INFLUXD_USE_HASHED_TOKENS="false" networks: - monitoring diff --git a/docs/INSTALL_MANUAL.md b/docs/INSTALL_MANUAL.md index 20ba0e0c6..238919b0c 100644 --- a/docs/INSTALL_MANUAL.md +++ b/docs/INSTALL_MANUAL.md @@ -10,9 +10,23 @@ Scrutiny is made up of three components: an influxdb Database, a collector and a ## InfluxDB -Please follow the official InfluxDB installation guide. Note, you'll need to install v2.2.0+. +Please follow the official InfluxDB installation guide. Note, you'll need to install v2.9.0+. -https://docs.influxdata.com/influxdb/v2.2/install/ +https://docs.influxdata.com/influxdb/v2/install/ + +**Important**: Starting with InfluxDB 2.9.0, token hashing is enabled by default. If you want to keep tokens unhashed (for compatibility or debugging), set the following environment variable before starting InfluxDB: + +1. Edit `/etc/default/influxdb2` and add: + +``` +ARG1="--use-hashed-tokens=false" +``` + +2. Edit `/lib/systemd/system/influxdb.service` and update the `ExecStart` line to pass the variable: + +``` +ExecStart=/usr/bin/influxd $ARG1 +``` ## Webapp/API diff --git a/docs/INSTALL_UNRAID.md b/docs/INSTALL_UNRAID.md index 416e97c70..410341942 100644 --- a/docs/INSTALL_UNRAID.md +++ b/docs/INSTALL_UNRAID.md @@ -39,6 +39,7 @@ For distributed deployments, separate templates are available: |----------|-------|---------| | `scrutiny-web.xml` | `ghcr.io/starosdev/scrutiny:latest-web` | Web dashboard and API. Requires external InfluxDB. | | `scrutiny-collector.xml` | `ghcr.io/starosdev/scrutiny:latest-collector` | S.M.A.R.T metrics collector. Point it at your web server. | +| `scrutiny-collector-omnibus.xml` | `ghcr.io/starosdev/scrutiny:latest-collector-omnibus` | Recommended single spoke container. Bundles the SMART, ZFS, MDADM, Btrfs, filesystem, and performance collectors. | | `scrutiny-collector-zfs.xml` | `ghcr.io/starosdev/scrutiny:latest-collector-zfs` | ZFS pool health collector (amd64 only). | | `scrutiny-collector-btrfs.xml` | `ghcr.io/starosdev/scrutiny:latest-collector-btrfs` | Btrfs filesystem health collector. Requires host Btrfs mounts and `btrfs-progs` access. | | `scrutiny-collector-performance.xml` | `ghcr.io/starosdev/scrutiny:latest-collector-performance` | fio drive benchmarks for throughput/IOPS/latency tracking. | diff --git a/docs/MDADM_MONITORING.md b/docs/MDADM_MONITORING.md index d3c4b240d..2f91f901d 100644 --- a/docs/MDADM_MONITORING.md +++ b/docs/MDADM_MONITORING.md @@ -107,6 +107,12 @@ curl -s http://localhost:8080/api/mdadm/summary | jq . 4. Open the MDADM page in the UI and verify the arrays show a state and size, not just static metadata. +Upgrade note: + +- builds that persist MDADM `host_id` on array re-registration do not retroactively backfill older rows +- if arrays were first registered before that fix, run one fresh MDADM collection on each affected host +- confirm `GET /api/mdadm/summary` now includes `host_id` for those arrays before expecting grouped host headings in the UI + ## Troubleshooting ### `No MDADM arrays found` @@ -145,6 +151,16 @@ Check the backend logs for a per-array registration error. Common causes include - the array UUID collided with another discovered array - the database rejected one array record while accepting the others +### Arrays are present but not grouped under the correct host + +Check: + +- `GET /api/mdadm/summary` includes a non-empty `host_id` for each affected array +- the affected host has completed at least one fresh MDADM collector registration after upgrading +- the collector `host.id` value is stable and not changing between runs + +If older rows were created before `host_id` was persisted on re-registration, force one new collection on each affected host to backfill the stored `host_id`. + ## Related Docs - [INSTALL_UNRAID.md](./INSTALL_UNRAID.md) diff --git a/docs/RELEASE_VERSION_VERIFICATION.md b/docs/RELEASE_VERSION_VERIFICATION.md index 1ec70a4fe..e544ea95d 100644 --- a/docs/RELEASE_VERSION_VERIFICATION.md +++ b/docs/RELEASE_VERSION_VERIFICATION.md @@ -20,7 +20,7 @@ Issue `#496` was investigated against the current release flow on May 10, 2026. - `scrutiny-collector-zfs-darwin-arm64` - `scrutiny-collector-btrfs-darwin-arm64` - `scrutiny-collector-performance-darwin-arm64` -- Ran each binary with no arguments and confirmed the startup banner showed the expected semantic version, for example `darwin.arm64-1.53.1`. +- Ran each binary with no arguments and confirmed the startup banner showed the expected semantic version, for example `darwin.arm64-1.53.1`. This check assumes `SCRUTINY_NO_LOGO` is unset, because banner suppression is opt-in. - Built the same five binaries locally from the `v1.53.1` tag using: ```bash @@ -42,4 +42,5 @@ GOOS=darwin GOARCH=arm64 go build -buildvcs=false \ ## Notes - Development builds may still show `dev-` when `goos` and `goarch` are not injected. +- If `SCRUTINY_NO_LOGO=true` is set, the banner check above will be suppressed by design. Unset it before using the no-arguments startup path for release verification. - A local `scrutiny start` check may fail before `/api/settings` is reachable if the environment does not provide the required SQLite path or InfluxDB dependency. That startup failure is separate from release-version injection. diff --git a/docs/TROUBLESHOOTING_INFLUXDB.md b/docs/TROUBLESHOOTING_INFLUXDB.md index 5eb2f21ba..e8a7cf99c 100644 --- a/docs/TROUBLESHOOTING_INFLUXDB.md +++ b/docs/TROUBLESHOOTING_INFLUXDB.md @@ -41,14 +41,14 @@ The growth rate is pretty unintuitive -- see https://github.com/AnalogJ/scrutiny InfluxDB is a required dependency for Scrutiny v0.4.0+. -https://docs.influxdata.com/influxdb/v2.2/install/ +https://docs.influxdata.com/influxdb/v2/install/ ## Persistence To ensure that all data is correctly stored, you must also persist the InfluxDB database directory - If you're using the Official Scrutiny Omnibus image (`ghcr.io/Starosdev/scrutiny:master-omnibus`), the path is `/opt/scrutiny/influxdb` -- If you're deploying in Hub/Spoke mode with the InfluxDB maintained image (`influxdb:2.2`), the path is `/var/lib/influxdb2` +- If you're deploying in Hub/Spoke mode with the InfluxDB maintained image (`influxdb:2.9`), the path is `/var/lib/influxdb2` If you attempt to restart Scrutiny but you forgot to persist the InfluxDB directory, you will get an error message like follows: @@ -95,7 +95,7 @@ panic: a username and password is required for a setup or ``` -Start the scrutiny server +time="2022-06-11T10:35:04-04:00" level=info msg="Start the scrutiny server" type=web time="2022-06-11T10:35:04-04:00" level=info msg="Trying to connect to scrutiny sqlite db: \n" time="2022-06-11T10:35:04-04:00" level=info msg="Successfully connected to scrutiny sqlite db: \n" panic: failed to check influxdb setup status - parse "://:": missing protocol scheme @@ -450,3 +450,55 @@ web: It's safe to change the InfluxDB Admin username/password after setup has completed, only the API token is used for subsequent communication with InfluxDB. + + +## Important: Token Hashing and Downgrade Compatibility + +**InfluxDB v2.9 introduced token hashing by default.** Scrutiny keeps this **disabled by default**, so most users can downgrade safely. + +If InfluxDB 2.9 was started **without explicitly disabling** token hashing, tokens will be hashed and become incompatible with v2.2, causing **HTTP 401 Unauthorized errors** upon downgrade. + +#### If token hashing is disabled (recommended path) +Proceed with the downgrade steps below — your tokens will work fine in v2.2. + +#### If token hashing is enabled +Before downgrading, you have two options: + +1. Restore from a pre-upgrade backup of your `influxdb2` directory taken before upgrading to v2.9. +2. Recreate your tokens and update Scrutiny's configuration with the new token values. + +``` +docker run --rm \ + -v ./influxdb2:/opt/scrutiny/influxdb \ + influxdb:2.2 \ + influxd recovery auth create-operator \ + --bolt-path /opt/scrutiny/influxdb/influxd.bolt \ + --org \ + --username +``` +The generated token should then replace `web.influxdb.token` before restarting Scrutiny. + +### Downgrading path: + +> Note for Hub/Spoke users: Substitute your configured container name and host:container data path in the commands below. + +For Omnibus users: + +1. Stop Scrutiny and backup your InfluxDB data: + +``` +# stop the container or stack via docker compose +docker stop scrutiny +cp -a ./influxdb2 ./influxdb.2.9-backup +``` + +2. Run the InfluxDB downgrade command: + +``` +docker run --rm \ + -v ./influxdb2:/opt/scrutiny/influxdb \ + influxdb:2.9.1 \ + influxd downgrade 2.1 \ + --bolt-path /opt/scrutiny/influxdb/influxd.bolt \ + --sqlite-path /opt/scrutiny/influxdb/influxd.sqlite +``` \ No newline at end of file diff --git a/ops/deploy-testing.sh b/ops/deploy-testing.sh index d83b6af1a..1d136d843 100755 --- a/ops/deploy-testing.sh +++ b/ops/deploy-testing.sh @@ -1,14 +1,14 @@ #!/usr/bin/env bash set -euo pipefail -REPO_DIR="${SCRUTINY_REPO_DIR:-/mnt/user/appdata/scrutiny-dev/repo}" -ENV_FILE="${SCRUTINY_ENV_FILE:-/mnt/user/appdata/scrutiny-dev/testing.env}" +REPO_DIR="${SCRUTINY_REPO_DIR:-/mnt/user/appdata/scrutiny-develop/repo}" +ENV_FILE="${SCRUTINY_ENV_FILE:-/mnt/user/appdata/scrutiny-develop/testing.env}" APPDATA_ROOT="${SCRUTINY_APPDATA_ROOT:-${REPO_DIR%/repo}}" COMPOSE_FILE="${SCRUTINY_COMPOSE_FILE:-$APPDATA_ROOT/docker-compose.yml}" -PROJECT="${SCRUTINY_PROJECT_NAME:-scrutiny-dev}" +PROJECT="${SCRUTINY_PROJECT_NAME:-scrutiny-develop}" BRANCH="${SCRUTINY_DEPLOY_BRANCH:-develop}" -echo "=== Scrutiny Testing Deploy ===" +echo "=== Scrutiny Develop Deploy ===" echo "Timestamp: $(date -u +%Y-%m-%dT%H:%M:%SZ)" echo "Target appdata root: $APPDATA_ROOT" diff --git a/pkg/startup/startup.go b/pkg/startup/startup.go new file mode 100644 index 000000000..4d6d86037 --- /dev/null +++ b/pkg/startup/startup.go @@ -0,0 +1,53 @@ +package startup + +import ( + "os" + "strconv" + "strings" + + "github.com/sirupsen/logrus" + "go.uber.org/automaxprocs/maxprocs" +) + +const ( + NoLogoEnv = "SCRUTINY_NO_LOGO" + logLevelKey = "log.level" +) + +type StringGetter interface { + GetString(key string) string +} + +func ShouldPrintBanner() bool { + return !logoSuppressed(os.Getenv(NoLogoEnv)) +} + +func NewBootstrapLogger(component string, cfg StringGetter) *logrus.Entry { + logger := logrus.WithField("type", component) + logger.Logger.SetLevel(bootstrapLevel(cfg)) + return logger +} + +func ConfigureMaxProcs(logger *logrus.Entry) { + _, err := maxprocs.Set(maxprocs.Logger(func(format string, args ...interface{}) { + logger.Infof(strings.TrimSuffix(format, "\n"), args...) + })) + if err != nil { + logger.Warnf("maxprocs: %v", err) + } +} + +func bootstrapLevel(cfg StringGetter) logrus.Level { + if level, err := logrus.ParseLevel(cfg.GetString(logLevelKey)); err == nil { + return level + } + return logrus.InfoLevel +} + +func logoSuppressed(value string) bool { + if strings.TrimSpace(value) == "" { + return false + } + parsed, err := strconv.ParseBool(strings.TrimSpace(value)) + return err == nil && parsed +} diff --git a/pkg/startup/startup_test.go b/pkg/startup/startup_test.go new file mode 100644 index 000000000..36cec3941 --- /dev/null +++ b/pkg/startup/startup_test.go @@ -0,0 +1,50 @@ +package startup + +import ( + "os" + "testing" + + "github.com/sirupsen/logrus" +) + +type stubConfig map[string]string + +func (s stubConfig) GetString(key string) string { + return s[key] +} + +func TestShouldPrintBanner(t *testing.T) { + original, hadOriginal := os.LookupEnv(NoLogoEnv) + t.Cleanup(func() { + if hadOriginal { + _ = os.Setenv(NoLogoEnv, original) + return + } + _ = os.Unsetenv(NoLogoEnv) + }) + + _ = os.Unsetenv(NoLogoEnv) + if !ShouldPrintBanner() { + t.Fatalf("expected banner to print when %s is unset", NoLogoEnv) + } + + _ = os.Setenv(NoLogoEnv, "true") + if ShouldPrintBanner() { + t.Fatalf("expected banner to be suppressed when %s=true", NoLogoEnv) + } + + _ = os.Setenv(NoLogoEnv, "invalid") + if !ShouldPrintBanner() { + t.Fatalf("expected invalid %s value to keep banner enabled", NoLogoEnv) + } +} + +func TestBootstrapLevel(t *testing.T) { + if level := bootstrapLevel(stubConfig{logLevelKey: "warn"}); level != logrus.WarnLevel { + t.Fatalf("expected warn level, got %s", level) + } + + if level := bootstrapLevel(stubConfig{logLevelKey: "not-a-level"}); level != logrus.InfoLevel { + t.Fatalf("expected info fallback, got %s", level) + } +} diff --git a/rootfs/etc/services.d/collector-btrfs-once/run b/rootfs/etc/services.d/collector-btrfs-once/run index bd0f8a4a4..f847609b8 100644 --- a/rootfs/etc/services.d/collector-btrfs-once/run +++ b/rootfs/etc/services.d/collector-btrfs-once/run @@ -1,26 +1,30 @@ #!/command/with-contenv bash +log_info() { + printf 'time="%s" level=info msg="%s" type=btrfs\n' "$(date -u +"%Y-%m-%dT%H:%M:%SZ")" "$1" +} + # Only run if Btrfs collection is enabled if [ -z "${COLLECTOR_BTRFS_CRON_SCHEDULE}" ] && [ "${COLLECTOR_BTRFS_RUN_STARTUP}" != "true" ]; then - echo 'INFO: Btrfs collector not enabled' + log_info "Btrfs collector not enabled" s6-svc -D /run/service/collector-btrfs-once exit 0 fi # ensure not run before if [ -f /tmp/btrfs-collector-init-performed ]; then - echo 'INFO: Btrfs collector init already performed' + log_info "Btrfs collector init already performed" s6-svc -D /run/service/collector-btrfs-once exit 0 fi -echo "waiting for scrutiny service to start" +log_info "waiting for scrutiny service to start" s6-svwait -u /run/service/scrutiny # wait until scrutiny is "Ready" -until $(curl --output /dev/null --silent --head --fail http://localhost:8080/api/health); do echo "scrutiny api not ready" && sleep 5; done +until $(curl --output /dev/null --silent --head --fail http://localhost:8080/api/health); do log_info "scrutiny api not ready" && sleep 5; done -echo "starting Btrfs collector (run-once mode)" +log_info "starting Btrfs collector (run-once mode)" COLLECTOR_CRON_SCHEDULE= COLLECTOR_BTRFS_RUN_STARTUP= /opt/scrutiny/bin/scrutiny-collector-btrfs run touch /tmp/btrfs-collector-init-performed diff --git a/rootfs/etc/services.d/collector-filesystem-once/run b/rootfs/etc/services.d/collector-filesystem-once/run index 7193935a4..65279f1e2 100644 --- a/rootfs/etc/services.d/collector-filesystem-once/run +++ b/rootfs/etc/services.d/collector-filesystem-once/run @@ -1,26 +1,30 @@ #!/command/with-contenv bash +log_info() { + printf 'time="%s" level=info msg="%s" type=filesystem\n' "$(date -u +"%Y-%m-%dT%H:%M:%SZ")" "$1" +} + # Only run if filesystem collection is enabled if [ -z "${COLLECTOR_FILESYSTEM_CRON_SCHEDULE}" ] && [ "${COLLECTOR_FILESYSTEM_RUN_STARTUP}" != "true" ]; then - echo 'INFO: Filesystem collector not enabled' + log_info "Filesystem collector not enabled" s6-svc -D /run/service/collector-filesystem-once exit 0 fi # ensure not run before if [ -f /tmp/filesystem-collector-init-performed ]; then - echo 'INFO: Filesystem collector init already performed' + log_info "Filesystem collector init already performed" s6-svc -D /run/service/collector-filesystem-once exit 0 fi -echo "waiting for scrutiny service to start" +log_info "waiting for scrutiny service to start" s6-svwait -u /run/service/scrutiny # wait until scrutiny is "Ready" -until $(curl --output /dev/null --silent --head --fail http://localhost:8080/api/health); do echo "scrutiny api not ready" && sleep 5; done +until $(curl --output /dev/null --silent --head --fail http://localhost:8080/api/health); do log_info "scrutiny api not ready" && sleep 5; done -echo "starting filesystem collector (run-once mode)" +log_info "starting filesystem collector (run-once mode)" COLLECTOR_CRON_SCHEDULE= /opt/scrutiny/bin/scrutiny-collector-filesystem run touch /tmp/filesystem-collector-init-performed diff --git a/rootfs/etc/services.d/collector-mdadm-once/run b/rootfs/etc/services.d/collector-mdadm-once/run index e25184dd7..1693ca7d4 100644 --- a/rootfs/etc/services.d/collector-mdadm-once/run +++ b/rootfs/etc/services.d/collector-mdadm-once/run @@ -1,26 +1,30 @@ #!/command/with-contenv bash +log_info() { + printf 'time="%s" level=info msg="%s" type=mdadm\n' "$(date -u +"%Y-%m-%dT%H:%M:%SZ")" "$1" +} + # Only run if MDADM collection is enabled if [ -z "${COLLECTOR_MDADM_CRON_SCHEDULE}" ] && [ "${COLLECTOR_MDADM_RUN_STARTUP}" != "true" ]; then - echo 'INFO: MDADM collector not enabled' + log_info "MDADM collector not enabled" s6-svc -D /run/service/collector-mdadm-once exit 0 fi # ensure not run before if [ -f /tmp/mdadm-collector-init-performed ]; then - echo 'INFO: MDADM collector init already performed' + log_info "MDADM collector init already performed" s6-svc -D /run/service/collector-mdadm-once exit 0 fi -echo "waiting for scrutiny service to start" +log_info "waiting for scrutiny service to start" s6-svwait -u /run/service/scrutiny # wait until scrutiny is "Ready" -until $(curl --output /dev/null --silent --head --fail http://localhost:8080/api/health); do echo "scrutiny api not ready" && sleep 5; done +until $(curl --output /dev/null --silent --head --fail http://localhost:8080/api/health); do log_info "scrutiny api not ready" && sleep 5; done -echo "starting MDADM collector (run-once mode)" +log_info "starting MDADM collector (run-once mode)" COLLECTOR_CRON_SCHEDULE= COLLECTOR_MDADM_RUN_STARTUP= /opt/scrutiny/bin/scrutiny-collector-mdadm run touch /tmp/mdadm-collector-init-performed diff --git a/rootfs/etc/services.d/collector-once/run b/rootfs/etc/services.d/collector-once/run index cfdbba6d3..c3eb7e731 100644 --- a/rootfs/etc/services.d/collector-once/run +++ b/rootfs/etc/services.d/collector-once/run @@ -1,19 +1,23 @@ #!/command/with-contenv bash +log_info() { + printf 'time="%s" level=info msg="%s" type=metrics\n' "$(date -u +"%Y-%m-%dT%H:%M:%SZ")" "$1" +} + # ensure not run (successfully) before if [ -f /tmp/custom-init-performed ]; then - echo 'INFO: custom init already performed' + log_info "custom init already performed" s6-svc -D /run/service/collector-once # prevent s6 from restarting service exit 0 fi -echo "waiting for scrutiny service to start" +log_info "waiting for scrutiny service to start" s6-svwait -u /run/service/scrutiny # wait until scrutiny is "Ready" -until $(curl --output /dev/null --silent --head --fail http://localhost:8080/api/health); do echo "scrutiny api not ready" && sleep 5; done +until $(curl --output /dev/null --silent --head --fail http://localhost:8080/api/health); do log_info "scrutiny api not ready" && sleep 5; done -echo "starting scrutiny collector (run-once mode. subsequent calls will be triggered via cron service)" +log_info "starting scrutiny collector (run-once mode. subsequent calls will be triggered via cron service)" COLLECTOR_CRON_SCHEDULE= COLLECTOR_RUN_STARTUP= /opt/scrutiny/bin/scrutiny-collector-metrics run # prevent script's core logic from running again diff --git a/rootfs/etc/services.d/collector-performance-once/run b/rootfs/etc/services.d/collector-performance-once/run index 86398aaed..bfc41a64b 100644 --- a/rootfs/etc/services.d/collector-performance-once/run +++ b/rootfs/etc/services.d/collector-performance-once/run @@ -1,26 +1,30 @@ #!/command/with-contenv bash +log_info() { + printf 'time="%s" level=info msg="%s" type=performance\n' "$(date -u +"%Y-%m-%dT%H:%M:%SZ")" "$1" +} + # Only run if performance collection is enabled if [ -z "${COLLECTOR_PERF_CRON_SCHEDULE}" ] && [ "${COLLECTOR_PERF_RUN_STARTUP}" != "true" ]; then - echo 'INFO: Performance collector not enabled' + log_info "Performance collector not enabled" s6-svc -D /run/service/collector-performance-once exit 0 fi # ensure not run before if [ -f /tmp/perf-collector-init-performed ]; then - echo 'INFO: Performance collector init already performed' + log_info "Performance collector init already performed" s6-svc -D /run/service/collector-performance-once exit 0 fi -echo "waiting for scrutiny service to start" +log_info "waiting for scrutiny service to start" s6-svwait -u /run/service/scrutiny # wait until scrutiny is "Ready" -until $(curl --output /dev/null --silent --head --fail http://localhost:8080/api/health); do echo "scrutiny api not ready" && sleep 5; done +until $(curl --output /dev/null --silent --head --fail http://localhost:8080/api/health); do log_info "scrutiny api not ready" && sleep 5; done -echo "starting performance collector (run-once mode)" +log_info "starting performance collector (run-once mode)" COLLECTOR_CRON_SCHEDULE= /opt/scrutiny/bin/scrutiny-collector-performance run touch /tmp/perf-collector-init-performed diff --git a/rootfs/etc/services.d/collector-zfs-once/run b/rootfs/etc/services.d/collector-zfs-once/run index a802e3f3d..12fc04e79 100644 --- a/rootfs/etc/services.d/collector-zfs-once/run +++ b/rootfs/etc/services.d/collector-zfs-once/run @@ -1,26 +1,30 @@ #!/command/with-contenv bash +log_info() { + printf 'time="%s" level=info msg="%s" type=zfs\n' "$(date -u +"%Y-%m-%dT%H:%M:%SZ")" "$1" +} + # Only run if ZFS collection is enabled if [ -z "${COLLECTOR_ZFS_CRON_SCHEDULE}" ] && [ "${COLLECTOR_ZFS_RUN_STARTUP}" != "true" ]; then - echo 'INFO: ZFS collector not enabled' + log_info "ZFS collector not enabled" s6-svc -D /run/service/collector-zfs-once exit 0 fi # ensure not run before if [ -f /tmp/zfs-collector-init-performed ]; then - echo 'INFO: ZFS collector init already performed' + log_info "ZFS collector init already performed" s6-svc -D /run/service/collector-zfs-once exit 0 fi -echo "waiting for scrutiny service to start" +log_info "waiting for scrutiny service to start" s6-svwait -u /run/service/scrutiny # wait until scrutiny is "Ready" -until $(curl --output /dev/null --silent --head --fail http://localhost:8080/api/health); do echo "scrutiny api not ready" && sleep 5; done +until $(curl --output /dev/null --silent --head --fail http://localhost:8080/api/health); do log_info "scrutiny api not ready" && sleep 5; done -echo "starting ZFS collector (run-once mode)" +log_info "starting ZFS collector (run-once mode)" COLLECTOR_CRON_SCHEDULE= /opt/scrutiny/bin/scrutiny-collector-zfs run touch /tmp/zfs-collector-init-performed diff --git a/rootfs/etc/services.d/cron/finish b/rootfs/etc/services.d/cron/finish index 5c2334223..c55005043 100644 --- a/rootfs/etc/services.d/cron/finish +++ b/rootfs/etc/services.d/cron/finish @@ -1,4 +1,4 @@ -#!/command/execlineb -S0 +#!/command/with-contenv bash -echo "cron exiting" +printf 'time="%s" level=info msg="%s" type=cron\n' "$(date -u +"%Y-%m-%dT%H:%M:%SZ")" "cron exiting" s6-svscanctl -t /var/run/s6/services diff --git a/rootfs/etc/services.d/cron/run b/rootfs/etc/services.d/cron/run index 025c1eba9..8cc4c6160 100755 --- a/rootfs/etc/services.d/cron/run +++ b/rootfs/etc/services.d/cron/run @@ -1,4 +1,4 @@ #!/command/with-contenv bash -echo "starting cron" +printf 'time="%s" level=info msg="%s" type=cron\n' "$(date -u +"%Y-%m-%dT%H:%M:%SZ")" "starting cron" exec cron -f -L 15 diff --git a/rootfs/etc/services.d/influxdb/run b/rootfs/etc/services.d/influxdb/run index 51345a850..0d025f0b6 100644 --- a/rootfs/etc/services.d/influxdb/run +++ b/rootfs/etc/services.d/influxdb/run @@ -1,9 +1,13 @@ #!/command/with-contenv bash +log_info() { + printf 'time="%s" level=info msg="%s" type=influxdb\n' "$(date -u +"%Y-%m-%dT%H:%M:%SZ")" "$1" +} + mkdir -p /opt/scrutiny/influxdb/ if [ -f "/opt/scrutiny/influxdb/config.yaml" ]; then - echo "influxdb config file already exists. skipping." + log_info "influxdb config file already exists. skipping." else cat << 'EOF' > /opt/scrutiny/influxdb/config.yaml bolt-path: /opt/scrutiny/influxdb/influxd.bolt @@ -15,5 +19,5 @@ storage-cache-snapshot-memory-size: 25000000 EOF fi -echo "starting influxdb" +log_info "starting influxdb" exec influxd run diff --git a/rootfs/etc/services.d/scrutiny/run b/rootfs/etc/services.d/scrutiny/run index 8145b520b..c40938dfb 100644 --- a/rootfs/etc/services.d/scrutiny/run +++ b/rootfs/etc/services.d/scrutiny/run @@ -1,7 +1,11 @@ #!/command/with-contenv bash -echo "waiting for influxdb" -until $(curl --output /dev/null --silent --head --fail http://localhost:8086/health); do echo "influxdb not ready" && sleep 5; done +log_info() { + printf 'time="%s" level=info msg="%s" type=web\n' "$(date -u +"%Y-%m-%dT%H:%M:%SZ")" "$1" +} -echo "starting scrutiny" +log_info "waiting for influxdb" +until $(curl --output /dev/null --silent --head --fail http://localhost:8086/health); do log_info "influxdb not ready" && sleep 5; done + +log_info "starting scrutiny" exec scrutiny start diff --git a/webapp/backend/cmd/scrutiny/scrutiny.go b/webapp/backend/cmd/scrutiny/scrutiny.go index 8b36c8605..8f01e06b4 100644 --- a/webapp/backend/cmd/scrutiny/scrutiny.go +++ b/webapp/backend/cmd/scrutiny/scrutiny.go @@ -7,9 +7,8 @@ import ( "os" "time" - _ "go.uber.org/automaxprocs" - utils "github.com/analogj/go-util/utils" + "github.com/analogj/scrutiny/pkg/startup" "github.com/analogj/scrutiny/webapp/backend/pkg/config" "github.com/analogj/scrutiny/webapp/backend/pkg/errors" "github.com/analogj/scrutiny/webapp/backend/pkg/version" @@ -28,14 +27,15 @@ const ( ) func main() { - // Create a bootstrap logger early so all startup errors use structured logging - bootstrapLogger := newBootstrapLogger() - cfg, createErr := config.Create() if createErr != nil { - bootstrapLogger.Fatalf("FATAL: %+v", createErr) + fmt.Printf("FATAL: %+v\n", createErr) + os.Exit(1) } + bootstrapLogger := newBootstrapLogger(cfg) + startup.ConfigureMaxProcs(bootstrapLogger) + if err := readOptionalConfig(cfg, resolveWebConfigPath(), bootstrapLogger); err != nil { bootstrapLogger.Error(color.HiRedString("CONFIG ERROR: %v", err)) os.Exit(1) @@ -62,10 +62,8 @@ OPTIONS: } -func newBootstrapLogger() *logrus.Entry { - bootstrapLogger := logrus.WithFields(logrus.Fields{"type": "web"}) - bootstrapLogger.Logger.SetLevel(logrus.InfoLevel) - return bootstrapLogger +func newBootstrapLogger(cfg config.Interface) *logrus.Entry { + return startup.NewBootstrapLogger("web", cfg) } func resolveWebConfigPath() string { @@ -98,7 +96,9 @@ func newCLIApp(cfg config.Interface, bootstrapLogger *logrus.Entry) *cli.App { }, }, Before: func(c *cli.Context) error { - color.New(color.FgGreen).Fprintf(c.App.Writer, "%s", scrutinyBanner("github.com/AnalogJ/scrutiny")) + if startup.ShouldPrintBanner() { + color.New(color.FgGreen).Fprintf(c.App.Writer, "%s", scrutinyBanner("github.com/AnalogJ/scrutiny")) + } return nil }, @@ -107,7 +107,6 @@ func newCLIApp(cfg config.Interface, bootstrapLogger *logrus.Entry) *cli.App { Name: "start", Usage: "Start the scrutiny server", Action: func(c *cli.Context) error { - fmt.Fprintln(c.App.Writer, c.Command.Usage) if c.IsSet("config") { if err := cfg.ReadConfig(c.String("config"), bootstrapLogger); err != nil { // Find and read the config file //ignore "could not find config file" @@ -132,6 +131,8 @@ func newCLIApp(cfg config.Interface, bootstrapLogger *logrus.Entry) *cli.App { return err } + webLogger.Info(c.Command.Usage) + settingsData, err := json.Marshal(cfg.AllSettings()) webLogger.Debug(string(settingsData), err) diff --git a/webapp/backend/pkg/database/scrutiny_repository_mdadm.go b/webapp/backend/pkg/database/scrutiny_repository_mdadm.go index 36f245e78..df1fb7c70 100644 --- a/webapp/backend/pkg/database/scrutiny_repository_mdadm.go +++ b/webapp/backend/pkg/database/scrutiny_repository_mdadm.go @@ -39,6 +39,7 @@ func (sr *scrutinyRepository) RegisterMdadmArray(ctx context.Context, array mode existing.Name = array.Name existing.Level = array.Level existing.Devices = array.Devices + existing.HostID = array.HostID existing.UpdatedAt = array.UpdatedAt if err := sr.gormClient.WithContext(ctx).Save(&existing).Error; err != nil { diff --git a/webapp/backend/pkg/database/scrutiny_repository_mdadm_test.go b/webapp/backend/pkg/database/scrutiny_repository_mdadm_test.go index 714ccdb34..320aa207a 100644 --- a/webapp/backend/pkg/database/scrutiny_repository_mdadm_test.go +++ b/webapp/backend/pkg/database/scrutiny_repository_mdadm_test.go @@ -46,6 +46,32 @@ func TestRegisterMdadmArrayUpdatesExistingArrayDevices(t *testing.T) { require.Equal(t, updated.Devices, loaded.Devices) } +func TestRegisterMdadmArrayUpdatesExistingArrayHostID(t *testing.T) { + repo := createMdadmTestRepository(t) + ctx := context.Background() + + initial := models.MDADMArray{ + UUID: "ad005595:61305895:4de01d37:669bf162", + Name: "md2", + Level: "raid5", + Devices: []string{"/dev/sde", "/dev/sdd"}, + } + require.NoError(t, repo.RegisterMdadmArray(ctx, initial)) + + updated := models.MDADMArray{ + UUID: initial.UUID, + Name: initial.Name, + Level: initial.Level, + Devices: initial.Devices, + HostID: "nas-01", + } + require.NoError(t, repo.RegisterMdadmArray(ctx, updated)) + + loaded, err := repo.GetMdadmArrayDetails(ctx, initial.UUID) + require.NoError(t, err) + require.Equal(t, updated.HostID, loaded.HostID) +} + func TestGetMdadmArraysExcludesLegacyBlankUUIDRows(t *testing.T) { repo := createMdadmTestRepository(t) ctx := context.Background() diff --git a/webapp/backend/pkg/web/handler/mdadm_get_summary.go b/webapp/backend/pkg/web/handler/mdadm_get_summary.go index b6b6418b1..91f2a95ea 100644 --- a/webapp/backend/pkg/web/handler/mdadm_get_summary.go +++ b/webapp/backend/pkg/web/handler/mdadm_get_summary.go @@ -39,11 +39,15 @@ func GetMdadmSummary(c *gin.Context) { summaries := make([]ArraySummary, 0, len(arrays)) for _, array := range arrays { + devices := array.Devices + if devices == nil { + devices = []string{} + } summary := ArraySummary{ UUID: array.UUID, Name: array.Name, Level: array.Level, - Devices: array.Devices, + Devices: devices, Label: array.Label, Archived: array.Archived, Muted: array.Muted, diff --git a/webapp/backend/pkg/web/handler/mdadm_get_summary_test.go b/webapp/backend/pkg/web/handler/mdadm_get_summary_test.go new file mode 100644 index 000000000..d64b2e12e --- /dev/null +++ b/webapp/backend/pkg/web/handler/mdadm_get_summary_test.go @@ -0,0 +1,50 @@ +package handler + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "testing" + + mock_database "github.com/analogj/scrutiny/webapp/backend/pkg/database/mock" + "github.com/analogj/scrutiny/webapp/backend/pkg/models" + "github.com/gin-gonic/gin" + "github.com/golang/mock/gomock" + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestGetMdadmSummaryNormalizesNullDevices(t *testing.T) { + gin.SetMode(gin.TestMode) + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + repo := mock_database.NewMockDeviceRepo(ctrl) + repo.EXPECT().GetMdadmArrays(gomock.Any()).Return([]models.MDADMArray{{ + UUID: "uuid-1", + Name: "md0", + Level: "raid1", + }}, nil) + repo.EXPECT().GetLatestMdadmMetrics(gomock.Any(), "uuid-1").Return(nil, nil) + + req := httptest.NewRequest(http.MethodGet, "/api/mdadm/summary", nil) + w := httptest.NewRecorder() + c, _ := gin.CreateTestContext(w) + c.Request = req + c.Set("DEVICE_REPOSITORY", repo) + c.Set("LOGGER", logrus.NewEntry(logrus.New())) + + GetMdadmSummary(c) + + require.Equal(t, http.StatusOK, w.Code) + var response struct { + Data []struct { + Devices []string `json:"devices"` + } `json:"data"` + } + require.NoError(t, json.Unmarshal(w.Body.Bytes(), &response)) + require.Len(t, response.Data, 1) + assert.NotNil(t, response.Data[0].Devices) + assert.Empty(t, response.Data[0].Devices) +} diff --git a/webapp/backend/pkg/web/handler/mdadm_register_arrays.go b/webapp/backend/pkg/web/handler/mdadm_register_arrays.go index 4fc6b15bb..4e187d23a 100644 --- a/webapp/backend/pkg/web/handler/mdadm_register_arrays.go +++ b/webapp/backend/pkg/web/handler/mdadm_register_arrays.go @@ -33,12 +33,16 @@ func RegisterMdadmArrays(c *gin.Context) { registrationErrors = append(registrationErrors, fmt.Sprintf("array %s rejected: missing UUID", collectorArray.Name)) continue } + devices := collectorArray.Devices + if devices == nil { + devices = []string{} + } array := models.MDADMArray{ UUID: trimmedUUID, Name: collectorArray.Name, Level: collectorArray.Level, - Devices: collectorArray.Devices, + Devices: devices, HostID: collectorArray.HostID, } diff --git a/webapp/backend/pkg/web/handler/mdadm_register_arrays_test.go b/webapp/backend/pkg/web/handler/mdadm_register_arrays_test.go index 4b9f1781f..bf1ebe2cc 100644 --- a/webapp/backend/pkg/web/handler/mdadm_register_arrays_test.go +++ b/webapp/backend/pkg/web/handler/mdadm_register_arrays_test.go @@ -2,6 +2,7 @@ package handler import ( "bytes" + "context" "encoding/json" "errors" "net/http" @@ -54,6 +55,37 @@ func TestRegisterMdadmArraysReturnsPartialSuccessWithErrorDetails(t *testing.T) assert.Contains(t, response.Errors[0], "uuid-2") } +func TestRegisterMdadmArraysPassesHostIDThroughRegistration(t *testing.T) { + gin.SetMode(gin.TestMode) + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + repo := mock_database.NewMockDeviceRepo(ctrl) + repo.EXPECT().RegisterMdadmArray(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, array models.MDADMArray) error { + assert.Equal(t, "uuid-1", array.UUID) + assert.Equal(t, "host-a", array.HostID) + return nil + }) + + body := `{"data":[{"uuid":"uuid-1","name":"md0","level":"raid1","devices":["/dev/sda"],"host_id":"host-a"}]}` + req := httptest.NewRequest(http.MethodPost, "/api/mdadm/arrays/register", bytes.NewBufferString(body)) + req.Header.Set("Content-Type", "application/json") + w := httptest.NewRecorder() + c, _ := gin.CreateTestContext(w) + c.Request = req + c.Set("DEVICE_REPOSITORY", repo) + c.Set("LOGGER", logrus.NewEntry(logrus.New())) + + RegisterMdadmArrays(c) + + require.Equal(t, http.StatusOK, w.Code) + var response models.MDADMArrayWrapper + require.NoError(t, json.Unmarshal(w.Body.Bytes(), &response)) + assert.True(t, response.Success) + require.Len(t, response.Data, 1) + assert.Equal(t, "host-a", response.Data[0].HostID) +} + func TestRegisterMdadmArraysRejectsMissingUUID(t *testing.T) { gin.SetMode(gin.TestMode) ctrl := gomock.NewController(t) @@ -80,3 +112,34 @@ func TestRegisterMdadmArraysRejectsMissingUUID(t *testing.T) { require.Len(t, response.Errors, 1) assert.Contains(t, response.Errors[0], "missing UUID") } + +func TestRegisterMdadmArraysNormalizesNullDevices(t *testing.T) { + gin.SetMode(gin.TestMode) + ctrl := gomock.NewController(t) + defer ctrl.Finish() + + repo := mock_database.NewMockDeviceRepo(ctrl) + repo.EXPECT().RegisterMdadmArray(gomock.Any(), gomock.Any()).DoAndReturn(func(_ context.Context, array models.MDADMArray) error { + assert.NotNil(t, array.Devices) + assert.Empty(t, array.Devices) + return nil + }) + + body := `{"data":[{"uuid":"uuid-1","name":"md0","level":"raid1","devices":null}]}` + req := httptest.NewRequest(http.MethodPost, "/api/mdadm/arrays/register", bytes.NewBufferString(body)) + req.Header.Set("Content-Type", "application/json") + w := httptest.NewRecorder() + c, _ := gin.CreateTestContext(w) + c.Request = req + c.Set("DEVICE_REPOSITORY", repo) + c.Set("LOGGER", logrus.NewEntry(logrus.New())) + + RegisterMdadmArrays(c) + + require.Equal(t, http.StatusOK, w.Code) + var response models.MDADMArrayWrapper + require.NoError(t, json.Unmarshal(w.Body.Bytes(), &response)) + require.Len(t, response.Data, 1) + assert.NotNil(t, response.Data[0].Devices) + assert.Empty(t, response.Data[0].Devices) +} diff --git a/webapp/frontend/src/app/modules/mdadm/mdadm.service.spec.ts b/webapp/frontend/src/app/modules/mdadm/mdadm.service.spec.ts new file mode 100644 index 000000000..f6f8cb4cd --- /dev/null +++ b/webapp/frontend/src/app/modules/mdadm/mdadm.service.spec.ts @@ -0,0 +1,30 @@ +import { HttpClient } from '@angular/common/http'; +import { TestBed } from '@angular/core/testing'; +import { of } from 'rxjs'; +import { MDADMService } from './mdadm.service'; + +describe('MDADMService', () => { + let service: MDADMService; + let httpClientSpy: jasmine.SpyObj; + + beforeEach(() => { + httpClientSpy = jasmine.createSpyObj('HttpClient', ['get']); + TestBed.configureTestingModule({ + providers: [MDADMService, { provide: HttpClient, useValue: httpClientSpy }], + }); + service = TestBed.inject(MDADMService); + }); + + it('normalizes null member devices in summary data', (done: DoneFn) => { + const response = { + success: true, + data: [{ uuid: 'uuid-1', name: 'md0', level: 'raid1', devices: null }], + } as any; + httpClientSpy.get.and.returnValue(of(response)); + + service.getSummaryData().subscribe((arrays) => { + expect(arrays[0].devices).toEqual([]); + done(); + }); + }); +}); diff --git a/webapp/frontend/src/app/modules/mdadm/mdadm.service.ts b/webapp/frontend/src/app/modules/mdadm/mdadm.service.ts index 1f9fe9e99..27846887c 100644 --- a/webapp/frontend/src/app/modules/mdadm/mdadm.service.ts +++ b/webapp/frontend/src/app/modules/mdadm/mdadm.service.ts @@ -33,7 +33,10 @@ export class MDADMService { getSummaryData(): Observable { return this._httpClient.get(getBasePath() + '/api/mdadm/summary').pipe( map((response: MDADMArrayResponseWrapper) => { - return response.data; + return response.data.map((array) => ({ + ...array, + devices: array.devices ?? [], + })); }), tap((response: MDADMArrayModel[]) => { this._data.next(response);