diff --git a/CHANGELOG.md b/CHANGELOG.md index b4a578c52..435b73b5d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1096,6 +1096,12 @@ All notable changes to Scrutiny will be documented in this file. ### Refactoring * **database:** extract hardcoded time ranges to constants ([deb2df0](https://github.com/Starosdev/scrutiny/commit/deb2df0bc718461c5a9826d6b6c1c1307b7122e8)), closes [#49](https://github.com/Starosdev/scrutiny/issues/49) +## Unreleased + +### Features + +* **smart/ui:** add SMART self-test history endpoint and device detail view + ## [1.1.1](https://github.com/Starosdev/scrutiny/compare/v1.1.0...v1.1.1) (2025-12-09) diff --git a/README.md b/README.md index 8a6b575e1..9bdc3a998 100644 --- a/README.md +++ b/README.md @@ -127,6 +127,7 @@ These S.M.A.R.T hard drive self-tests can help you detect and replace failing ha - **Mobile-Optimized Interface** - Bottom tab bar (Home, Drives, ZFS, Workload, Settings), health overview home tab, card-based data views, and responsive layouts below 960px - **API Timeout Configuration** - Adjust timeouts for slow storage systems - **Performance Benchmarking** - fio-based benchmarks for throughput, IOPS, and latency with historical tracking +- **ATA Self-Test History** - Persist and display recent ATA SMART self-test log entries on device detail pages - **Scheduled Reports** [WIP] - Automated health reports on daily/weekly/monthly schedules with HTML emails and PDF export - **Missed Ping Digest** - Batch notification when multiple collectors go unreachable - **HTML Email Notifications** - Rich HTML formatting with plain-text fallback for SMTP notifications, including reports, test notifications, collector errors, missed ping digests, heartbeat, performance degradation, replacement risk, and MDADM degradation alerts @@ -503,6 +504,16 @@ Some SSDs report misleading wear percentages through SMART. To supplement those This value is user-supplied and does not replace the existing SMART wear indicators. +## ATA SMART Self-Test History + +Scrutiny now retains ATA SMART self-test log entries that are already present in uploaded `smartctl` payloads. On ATA drive detail pages, the web UI shows a **SMART Self-Tests** card with the recorded test type, pass/attention status, controller detail string, and power-on age. + +- Data is read from normal SMART uploads; no separate self-test collector is required +- Only ATA devices show this section today +- Scrutiny keeps the most recent 21 recorded entries per physical ATA device identity +- The API route `GET /api/device/{id}/selftest` returns the same history used by the device detail page +- This feature records and displays history only; it does not trigger or schedule drive self-tests from the web UI + ## SMART Attribute Overrides Scrutiny allows you to customize how individual SMART attributes are evaluated. Use this to suppress false positives, ignore noisy attributes, force specific statuses, or set custom warning/failure thresholds. diff --git a/docs/API.md b/docs/API.md index 8141239ca..9396b071c 100644 --- a/docs/API.md +++ b/docs/API.md @@ -16,7 +16,7 @@ That includes: - authentication and session login - health and diagnostics -- device registration, uploads, details, actions, and performance +- device registration, uploads, details, self-test history, actions, and performance - settings, SMART overrides, and notification URLs - replacement-risk metadata, including consumer ATA profile usage - report generation and report history @@ -41,6 +41,7 @@ See [AUTH.md](./AUTH.md) for configuration and deployment details. - The OpenAPI document is the source of truth. Do not add new standalone API tables elsewhere in the repo. - Some collector payloads are intentionally documented as structured objects with representative fields because the backend accepts large collector-origin JSON models. +- `GET /api/device/{id}/selftest` returns ATA SMART self-test history already recorded during normal SMART uploads. The separate `POST /api/device/{id}/selftest` route remains reserved for future ingestion work. - Notification URL endpoints cover existing Shoutrrr syntax, explicit `apprise+...` targets, `script://` targets, and raw `http(s)` webhooks. - The replacement-risk endpoint includes ATA-specific metadata describing whether a bundled consumer-drive profile was enabled and applied for that score. - For operator-facing behavior and the global opt-out setting, see [CONSUMER_DRIVE_PROFILES.md](./CONSUMER_DRIVE_PROFILES.md). diff --git a/docs/openapi.yaml b/docs/openapi.yaml index 8954fa3ad..c87e46678 100644 --- a/docs/openapi.yaml +++ b/docs/openapi.yaml @@ -321,6 +321,23 @@ paths: "500": $ref: "#/components/responses/ErrorResponse" /api/device/{id}/selftest: + get: + tags: [Devices] + summary: Get ATA SMART self-test history for a device + description: Returns the ATA SMART self-test entries already recorded from normal SMART uploads for the resolved device identity. + parameters: + - $ref: "#/components/parameters/DeviceId" + responses: + "200": + description: ATA SMART self-test history + content: + application/json: + schema: + $ref: "#/components/schemas/DeviceSelfTestsResponse" + "404": + $ref: "#/components/responses/ErrorResponse" + "500": + $ref: "#/components/responses/ErrorResponse" post: tags: [Devices] summary: Upload self-test data for a device @@ -1608,6 +1625,61 @@ components: type: object additionalProperties: true required: [success, data] + DeviceSelfTest: + type: object + properties: + id: + type: integer + created_at: + type: string + format: date-time + updated_at: + type: string + format: date-time + deleted_at: + type: string + format: date-time + nullable: true + device_id: + type: string + device_wwn: + type: string + type_value: + type: integer + type_string: + type: string + status_value: + type: integer + status_string: + type: string + status_passed: + type: boolean + lifetime_hours: + type: integer + required: + - id + - device_id + - device_wwn + - type_value + - type_string + - status_value + - status_string + - status_passed + - lifetime_hours + DeviceSelfTestsResponse: + type: object + properties: + success: + type: boolean + data: + type: object + properties: + self_tests: + type: array + items: + $ref: "#/components/schemas/DeviceSelfTest" + required: [self_tests] + required: [success, data] PerformanceUploadRequest: type: object properties: diff --git a/webapp/backend/pkg/database/interface.go b/webapp/backend/pkg/database/interface.go index de844aa6d..302d3729a 100644 --- a/webapp/backend/pkg/database/interface.go +++ b/webapp/backend/pkg/database/interface.go @@ -55,6 +55,7 @@ type DeviceRepo interface { SaveSmartAttributes(ctx context.Context, wwn string, collectorSmartData collector.SmartInfo) (measurements.Smart, error) GetSmartAttributeHistory(ctx context.Context, wwn string, durationKey string, selectEntries int, selectEntriesOffset int, attributes []string) ([]measurements.Smart, error) + GetDeviceSelfTests(ctx context.Context, deviceID string) ([]models.DeviceSelfTest, error) // GetPreviousSmartSubmission returns the previous raw SMART submission (without daily aggregation) // for use in repeat notification detection. Returns the submission before the most recent one. GetPreviousSmartSubmission(ctx context.Context, wwn string) ([]measurements.Smart, error) diff --git a/webapp/backend/pkg/database/migrations/m20260616000000/device_selftests.go b/webapp/backend/pkg/database/migrations/m20260616000000/device_selftests.go new file mode 100644 index 000000000..39a2d4e17 --- /dev/null +++ b/webapp/backend/pkg/database/migrations/m20260616000000/device_selftests.go @@ -0,0 +1,10 @@ +package m20260616000000 + +import ( + "github.com/analogj/scrutiny/webapp/backend/pkg/models" + "gorm.io/gorm" +) + +func Migrate(db *gorm.DB) error { + return db.AutoMigrate(&models.DeviceSelfTest{}) +} diff --git a/webapp/backend/pkg/database/mock/mock_database.go b/webapp/backend/pkg/database/mock/mock_database.go index 738739a23..d546e43c1 100644 --- a/webapp/backend/pkg/database/mock/mock_database.go +++ b/webapp/backend/pkg/database/mock/mock_database.go @@ -125,20 +125,6 @@ func (mr *MockDeviceRepoMockRecorder) DeleteNotifyUrl(ctx, id interface{}) *gomo return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteNotifyUrl", reflect.TypeOf((*MockDeviceRepo)(nil).DeleteNotifyUrl), ctx, id) } -// UpdateNotifyUrlHeartbeat mocks base method. -func (m *MockDeviceRepo) UpdateNotifyUrlHeartbeat(ctx context.Context, id uint, enabled bool) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "UpdateNotifyUrlHeartbeat", ctx, id, enabled) - ret0, _ := ret[0].(error) - return ret0 -} - -// UpdateNotifyUrlHeartbeat indicates an expected call of UpdateNotifyUrlHeartbeat. -func (mr *MockDeviceRepoMockRecorder) UpdateNotifyUrlHeartbeat(ctx, id, enabled interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateNotifyUrlHeartbeat", reflect.TypeOf((*MockDeviceRepo)(nil).UpdateNotifyUrlHeartbeat), ctx, id, enabled) -} - // DeleteZFSPool mocks base method. func (m *MockDeviceRepo) DeleteZFSPool(ctx context.Context, guid string) error { m.ctrl.T.Helper() @@ -318,6 +304,21 @@ func (mr *MockDeviceRepoMockRecorder) GetDeviceDetails(ctx, deviceID interface{} return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetDeviceDetails", reflect.TypeOf((*MockDeviceRepo)(nil).GetDeviceDetails), ctx, deviceID) } +// GetDeviceSelfTests mocks base method. +func (m *MockDeviceRepo) GetDeviceSelfTests(ctx context.Context, deviceID string) ([]models.DeviceSelfTest, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetDeviceSelfTests", ctx, deviceID) + ret0, _ := ret[0].([]models.DeviceSelfTest) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetDeviceSelfTests indicates an expected call of GetDeviceSelfTests. +func (mr *MockDeviceRepoMockRecorder) GetDeviceSelfTests(ctx, deviceID interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetDeviceSelfTests", reflect.TypeOf((*MockDeviceRepo)(nil).GetDeviceSelfTests), ctx, deviceID) +} + // GetDevices mocks base method. func (m *MockDeviceRepo) GetDevices(ctx context.Context) ([]models.Device, error) { m.ctrl.T.Helper() @@ -1159,6 +1160,20 @@ func (mr *MockDeviceRepoMockRecorder) UpdateMdadmArrayMuted(ctx, uuid, muted int return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateMdadmArrayMuted", reflect.TypeOf((*MockDeviceRepo)(nil).UpdateMdadmArrayMuted), ctx, uuid, muted) } +// UpdateNotifyUrlHeartbeat mocks base method. +func (m *MockDeviceRepo) UpdateNotifyUrlHeartbeat(ctx context.Context, id uint, enabled bool) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UpdateNotifyUrlHeartbeat", ctx, id, enabled) + ret0, _ := ret[0].(error) + return ret0 +} + +// UpdateNotifyUrlHeartbeat indicates an expected call of UpdateNotifyUrlHeartbeat. +func (mr *MockDeviceRepoMockRecorder) UpdateNotifyUrlHeartbeat(ctx, id, enabled interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateNotifyUrlHeartbeat", reflect.TypeOf((*MockDeviceRepo)(nil).UpdateNotifyUrlHeartbeat), ctx, id, enabled) +} + // UpdateZFSPoolArchived mocks base method. func (m *MockDeviceRepo) UpdateZFSPoolArchived(ctx context.Context, guid string, archived bool) error { m.ctrl.T.Helper() diff --git a/webapp/backend/pkg/database/scrutiny_repository_device_self_tests.go b/webapp/backend/pkg/database/scrutiny_repository_device_self_tests.go new file mode 100644 index 000000000..7e218f8f0 --- /dev/null +++ b/webapp/backend/pkg/database/scrutiny_repository_device_self_tests.go @@ -0,0 +1,103 @@ +package database + +import ( + "context" + "fmt" + "strings" + + "github.com/analogj/scrutiny/webapp/backend/pkg" + "github.com/analogj/scrutiny/webapp/backend/pkg/models" + "github.com/analogj/scrutiny/webapp/backend/pkg/models/collector" + "gorm.io/gorm" + "gorm.io/gorm/clause" +) + +const maxDeviceSelfTestsPerDevice = 21 + +func deviceSelfTestIdentity(device *models.Device) string { + if strings.TrimSpace(device.WWN) != "" { + return strings.TrimSpace(device.WWN) + } + return device.DeviceID +} + +func (sr *scrutinyRepository) syncDeviceSelfTests(ctx context.Context, device *models.Device, collectorSmartData *collector.SmartInfo) error { + if collectorSmartData.Device.Protocol != pkg.DeviceProtocolAta { + return nil + } + + selfTestEntries := collectorSmartData.AtaSmartSelfTestLog.Entries() + if len(selfTestEntries) == 0 { + return nil + } + + deviceIdentity := deviceSelfTestIdentity(device) + + return sr.gormClient.WithContext(ctx).Transaction(func(tx *gorm.DB) error { + for _, selfTestEntry := range selfTestEntries { + row := models.DeviceSelfTest{ + DeviceID: device.DeviceID, + DeviceWWN: device.WWN, + DeviceIdentity: deviceIdentity, + TypeValue: selfTestEntry.Type.Value, + TypeString: selfTestEntry.Type.String, + StatusValue: selfTestEntry.Status.Value, + StatusString: selfTestEntry.Status.String, + StatusPassed: selfTestEntry.Status.Passed, + LifetimeHours: selfTestEntry.LifetimeHours, + } + + if err := tx.Clauses(clause.OnConflict{ + Columns: []clause.Column{ + {Name: "device_identity"}, + {Name: "type_value"}, + {Name: "lifetime_hours"}, + }, + DoUpdates: clause.AssignmentColumns([]string{ + "device_id", + "device_wwn", + "type_string", + "status_value", + "status_string", + "status_passed", + "updated_at", + }), + }).Create(&row).Error; err != nil { + return err + } + } + + var staleIDs []uint + if err := tx.Model(&models.DeviceSelfTest{}). + Where("device_identity = ?", deviceIdentity). + Order("lifetime_hours DESC, updated_at DESC, id DESC"). + Offset(maxDeviceSelfTestsPerDevice). + Pluck("id", &staleIDs).Error; err != nil { + return err + } + + if len(staleIDs) == 0 { + return nil + } + + return tx.Delete(&models.DeviceSelfTest{}, staleIDs).Error + }) +} + +func (sr *scrutinyRepository) GetDeviceSelfTests(ctx context.Context, deviceID string) ([]models.DeviceSelfTest, error) { + device, err := sr.GetDeviceDetails(ctx, deviceID) + if err != nil { + return nil, err + } + + deviceIdentity := deviceSelfTestIdentity(&device) + selfTests := []models.DeviceSelfTest{} + if err := sr.gormClient.WithContext(ctx). + Where("device_identity = ?", deviceIdentity). + Order("lifetime_hours DESC, id DESC"). + Find(&selfTests).Error; err != nil { + return nil, fmt.Errorf("could not get device self-tests from DB: %v", err) + } + + return selfTests, nil +} diff --git a/webapp/backend/pkg/database/scrutiny_repository_device_self_tests_test.go b/webapp/backend/pkg/database/scrutiny_repository_device_self_tests_test.go new file mode 100644 index 000000000..24d0a4000 --- /dev/null +++ b/webapp/backend/pkg/database/scrutiny_repository_device_self_tests_test.go @@ -0,0 +1,212 @@ +package database + +import ( + "context" + "encoding/json" + "errors" + "os" + "path/filepath" + "testing" + + mock_config "github.com/analogj/scrutiny/webapp/backend/pkg/config/mock" + "github.com/analogj/scrutiny/webapp/backend/pkg/models" + "github.com/analogj/scrutiny/webapp/backend/pkg/models/collector" + "github.com/glebarez/sqlite" + "github.com/golang/mock/gomock" + influxapi "github.com/influxdata/influxdb-client-go/v2/api" + "github.com/influxdata/influxdb-client-go/v2/api/write" + "github.com/influxdata/influxdb-client-go/v2/domain" + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/require" + "gorm.io/gorm" +) + +type stubWriteAPI struct{} + +func (s *stubWriteAPI) WriteRecord(ctx context.Context, line ...string) error { return nil } +func (s *stubWriteAPI) WritePoint(ctx context.Context, point ...*write.Point) error { + return nil +} +func (s *stubWriteAPI) EnableBatching() {} +func (s *stubWriteAPI) Flush(ctx context.Context) error { return nil } + +var _ influxapi.WriteAPIBlocking = (*stubWriteAPI)(nil) + +type stubQueryAPI struct{} + +func (s *stubQueryAPI) QueryRaw(ctx context.Context, query string, dialect *domain.Dialect) (string, error) { + return "", errors.New("not implemented in tests") +} + +func (s *stubQueryAPI) QueryRawWithParams(ctx context.Context, query string, dialect *domain.Dialect, params interface{}) (string, error) { + return "", errors.New("not implemented in tests") +} + +func (s *stubQueryAPI) Query(ctx context.Context, query string) (*influxapi.QueryTableResult, error) { + return nil, errors.New("not implemented in tests") +} + +func (s *stubQueryAPI) QueryWithParams(ctx context.Context, query string, params interface{}) (*influxapi.QueryTableResult, error) { + return nil, errors.New("not implemented in tests") +} + +var _ influxapi.QueryAPI = (*stubQueryAPI)(nil) + +func createDeviceSelfTestRepository(t *testing.T) *scrutinyRepository { + t.Helper() + + mockCtrl := gomock.NewController(t) + t.Cleanup(func() { mockCtrl.Finish() }) + + fakeConfig := mock_config.NewMockInterface(mockCtrl) + fakeConfig.EXPECT().Get(gomock.Any()).Return(nil).AnyTimes() + fakeConfig.EXPECT().GetBool(gomock.Any()).Return(false).AnyTimes() + fakeConfig.EXPECT().GetInt(gomock.Any()).Return(0).AnyTimes() + fakeConfig.EXPECT().GetIntSlice(gomock.Any()).Return(nil).AnyTimes() + fakeConfig.EXPECT().GetString(gomock.Any()).Return("metrics").AnyTimes() + fakeConfig.EXPECT().GetStringSlice(gomock.Any()).Return(nil).AnyTimes() + fakeConfig.EXPECT().IsSet(gomock.Any()).Return(false).AnyTimes() + + db, err := gorm.Open(sqlite.Open("file:"+t.Name()+"?mode=memory&cache=shared"), &gorm.Config{}) + require.NoError(t, err) + require.NoError(t, db.AutoMigrate(&models.Device{}, &models.DeviceSelfTest{}, &models.AttributeOverride{})) + + return &scrutinyRepository{ + appConfig: fakeConfig, + gormClient: db, + logger: logrus.New(), + influxWriteApi: &stubWriteAPI{}, + influxQueryApi: &stubQueryAPI{}, + } +} + +func loadSmartInfoFixture(t *testing.T, fixturePath string) collector.SmartInfo { + t.Helper() + + payload, err := os.ReadFile(fixturePath) + require.NoError(t, err) + + var smartInfo collector.SmartInfo + require.NoError(t, json.Unmarshal(payload, &smartInfo)) + return smartInfo +} + +func TestSaveSmartAttributesPersistsAtaSelfTests(t *testing.T) { + repo := createDeviceSelfTestRepository(t) + ctx := context.Background() + + device := models.Device{ + DeviceID: "device-1", + WWN: "wwn-1", + DeviceProtocol: "ATA", + } + require.NoError(t, repo.gormClient.WithContext(ctx).Create(&device).Error) + + smartInfo := loadSmartInfoFixture(t, filepath.Join("..", "web", "testdata", "upload-device-metrics-req.json")) + + _, err := repo.SaveSmartAttributes(ctx, device.WWN, smartInfo) + require.NoError(t, err) + + var selfTests []models.DeviceSelfTest + require.NoError(t, repo.gormClient.WithContext(ctx). + Order("lifetime_hours DESC, id DESC"). + Find(&selfTests).Error) + + require.Len(t, selfTests, 21) + require.Equal(t, device.DeviceID, selfTests[0].DeviceID) + require.Equal(t, device.WWN, selfTests[0].DeviceWWN) + require.Equal(t, 1708, selfTests[0].LifetimeHours) + require.Equal(t, "Short offline", selfTests[0].TypeString) + require.Equal(t, "Completed without error", selfTests[0].StatusString) + require.True(t, selfTests[0].StatusPassed) + require.Equal(t, 1157, selfTests[len(selfTests)-1].LifetimeHours) +} + +func TestSyncDeviceSelfTestsDedupesByDeviceIdentity(t *testing.T) { + repo := createDeviceSelfTestRepository(t) + ctx := context.Background() + + initialDevice := models.Device{ + DeviceID: "device-1", + WWN: "wwn-1", + DeviceProtocol: "ATA", + } + replacementDevice := models.Device{ + DeviceID: "device-2", + WWN: "wwn-1", + DeviceProtocol: "ATA", + } + + initialPayload := collector.SmartInfo{} + initialPayload.Device.Protocol = "ATA" + initialPayload.AtaSmartSelfTestLog.Standard.Table = []collector.AtaSmartSelfTestLogEntry{ + { + LifetimeHours: 100, + }, + } + initialPayload.AtaSmartSelfTestLog.Standard.Table[0].Type.Value = 1 + initialPayload.AtaSmartSelfTestLog.Standard.Table[0].Type.String = "Short offline" + initialPayload.AtaSmartSelfTestLog.Standard.Table[0].Status.Value = 1 + initialPayload.AtaSmartSelfTestLog.Standard.Table[0].Status.String = "Aborted by host" + initialPayload.AtaSmartSelfTestLog.Standard.Table[0].Status.Passed = false + + updatedPayload := collector.SmartInfo{} + updatedPayload.Device.Protocol = "ATA" + updatedPayload.AtaSmartSelfTestLog.Standard.Table = []collector.AtaSmartSelfTestLogEntry{ + { + LifetimeHours: 100, + }, + } + updatedPayload.AtaSmartSelfTestLog.Standard.Table[0].Type.Value = 1 + updatedPayload.AtaSmartSelfTestLog.Standard.Table[0].Type.String = "Short offline" + updatedPayload.AtaSmartSelfTestLog.Standard.Table[0].Status.Value = 0 + updatedPayload.AtaSmartSelfTestLog.Standard.Table[0].Status.String = "Completed without error" + updatedPayload.AtaSmartSelfTestLog.Standard.Table[0].Status.Passed = true + + require.NoError(t, repo.syncDeviceSelfTests(ctx, &initialDevice, &initialPayload)) + require.NoError(t, repo.syncDeviceSelfTests(ctx, &replacementDevice, &updatedPayload)) + + var selfTests []models.DeviceSelfTest + require.NoError(t, repo.gormClient.WithContext(ctx).Find(&selfTests).Error) + require.Len(t, selfTests, 1) + require.Equal(t, "device-2", selfTests[0].DeviceID) + require.Equal(t, "wwn-1", selfTests[0].DeviceWWN) + require.Equal(t, "Completed without error", selfTests[0].StatusString) + require.True(t, selfTests[0].StatusPassed) +} + +func TestSyncDeviceSelfTestsPrunesOldEntries(t *testing.T) { + repo := createDeviceSelfTestRepository(t) + ctx := context.Background() + + device := models.Device{ + DeviceID: "device-1", + WWN: "wwn-1", + DeviceProtocol: "ATA", + } + + payload := collector.SmartInfo{} + payload.Device.Protocol = "ATA" + for lifetime := 1; lifetime <= 25; lifetime++ { + entry := collector.AtaSmartSelfTestLogEntry{ + LifetimeHours: lifetime, + } + entry.Type.Value = 1 + entry.Type.String = "Short offline" + entry.Status.Value = 0 + entry.Status.String = "Completed without error" + entry.Status.Passed = true + payload.AtaSmartSelfTestLog.Standard.Table = append(payload.AtaSmartSelfTestLog.Standard.Table, entry) + } + + require.NoError(t, repo.syncDeviceSelfTests(ctx, &device, &payload)) + + var selfTests []models.DeviceSelfTest + require.NoError(t, repo.gormClient.WithContext(ctx). + Order("lifetime_hours DESC"). + Find(&selfTests).Error) + + require.Len(t, selfTests, 21) + require.Equal(t, 25, selfTests[0].LifetimeHours) + require.Equal(t, 5, selfTests[len(selfTests)-1].LifetimeHours) +} diff --git a/webapp/backend/pkg/database/scrutiny_repository_device_smart_attributes.go b/webapp/backend/pkg/database/scrutiny_repository_device_smart_attributes.go index a46f0ef39..2552259f2 100644 --- a/webapp/backend/pkg/database/scrutiny_repository_device_smart_attributes.go +++ b/webapp/backend/pkg/database/scrutiny_repository_device_smart_attributes.go @@ -27,8 +27,9 @@ func (sr *scrutinyRepository) SaveSmartAttributes(ctx context.Context, wwn strin return measurements.Smart{}, err } - // Look up DeviceID for dual-tagging in InfluxDB - device, devErr := sr.GetDeviceDetails(ctx, wwn) + // Look up the device via WWN so the current upload can carry a stable device_id tag + // and self-test history can be attached to the correct SQLite row set. + device, devErr := sr.GetDeviceByWWN(ctx, wwn) if devErr == nil { deviceSmartData.DeviceID = device.DeviceID } @@ -48,6 +49,12 @@ func (sr *scrutinyRepository) SaveSmartAttributes(ctx context.Context, wwn strin tags, fields := deviceSmartData.Flatten() + if devErr == nil { + if err := sr.syncDeviceSelfTests(ctx, &device, &collectorSmartData); err != nil { + return measurements.Smart{}, err + } + } + // write point immediately return deviceSmartData, sr.saveDatapoint(sr.influxWriteApi, "smart", tags, fields, deviceSmartData.Date, ctx) } diff --git a/webapp/backend/pkg/database/scrutiny_repository_migrations.go b/webapp/backend/pkg/database/scrutiny_repository_migrations.go index 7c54b419f..ff5c58b7b 100644 --- a/webapp/backend/pkg/database/scrutiny_repository_migrations.go +++ b/webapp/backend/pkg/database/scrutiny_repository_migrations.go @@ -33,6 +33,7 @@ import ( "github.com/analogj/scrutiny/webapp/backend/pkg/database/migrations/m20260523000000" m20260608000000 "github.com/analogj/scrutiny/webapp/backend/pkg/database/migrations/m20260608000000" m20260610000000 "github.com/analogj/scrutiny/webapp/backend/pkg/database/migrations/m20260610000000" + m20260616000000 "github.com/analogj/scrutiny/webapp/backend/pkg/database/migrations/m20260616000000" "github.com/analogj/scrutiny/webapp/backend/pkg/deviceid" "github.com/analogj/scrutiny/webapp/backend/pkg/models" "github.com/analogj/scrutiny/webapp/backend/pkg/models/collector" @@ -1158,9 +1159,13 @@ missed_ping_timeout_override INTEGER DEFAULT 0 }, }, { - ID: "m20260610000000", // add host_id column to mdadm_arrays table (#579) + ID: "m20260610000000", // add host_id column to mdadm_arrays table (#579) Migrate: m20260610000000.Migrate, }, + { + ID: "m20260616000000", // add ATA SMART self-test history table (#491) + Migrate: m20260616000000.Migrate, + }, }) if err := m.Migrate(); err != nil { diff --git a/webapp/backend/pkg/database/scrutiny_repository_migrations_test.go b/webapp/backend/pkg/database/scrutiny_repository_migrations_test.go index 224a2a7f9..c717898d5 100644 --- a/webapp/backend/pkg/database/scrutiny_repository_migrations_test.go +++ b/webapp/backend/pkg/database/scrutiny_repository_migrations_test.go @@ -510,3 +510,10 @@ CREATE TABLE devices ( require.NoError(t, repo.gormClient.Raw(`SELECT device_id FROM devices LIMIT 1`).Scan(&remainingID).Error) require.Equal(t, "valid-uuid", remainingID, "valid device should be preserved") } + +func TestMigrateCreatesDeviceSelfTestsTable(t *testing.T) { + repo := createMigrationTestRepository(t) + + require.NoError(t, repo.Migrate(context.Background())) + require.True(t, repo.gormClient.Migrator().HasTable(&models.DeviceSelfTest{})) +} diff --git a/webapp/backend/pkg/models/collector/smart.go b/webapp/backend/pkg/models/collector/smart.go index 90cc7b2ca..8eee4fd68 100644 --- a/webapp/backend/pkg/models/collector/smart.go +++ b/webapp/backend/pkg/models/collector/smart.go @@ -2,6 +2,40 @@ package collector import "github.com/analogj/scrutiny/webapp/backend/pkg/models/common" +type AtaSmartSelfTestLogEntry struct { + Type struct { + String string `json:"string"` + Value int `json:"value"` + } `json:"type"` + Status struct { + String string `json:"string"` + Value int `json:"value"` + Passed bool `json:"passed"` + } `json:"status"` + LifetimeHours int `json:"lifetime_hours"` +} + +type AtaSmartSelfTestLogTable struct { + Table []AtaSmartSelfTestLogEntry `json:"table"` + Revision int `json:"revision"` + Sectors int `json:"sectors,omitempty"` + Count int `json:"count"` + ErrorCountTotal int `json:"error_count_total"` + ErrorCountOutdated int `json:"error_count_outdated"` +} + +type AtaSmartSelfTestLog struct { + Standard AtaSmartSelfTestLogTable `json:"standard"` + Extended AtaSmartSelfTestLogTable `json:"extended"` +} + +func (l *AtaSmartSelfTestLog) Entries() []AtaSmartSelfTestLogEntry { + if len(l.Standard.Table) > 0 { + return l.Standard.Table + } + return l.Extended.Table +} + type SmartInfo struct { JSONFormatVersion []int `json:"json_format_version"` Smartctl struct { @@ -169,26 +203,7 @@ type SmartInfo struct { } `json:"table"` } `json:"summary"` } `json:"ata_smart_error_log"` - AtaSmartSelfTestLog struct { - Standard struct { - Revision int `json:"revision"` - Table []struct { - Type struct { - Value int `json:"value"` - String string `json:"string"` - } `json:"type"` - Status struct { - Value int `json:"value"` - String string `json:"string"` - Passed bool `json:"passed"` - } `json:"status"` - LifetimeHours int `json:"lifetime_hours"` - } `json:"table"` - Count int `json:"count"` - ErrorCountTotal int `json:"error_count_total"` - ErrorCountOutdated int `json:"error_count_outdated"` - } `json:"standard"` - } `json:"ata_smart_self_test_log"` + AtaSmartSelfTestLog AtaSmartSelfTestLog `json:"ata_smart_self_test_log"` AtaSmartSelectiveSelfTestLog struct { Revision int `json:"revision"` Table []struct { diff --git a/webapp/backend/pkg/models/collector/smart_test.go b/webapp/backend/pkg/models/collector/smart_test.go index a3139a3bf..1a6db181b 100644 --- a/webapp/backend/pkg/models/collector/smart_test.go +++ b/webapp/backend/pkg/models/collector/smart_test.go @@ -122,3 +122,53 @@ func TestSmartInfo_LargeLBAValues(t *testing.T) { assert.Equal(t, uint64(18446744073709551615), smartInfo.AtaSmartErrorLog.Summary.Table[0].PreviousCommands[0].Registers.Lba) }) } + +func TestSmartInfo_AtaSmartSelfTestLogEntries(t *testing.T) { + t.Run("should prefer standard self-test entries when present", func(t *testing.T) { + jsonData := `{ + "ata_smart_self_test_log": { + "standard": { + "revision": 1, + "table": [ + { + "type": { "value": 1, "string": "Short offline" }, + "status": { "value": 0, "string": "Completed without error", "passed": true }, + "lifetime_hours": 100 + } + ] + } + } + }` + + var smartInfo SmartInfo + err := json.Unmarshal([]byte(jsonData), &smartInfo) + require.NoError(t, err) + require.Len(t, smartInfo.AtaSmartSelfTestLog.Entries(), 1) + assert.Equal(t, 100, smartInfo.AtaSmartSelfTestLog.Entries()[0].LifetimeHours) + }) + + t.Run("should fall back to extended self-test entries", func(t *testing.T) { + jsonData := `{ + "ata_smart_self_test_log": { + "extended": { + "revision": 1, + "sectors": 1, + "table": [ + { + "type": { "value": 2, "string": "Extended offline" }, + "status": { "value": 0, "string": "Completed without error", "passed": true }, + "lifetime_hours": 200 + } + ] + } + } + }` + + var smartInfo SmartInfo + err := json.Unmarshal([]byte(jsonData), &smartInfo) + require.NoError(t, err) + require.Len(t, smartInfo.AtaSmartSelfTestLog.Entries(), 1) + assert.Equal(t, 2, smartInfo.AtaSmartSelfTestLog.Entries()[0].Type.Value) + assert.Equal(t, 200, smartInfo.AtaSmartSelfTestLog.Entries()[0].LifetimeHours) + }) +} diff --git a/webapp/backend/pkg/models/device_selftests.go b/webapp/backend/pkg/models/device_selftests.go new file mode 100644 index 000000000..853bbb419 --- /dev/null +++ b/webapp/backend/pkg/models/device_selftests.go @@ -0,0 +1,19 @@ +package models + +import "time" + +type DeviceSelfTest struct { + CreatedAt time.Time `json:"created_at"` + UpdatedAt time.Time `json:"updated_at"` + DeletedAt *time.Time `json:"deleted_at,omitempty"` + DeviceIdentity string `json:"-" gorm:"index:idx_device_self_tests_identity,unique;index:idx_device_self_tests_history"` + DeviceID string `json:"device_id" gorm:"index"` + DeviceWWN string `json:"device_wwn" gorm:"index"` + TypeString string `json:"type_string"` + StatusString string `json:"status_string"` + ID uint `json:"id" gorm:"primaryKey"` + TypeValue int `json:"type_value" gorm:"index:idx_device_self_tests_identity,unique"` + StatusValue int `json:"status_value"` + LifetimeHours int `json:"lifetime_hours" gorm:"index:idx_device_self_tests_identity,unique;index:idx_device_self_tests_history"` + StatusPassed bool `json:"status_passed"` +} diff --git a/webapp/backend/pkg/web/handler/get_device_self_tests.go b/webapp/backend/pkg/web/handler/get_device_self_tests.go new file mode 100644 index 000000000..9a29ee476 --- /dev/null +++ b/webapp/backend/pkg/web/handler/get_device_self_tests.go @@ -0,0 +1,34 @@ +package handler + +import ( + "net/http" + + "github.com/analogj/scrutiny/webapp/backend/pkg/database" + "github.com/gin-gonic/gin" + "github.com/sirupsen/logrus" +) + +// GetDeviceSelfTests retrieves ATA SMART self-test history for a device. +func GetDeviceSelfTests(c *gin.Context) { + deviceRepo := c.MustGet("DEVICE_REPOSITORY").(database.DeviceRepo) + logger := c.MustGet("LOGGER").(*logrus.Entry) + + device, err := ResolveDevice(c, logger, deviceRepo) + if err != nil { + return + } + + selfTests, err := deviceRepo.GetDeviceSelfTests(c, device.DeviceID) + if err != nil { + logger.Errorln("An error occurred while retrieving device self-test history", err) + c.JSON(http.StatusInternalServerError, gin.H{"success": false}) + return + } + + c.JSON(http.StatusOK, gin.H{ + "success": true, + "data": gin.H{ + "self_tests": selfTests, + }, + }) +} diff --git a/webapp/backend/pkg/web/handler/get_device_self_tests_test.go b/webapp/backend/pkg/web/handler/get_device_self_tests_test.go new file mode 100644 index 000000000..4d45935d8 --- /dev/null +++ b/webapp/backend/pkg/web/handler/get_device_self_tests_test.go @@ -0,0 +1,98 @@ +package handler_test + +import ( + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "testing" + + mock_database "github.com/analogj/scrutiny/webapp/backend/pkg/database/mock" + "github.com/analogj/scrutiny/webapp/backend/pkg/models" + "github.com/analogj/scrutiny/webapp/backend/pkg/web/handler" + "github.com/gin-gonic/gin" + "github.com/golang/mock/gomock" + "github.com/sirupsen/logrus" + "github.com/stretchr/testify/require" +) + +func TestGetDeviceSelfTests(t *testing.T) { + gin.SetMode(gin.TestMode) + + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + fakeRepo := mock_database.NewMockDeviceRepo(mockCtrl) + device := models.Device{DeviceID: "device-1", WWN: testDeviceWWN, DeviceName: "/dev/sda"} + selfTests := []models.DeviceSelfTest{ + { + DeviceID: "device-1", + DeviceWWN: testDeviceWWN, + TypeValue: 1, + TypeString: "Short offline", + StatusValue: 0, + StatusString: "Completed without error", + StatusPassed: true, + LifetimeHours: 1708, + }, + } + + fakeRepo.EXPECT().GetDeviceDetails(gomock.Any(), "device-1").Return(device, nil) + fakeRepo.EXPECT().GetDeviceSelfTests(gomock.Any(), "device-1").Return(selfTests, nil) + + logger := logrus.WithField("test", t.Name()) + r := gin.New() + r.Use(func(c *gin.Context) { + c.Set("DEVICE_REPOSITORY", fakeRepo) + c.Set("LOGGER", logger) + c.Next() + }) + r.GET("/api/device/:id/selftest", handler.GetDeviceSelfTests) + + w := httptest.NewRecorder() + req, _ := http.NewRequest(http.MethodGet, "/api/device/device-1/selftest", nil) + r.ServeHTTP(w, req) + + require.Equal(t, http.StatusOK, w.Code) + + var response struct { + Success bool `json:"success"` + Data struct { + SelfTests []models.DeviceSelfTest `json:"self_tests"` + } `json:"data"` + } + require.NoError(t, json.Unmarshal(w.Body.Bytes(), &response)) + require.True(t, response.Success) + require.Len(t, response.Data.SelfTests, 1) + require.Equal(t, 1708, response.Data.SelfTests[0].LifetimeHours) + require.Equal(t, "Short offline", response.Data.SelfTests[0].TypeString) +} + +func TestGetDeviceSelfTestsReturnsServerErrorOnRepositoryFailure(t *testing.T) { + gin.SetMode(gin.TestMode) + + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + + fakeRepo := mock_database.NewMockDeviceRepo(mockCtrl) + device := models.Device{DeviceID: "device-1", WWN: testDeviceWWN, DeviceName: "/dev/sda"} + + fakeRepo.EXPECT().GetDeviceDetails(gomock.Any(), "device-1").Return(device, nil) + fakeRepo.EXPECT().GetDeviceSelfTests(gomock.Any(), "device-1").Return(nil, fmt.Errorf("db failed")) + + logger := logrus.WithField("test", t.Name()) + r := gin.New() + r.Use(func(c *gin.Context) { + c.Set("DEVICE_REPOSITORY", fakeRepo) + c.Set("LOGGER", logger) + c.Next() + }) + r.GET("/api/device/:id/selftest", handler.GetDeviceSelfTests) + + w := httptest.NewRecorder() + req, _ := http.NewRequest(http.MethodGet, "/api/device/device-1/selftest", nil) + r.ServeHTTP(w, req) + + require.Equal(t, http.StatusInternalServerError, w.Code) + require.Contains(t, w.Body.String(), `"success":false`) +} diff --git a/webapp/backend/pkg/web/server.go b/webapp/backend/pkg/web/server.go index e2a3ece83..0feec42e4 100644 --- a/webapp/backend/pkg/web/server.go +++ b/webapp/backend/pkg/web/server.go @@ -137,6 +137,7 @@ func (ae *AppEngine) Setup(logger *logrus.Entry) *gin.Engine { api.POST("/device/:id/smart", handler.UploadDeviceMetrics) // used by Collector to upload data api.POST("/device/:id/selftest", handler.UploadDeviceSelfTests) + api.GET("/device/:id/selftest", handler.GetDeviceSelfTests) // used by UI to view self-test history api.GET("/device/:id/details", handler.GetDeviceDetails) // used by Details api.POST("/device/:id/archive", handler.ArchiveDevice) // used by UI to archive device api.POST("/device/:id/unarchive", handler.UnarchiveDevice) // used by UI to unarchive device diff --git a/webapp/frontend/src/app/core/models/device-selftest-model.ts b/webapp/frontend/src/app/core/models/device-selftest-model.ts new file mode 100644 index 000000000..dc936c142 --- /dev/null +++ b/webapp/frontend/src/app/core/models/device-selftest-model.ts @@ -0,0 +1,13 @@ +export interface DeviceSelfTestModel { + id: number; + created_at: string; + updated_at: string; + device_id: string; + device_wwn: string; + type_value: number; + type_string: string; + status_value: number; + status_string: string; + status_passed: boolean; + lifetime_hours: number; +} diff --git a/webapp/frontend/src/app/core/models/device-selftests-response-wrapper.ts b/webapp/frontend/src/app/core/models/device-selftests-response-wrapper.ts new file mode 100644 index 000000000..a67ae37cb --- /dev/null +++ b/webapp/frontend/src/app/core/models/device-selftests-response-wrapper.ts @@ -0,0 +1,9 @@ +import { DeviceSelfTestModel } from 'app/core/models/device-selftest-model'; + +export interface DeviceSelfTestsResponseWrapper { + success: boolean; + errors?: any[]; + data: { + self_tests: DeviceSelfTestModel[]; + }; +} diff --git a/webapp/frontend/src/app/modules/detail/detail.component.html b/webapp/frontend/src/app/modules/detail/detail.component.html index cbe94e6be..dbc3065f1 100644 --- a/webapp/frontend/src/app/modules/detail/detail.component.html +++ b/webapp/frontend/src/app/modules/detail/detail.component.html @@ -313,6 +313,85 @@
| Type | +Status | +Detail | +Power-On Age | +
|---|---|---|---|
| {{ selfTest.type_string }} | ++ + + {{ selfTestStatusLabel(selfTest) }} + + | +{{ selfTest.status_string }} | ++ {{ selfTest.lifetime_hours | deviceHours : config.powered_on_hours_unit : { round: true, largest: 1, units: ['y', 'd', 'h'] } }} + | +