diff --git a/collector/pkg/config/interface.go b/collector/pkg/config/interface.go index 68bf9bc3..717d7628 100644 --- a/collector/pkg/config/interface.go +++ b/collector/pkg/config/interface.go @@ -20,6 +20,7 @@ type Interface interface { GetInt(key string) int GetString(key string) string GetStringSlice(key string) []string + GetIntSlice(key string) []int UnmarshalKey(key string, rawVal interface{}, decoderOpts ...viper.DecoderConfigOption) error GetDeviceOverrides() []models.ScanOverride diff --git a/collector/pkg/config/mock/mock_config.go b/collector/pkg/config/mock/mock_config.go index 98a19bb6..b060eb9f 100644 --- a/collector/pkg/config/mock/mock_config.go +++ b/collector/pkg/config/mock/mock_config.go @@ -133,6 +133,20 @@ func (mr *MockInterfaceMockRecorder) GetInt(key interface{}) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetInt", reflect.TypeOf((*MockInterface)(nil).GetInt), key) } +// GetIntSlice mocks base method. +func (m *MockInterface) GetIntSlice(key string) []int { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetIntSlice", key) + ret0, _ := ret[0].([]int) + return ret0 +} + +// GetIntSlice indicates an expected call of GetIntSlice. +func (mr *MockInterfaceMockRecorder) GetIntSlice(key interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetIntSlice", reflect.TypeOf((*MockInterface)(nil).GetIntSlice), key) +} + // GetString mocks base method. func (m *MockInterface) GetString(key string) string { m.ctrl.T.Helper() diff --git a/example.scrutiny.yaml b/example.scrutiny.yaml index 039bcbf0..311d6acb 100644 --- a/example.scrutiny.yaml +++ b/example.scrutiny.yaml @@ -47,6 +47,11 @@ web: # org: 'my-org' # bucket: 'bucket' retention_policy: true + + # Prometheus metrics endpoint configuration + metrics: + # Enable or disable Prometheus metrics endpoint (/api/metrics) + enabled: true # if you wish to disable TLS certificate verification, # when using self-signed certificates for example, # then uncomment the lines below and set `insecure_skip_verify: true` @@ -57,6 +62,10 @@ log: file: '' #absolute or relative paths allowed, eg. web.log level: INFO +failures: + transient: + ata: + - 195 # Hardware_ECC_Recovered, see https://superuser.com/a/1511916/169872 # Notification "urls" look like the following. For more information about service specific configuration see # Shoutrrr's documentation: https://containrrr.dev/shoutrrr/services/overview/ diff --git a/go.mod b/go.mod index bc60bf6d..e1a234da 100644 --- a/go.mod +++ b/go.mod @@ -13,17 +13,20 @@ require ( github.com/influxdata/influxdb-client-go/v2 v2.9.0 github.com/jaypipes/ghw v0.6.1 github.com/mitchellh/mapstructure v1.5.0 + github.com/prometheus/client_golang v1.17.0 github.com/samber/lo v1.25.0 github.com/sirupsen/logrus v1.6.0 github.com/spf13/viper v1.15.0 github.com/stretchr/testify v1.8.1 github.com/urfave/cli/v2 v2.2.0 - golang.org/x/sync v0.1.0 + golang.org/x/sync v0.3.0 gorm.io/gorm v1.23.5 ) require ( github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/deepmap/oapi-codegen v1.8.2 // indirect @@ -49,12 +52,16 @@ require ( github.com/magiconair/properties v1.8.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.18 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/mitchellh/go-homedir v1.1.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect github.com/pelletier/go-toml/v2 v2.0.6 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 // indirect + github.com/prometheus/common v0.44.0 // indirect + github.com/prometheus/procfs v0.11.1 // indirect github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 // indirect github.com/russross/blackfriday/v2 v2.1.0 // indirect github.com/spf13/afero v1.9.3 // indirect @@ -65,11 +72,11 @@ require ( github.com/ugorji/go/codec v1.1.7 // indirect golang.org/x/crypto v0.1.0 // indirect golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17 // indirect - golang.org/x/net v0.8.0 // indirect - golang.org/x/sys v0.7.0 // indirect - golang.org/x/term v0.6.0 // indirect - golang.org/x/text v0.8.0 // indirect - google.golang.org/protobuf v1.28.1 // indirect + golang.org/x/net v0.10.0 // indirect + golang.org/x/sys v0.11.0 // indirect + golang.org/x/term v0.8.0 // indirect + golang.org/x/text v0.9.0 // indirect + google.golang.org/protobuf v1.31.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index 5985bb87..72c2d863 100644 --- a/go.sum +++ b/go.sum @@ -44,7 +44,11 @@ github.com/StackExchange/wmi v0.0.0-20190523213315-cbe66965904d/go.mod h1:3eOhrU github.com/analogj/go-util v0.0.0-20190301173314-5295e364eb14 h1:wsrSjiqQtseStRIoLLxS4C5IEtXkazZVEPDHq8jW7r8= github.com/analogj/go-util v0.0.0-20190301173314-5295e364eb14/go.mod h1:lJQVqFKMV5/oDGYR2bra2OljcF3CvolAoyDRyOA4k4E= github.com/andybalholm/cascadia v1.1.0/go.mod h1:GsXiBklL0woXo1j/WYWtSYYC4ouU9PqHO0sqidkEA4Y= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -274,7 +278,7 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJ github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= -github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= @@ -313,6 +317,8 @@ github.com/mattn/go-isatty v0.0.18/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D github.com/mattn/go-sqlite3 v1.14.0/go.mod h1:JIl7NbARA7phWnGvh0LKTyg7S9BA+6gx71ShQilpsus= github.com/mattn/go-sqlite3 v1.14.12 h1:TJ1bhYJPV44phC+IMu1u2K/i5RriLTPe+yc68XDJ1Z0= github.com/mattn/go-sqlite3 v1.14.12/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= +github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= +github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= @@ -324,7 +330,6 @@ github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= -github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= github.com/onsi/ginkgo/v2 v2.9.2 h1:BA2GMJOtfGAfagzYtrAlufIP0lq6QERkFmHLMLPwFSU= github.com/onsi/gomega v1.27.6 h1:ENqfyGeS5AX/rlXDd/ETokDz93u0YufY1Pgxuy/PvWE= github.com/pelletier/go-toml/v2 v2.0.6 h1:nrzqCb7j9cDFj2coyLNLaZuJTLjWjlaz6nvTvIwycIU= @@ -336,11 +341,19 @@ github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINE github.com/pkg/sftp v1.13.1/go.mod h1:3HaPG6Dq1ILlpPZRO0HVMrsydcdLt6HRDccSgb87qRg= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q= +github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 h1:v7DLqVdK4VrYkVD5diGdl4sxJurKJEMnODWRJlxV9oM= +github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= +github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY= +github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY= +github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwaUuI= +github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY= github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0 h1:OdAsTTz6OkFY5QxjkYwrChwuRruF69c169dPK26NUlk= github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= @@ -501,8 +514,8 @@ golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= -golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= -golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -523,8 +536,8 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -577,12 +590,12 @@ golang.org/x/sys v0.0.0-20220405052023-b1e9470b6e64/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= -golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= -golang.org/x/term v0.6.0 h1:clScbb1cHjoCkyRbWwBEUZ5H/tIFu5TAXIqaZD0Gcjw= -golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= +golang.org/x/term v0.8.0 h1:n5xxQn2i3PC0yLAbjTpNT85q/Kgzcr2gIoX9OrJUols= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -590,8 +603,8 @@ golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= -golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -748,11 +761,11 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= -google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f h1:BLraFXnmrev5lT+xlilqcH8XK9/i0At2xKjWk4p6zsU= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= diff --git a/webapp/backend/pkg/config/config.go b/webapp/backend/pkg/config/config.go index 605de860..07370c4f 100644 --- a/webapp/backend/pkg/config/config.go +++ b/webapp/backend/pkg/config/config.go @@ -1,12 +1,13 @@ package config import ( - "github.com/analogj/go-util/utils" - "github.com/analogj/scrutiny/webapp/backend/pkg/errors" - "github.com/spf13/viper" "log" "os" "strings" + + "github.com/analogj/go-util/utils" + "github.com/analogj/scrutiny/webapp/backend/pkg/errors" + "github.com/spf13/viper" ) const DB_USER_SETTINGS_SUBKEY = "user" @@ -52,6 +53,11 @@ func (c *configuration) Init() error { c.SetDefault("web.influxdb.tls.insecure_skip_verify", false) c.SetDefault("web.influxdb.retention_policy", true) + c.SetDefault("failures.transient.ata", []int{195}) + + // Metrics settings + c.SetDefault("web.metrics.enabled", true) + //c.SetDefault("disks.include", []string{}) //c.SetDefault("disks.exclude", []string{}) diff --git a/webapp/backend/pkg/config/interface.go b/webapp/backend/pkg/config/interface.go index d041dc22..ceca3e93 100644 --- a/webapp/backend/pkg/config/interface.go +++ b/webapp/backend/pkg/config/interface.go @@ -25,5 +25,6 @@ type Interface interface { GetInt64(key string) int64 GetString(key string) string GetStringSlice(key string) []string + GetIntSlice(key string) []int UnmarshalKey(key string, rawVal interface{}, decoderOpts ...viper.DecoderConfigOption) error } diff --git a/webapp/backend/pkg/config/mock/mock_config.go b/webapp/backend/pkg/config/mock/mock_config.go index 1b61b2cb..aed5136d 100644 --- a/webapp/backend/pkg/config/mock/mock_config.go +++ b/webapp/backend/pkg/config/mock/mock_config.go @@ -119,6 +119,20 @@ func (mr *MockInterfaceMockRecorder) GetInt64(key interface{}) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetInt64", reflect.TypeOf((*MockInterface)(nil).GetInt64), key) } +// GetIntSlice mocks base method. +func (m *MockInterface) GetIntSlice(key string) []int { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetIntSlice", key) + ret0, _ := ret[0].([]int) + return ret0 +} + +// GetIntSlice indicates an expected call of GetIntSlice. +func (mr *MockInterfaceMockRecorder) GetIntSlice(key interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetIntSlice", reflect.TypeOf((*MockInterface)(nil).GetIntSlice), key) +} + // GetString mocks base method. func (m *MockInterface) GetString(key string) string { m.ctrl.T.Helper() diff --git a/webapp/backend/pkg/constants.go b/webapp/backend/pkg/constants.go index a82c9c35..009f13f8 100644 --- a/webapp/backend/pkg/constants.go +++ b/webapp/backend/pkg/constants.go @@ -4,8 +4,9 @@ const DeviceProtocolAta = "ATA" const DeviceProtocolScsi = "SCSI" const DeviceProtocolNvme = "NVMe" -//go:generate stringer -type=AttributeStatus // AttributeStatus bitwise flag, 1,2,4,8,16,32,etc +// +//go:generate stringer -type=AttributeStatus type AttributeStatus uint8 const ( @@ -23,8 +24,9 @@ func AttributeStatusClear(b, flag AttributeStatus) AttributeStatus { return b & func AttributeStatusToggle(b, flag AttributeStatus) AttributeStatus { return b ^ flag } func AttributeStatusHas(b, flag AttributeStatus) bool { return b&flag != 0 } -//go:generate stringer -type=DeviceStatus // DeviceStatus bitwise flag, 1,2,4,8,16,32,etc +// +//go:generate stringer -type=DeviceStatus type DeviceStatus uint8 const ( diff --git a/webapp/backend/pkg/database/interface.go b/webapp/backend/pkg/database/interface.go index ee7066e7..bd5e2991 100644 --- a/webapp/backend/pkg/database/interface.go +++ b/webapp/backend/pkg/database/interface.go @@ -21,12 +21,13 @@ type DeviceRepo interface { UpdateDeviceStatus(ctx context.Context, wwn string, status pkg.DeviceStatus) (models.Device, error) GetDeviceDetails(ctx context.Context, wwn string) (models.Device, error) UpdateDeviceArchived(ctx context.Context, wwn string, archived bool) error + UpdateDeviceMuted(ctx context.Context, wwn string, archived bool) error DeleteDevice(ctx context.Context, wwn string) error SaveSmartAttributes(ctx context.Context, wwn string, collectorSmartData collector.SmartInfo) (measurements.Smart, error) GetSmartAttributeHistory(ctx context.Context, wwn string, durationKey string, selectEntries int, selectEntriesOffset int, attributes []string) ([]measurements.Smart, error) - SaveSmartTemperature(ctx context.Context, wwn string, deviceProtocol string, collectorSmartData collector.SmartInfo) error + SaveSmartTemperature(ctx context.Context, wwn string, deviceProtocol string, collectorSmartData collector.SmartInfo, retrieveSCTTemperatureHistory bool) error GetSummary(ctx context.Context) (map[string]*models.DeviceSummary, error) GetSmartTemperatureHistory(ctx context.Context, durationKey string) (map[string][]measurements.SmartTemperature, error) diff --git a/webapp/backend/pkg/database/migrations/m20250221084400/device.go b/webapp/backend/pkg/database/migrations/m20250221084400/device.go index b0a2e5af..9dfafd69 100644 --- a/webapp/backend/pkg/database/migrations/m20250221084400/device.go +++ b/webapp/backend/pkg/database/migrations/m20250221084400/device.go @@ -6,8 +6,9 @@ import ( ) type Device struct { - Archived bool `json:"archived"` //GORM attributes, see: http://gorm.io/docs/conventions.html + Archived bool `json:"archived"` + Muted bool `json:muted` CreatedAt time.Time UpdatedAt time.Time DeletedAt *time.Time diff --git a/webapp/backend/pkg/database/migrations/m20251108044508/device.go b/webapp/backend/pkg/database/migrations/m20251108044508/device.go new file mode 100644 index 00000000..273c2293 --- /dev/null +++ b/webapp/backend/pkg/database/migrations/m20251108044508/device.go @@ -0,0 +1,43 @@ +package m20251108044508 + +import ( + "github.com/analogj/scrutiny/webapp/backend/pkg" + "time" +) + + +type Device struct { + //GORM attributes, see: http://gorm.io/docs/conventions.html + Archived bool `json:"archived"` + Muted bool `json:muted` + CreatedAt time.Time + UpdatedAt time.Time + DeletedAt *time.Time + + WWN string `json:"wwn" gorm:"primary_key"` + + DeviceName string `json:"device_name"` + DeviceUUID string `json:"device_uuid"` + DeviceSerialID string `json:"device_serial_id"` + DeviceLabel string `json:"device_label"` + + Manufacturer string `json:"manufacturer"` + ModelName string `json:"model_name"` + InterfaceType string `json:"interface_type"` + InterfaceSpeed string `json:"interface_speed"` + SerialNumber string `json:"serial_number"` + Firmware string `json:"firmware"` + RotationSpeed int `json:"rotational_speed"` + Capacity int64 `json:"capacity"` + FormFactor string `json:"form_factor"` + SmartSupport bool `json:"smart_support"` + DeviceProtocol string `json:"device_protocol"` //protocol determines which smart attribute types are available (ATA, NVMe, SCSI) + DeviceType string `json:"device_type"` //device type is used for querying with -d/t flag, should only be used by collector. + + // User provided metadata + Label string `json:"label"` + HostId string `json:"host_id"` + + // Data set by Scrutiny + DeviceStatus pkg.DeviceStatus `json:"device_status"` +} diff --git a/webapp/backend/pkg/database/mock/mock_database.go b/webapp/backend/pkg/database/mock/mock_database.go index f5fefc2a..a1b5f7de 100644 --- a/webapp/backend/pkg/database/mock/mock_database.go +++ b/webapp/backend/pkg/database/mock/mock_database.go @@ -66,6 +66,20 @@ func (mr *MockDeviceRepoMockRecorder) UpdateDeviceArchived(ctx, wwn, archived in return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateDeviceArchived", reflect.TypeOf((*MockDeviceRepo)(nil).UpdateDeviceArchived), ctx, wwn, archived) } +// UpdateDeviceMuted mocks base method. +func (m *MockDeviceRepo) UpdateDeviceMuted(ctx context.Context, wwn string, archived bool) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "UpdateDeviceMuted", ctx, wwn) + ret0, _ := ret[0].(error) + return ret0 +} + +// UpdateDeviceMuted indicates an expected call of UpdateDeviceMuted. +func (mr *MockDeviceRepoMockRecorder) UpdateDeviceMuted(ctx, wwn, archived interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "UpdateDeviceMuted", reflect.TypeOf((*MockDeviceRepo)(nil).UpdateDeviceMuted), ctx, wwn, archived) +} + // DeleteDevice mocks base method. func (m *MockDeviceRepo) DeleteDevice(ctx context.Context, wwn string) error { m.ctrl.T.Helper() @@ -228,17 +242,17 @@ func (mr *MockDeviceRepoMockRecorder) SaveSmartAttributes(ctx, wwn, collectorSma } // SaveSmartTemperature mocks base method. -func (m *MockDeviceRepo) SaveSmartTemperature(ctx context.Context, wwn, deviceProtocol string, collectorSmartData collector.SmartInfo) error { +func (m *MockDeviceRepo) SaveSmartTemperature(ctx context.Context, wwn, deviceProtocol string, collectorSmartData collector.SmartInfo, retrieveSCTTemperatureHistory bool) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "SaveSmartTemperature", ctx, wwn, deviceProtocol, collectorSmartData) + ret := m.ctrl.Call(m, "SaveSmartTemperature", ctx, wwn, deviceProtocol, collectorSmartData, retrieveSCTTemperatureHistory) ret0, _ := ret[0].(error) return ret0 } // SaveSmartTemperature indicates an expected call of SaveSmartTemperature. -func (mr *MockDeviceRepoMockRecorder) SaveSmartTemperature(ctx, wwn, deviceProtocol, collectorSmartData interface{}) *gomock.Call { +func (mr *MockDeviceRepoMockRecorder) SaveSmartTemperature(ctx, wwn, deviceProtocol, collectorSmartData, retrieveSCTTemperatureHistory interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SaveSmartTemperature", reflect.TypeOf((*MockDeviceRepo)(nil).SaveSmartTemperature), ctx, wwn, deviceProtocol, collectorSmartData) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SaveSmartTemperature", reflect.TypeOf((*MockDeviceRepo)(nil).SaveSmartTemperature), ctx, wwn, deviceProtocol, collectorSmartData, retrieveSCTTemperatureHistory) } // UpdateDevice mocks base method. diff --git a/webapp/backend/pkg/database/scrutiny_repository.go b/webapp/backend/pkg/database/scrutiny_repository.go index da95914c..f2fa3faf 100644 --- a/webapp/backend/pkg/database/scrutiny_repository.go +++ b/webapp/backend/pkg/database/scrutiny_repository.go @@ -29,6 +29,7 @@ const ( // 60seconds * 60minutes * 24hours * 7 days * (52 + 52 + 4)weeks RETENTION_PERIOD_25_MONTHS_IN_SECONDS = 65_318_400 + DURATION_KEY_DAY = "day" DURATION_KEY_WEEK = "week" DURATION_KEY_MONTH = "month" DURATION_KEY_YEAR = "year" @@ -445,6 +446,7 @@ func (sr *scrutinyRepository) GetSummary(ctx context.Context) (map[string]*model func (sr *scrutinyRepository) lookupBucketName(durationKey string) string { switch durationKey { + case DURATION_KEY_DAY: case DURATION_KEY_WEEK: //data stored in the last week return sr.appConfig.GetString("web.influxdb.bucket") @@ -462,8 +464,10 @@ func (sr *scrutinyRepository) lookupBucketName(durationKey string) string { } func (sr *scrutinyRepository) lookupDuration(durationKey string) []string { - switch durationKey { + case DURATION_KEY_DAY: + //data stored in the last day + return []string{"-1d", "now()"} case DURATION_KEY_WEEK: //data stored in the last week return []string{"-1w", "now()"} @@ -480,8 +484,22 @@ func (sr *scrutinyRepository) lookupDuration(durationKey string) []string { return []string{"-1w", "now()"} } +func (sr *scrutinyRepository) lookupResolution(durationKey string) string { + switch durationKey { + case DURATION_KEY_DAY: + // Return data with higher resolution for daily summaries + return "10m" + default: + // Return data with 1h resolution for other summaries + return "1h" + } +} + func (sr *scrutinyRepository) lookupNestedDurationKeys(durationKey string) []string { switch durationKey { + case DURATION_KEY_DAY: + //all data is stored in a single bucket, but we want a finer resolution + return []string{DURATION_KEY_DAY} case DURATION_KEY_WEEK: //all data is stored in a single bucket return []string{DURATION_KEY_WEEK} diff --git a/webapp/backend/pkg/database/scrutiny_repository_device.go b/webapp/backend/pkg/database/scrutiny_repository_device.go index 8ba0e8c8..4147cc37 100644 --- a/webapp/backend/pkg/database/scrutiny_repository_device.go +++ b/webapp/backend/pkg/database/scrutiny_repository_device.go @@ -84,6 +84,16 @@ func (sr *scrutinyRepository) UpdateDeviceArchived(ctx context.Context, wwn stri return sr.gormClient.Model(&device).Where("wwn = ?", wwn).Update("archived", archived).Error } +// Update Device Muted State +func (sr *scrutinyRepository) UpdateDeviceMuted(ctx context.Context, wwn string, muted bool) error { + var device models.Device + if err := sr.gormClient.WithContext(ctx).Where("wwn = ?", wwn).First(&device).Error; err != nil { + return fmt.Errorf("Could not get device from DB: %v", err) + } + + return sr.gormClient.Model(&device).Where("wwn = ?", wwn).Update("muted", muted).Error +} + func (sr *scrutinyRepository) DeleteDevice(ctx context.Context, wwn string) error { if err := sr.gormClient.WithContext(ctx).Where("wwn = ?", wwn).Delete(&models.Device{}).Error; err != nil { return err diff --git a/webapp/backend/pkg/database/scrutiny_repository_device_smart_attributes.go b/webapp/backend/pkg/database/scrutiny_repository_device_smart_attributes.go index 96bbad05..6ddc58e5 100644 --- a/webapp/backend/pkg/database/scrutiny_repository_device_smart_attributes.go +++ b/webapp/backend/pkg/database/scrutiny_repository_device_smart_attributes.go @@ -13,12 +13,12 @@ import ( log "github.com/sirupsen/logrus" ) -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// ////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // SMART -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// ////////////////////////////////////////////////////////////////////////////////////////////////////////////////// func (sr *scrutinyRepository) SaveSmartAttributes(ctx context.Context, wwn string, collectorSmartData collector.SmartInfo) (measurements.Smart, error) { deviceSmartData := measurements.Smart{} - err := deviceSmartData.FromCollectorSmartInfo(wwn, collectorSmartData) + err := deviceSmartData.FromCollectorSmartInfo(sr.appConfig, wwn, collectorSmartData) if err != nil { sr.logger.Errorln("Could not process SMART metrics", err) return measurements.Smart{}, err @@ -149,12 +149,17 @@ func (sr *scrutinyRepository) aggregateSmartAttributesQuery(wwn string, duration if len(nestedDurationKeys) == 1 { //there's only one bucket being queried, no need to union, just aggregate the dataset and return - partialQueryStr = append(partialQueryStr, []string{ - sr.generateSmartAttributesSubquery(wwn, nestedDurationKeys[0], selectEntries, selectEntriesOffset, attributes), + subqueryParts := []string{ + sr.generateSmartAttributesSubquery(wwn, nestedDurationKeys[0], 0, 0, attributes), fmt.Sprintf(`%sData`, nestedDurationKeys[0]), `|> sort(columns: ["_time"], desc: true)`, - `|> yield()`, - }...) + } + if selectEntries > 0 { + // Use limit() instead of tail() after desc sort to get the newest entries + subqueryParts = append(subqueryParts, fmt.Sprintf(`|> limit(n: %d, offset: %d)`, selectEntries, selectEntriesOffset)) + } + subqueryParts = append(subqueryParts, `|> yield()`) + partialQueryStr = append(partialQueryStr, subqueryParts...) return strings.Join(partialQueryStr, "\n") } @@ -177,7 +182,9 @@ func (sr *scrutinyRepository) aggregateSmartAttributesQuery(wwn string, duration `|> sort(columns: ["_time"], desc: true)`, }...) if selectEntries > 0 { - partialQueryStr = append(partialQueryStr, fmt.Sprintf(`|> tail(n: %d, offset: %d)`, selectEntries, selectEntriesOffset)) + // Use limit() instead of tail() after desc sort to get the newest entries + // tail() would get the oldest entries from the end, but we want the newest from the beginning + partialQueryStr = append(partialQueryStr, fmt.Sprintf(`|> limit(n: %d, offset: %d)`, selectEntries, selectEntriesOffset)) } partialQueryStr = append(partialQueryStr, `|> yield(name: "last")`) @@ -196,7 +203,7 @@ func (sr *scrutinyRepository) generateSmartAttributesSubquery(wwn string, durati } partialQueryStr = append(partialQueryStr, `|> aggregateWindow(every: 1d, fn: last, createEmpty: false)`) - + if selectEntries > 0 { partialQueryStr = append(partialQueryStr, fmt.Sprintf(`|> tail(n: %d, offset: %d)`, selectEntries, selectEntriesOffset)) } diff --git a/webapp/backend/pkg/database/scrutiny_repository_migrations.go b/webapp/backend/pkg/database/scrutiny_repository_migrations.go index e12fae83..d8373c36 100644 --- a/webapp/backend/pkg/database/scrutiny_repository_migrations.go +++ b/webapp/backend/pkg/database/scrutiny_repository_migrations.go @@ -13,6 +13,7 @@ import ( "github.com/analogj/scrutiny/webapp/backend/pkg/database/migrations/m20220509170100" "github.com/analogj/scrutiny/webapp/backend/pkg/database/migrations/m20220716214900" "github.com/analogj/scrutiny/webapp/backend/pkg/database/migrations/m20250221084400" + "github.com/analogj/scrutiny/webapp/backend/pkg/database/migrations/m20251108044508" "github.com/analogj/scrutiny/webapp/backend/pkg/models" "github.com/analogj/scrutiny/webapp/backend/pkg/models/collector" "github.com/analogj/scrutiny/webapp/backend/pkg/models/measurements" @@ -409,6 +410,29 @@ func (sr *scrutinyRepository) Migrate(ctx context.Context) error { return tx.AutoMigrate(m20250221084400.Device{}) }, }, + { + ID: "m20250609210800", // add retrieve_sct_history setting. + Migrate: func(tx *gorm.DB) error { + //add retrieve_sct_history setting default. + var defaultSettings = []m20220716214900.Setting{ + { + SettingKeyName: "collector.retrieve_sct_temperature_history", + SettingKeyDescription: "Whether to retrieve SCT Temperature history (true | false)", + SettingDataType: "bool", + SettingValueBool: true, + }, + } + return tx.Create(&defaultSettings).Error + }, + }, + { + ID: "m20251108044508", // add muted to device data + Migrate: func(tx *gorm.DB) error { + //migrate the device database. + // adding column (muted) + return tx.AutoMigrate(m20251108044508.Device{}) + }, + }, }) if err := m.Migrate(); err != nil { @@ -529,7 +553,7 @@ func m20201107210306_FromPreInfluxDBSmartResultsCreatePostInfluxDBSmartResults(d }) } - postDeviceSmartData.ProcessAtaSmartInfo(preAtaSmartAttributesTable) + postDeviceSmartData.ProcessAtaSmartInfo(nil, preAtaSmartAttributesTable) } else if preDevice.IsNvme() { //info collector.SmartInfo @@ -630,7 +654,7 @@ func m20201107210306_FromPreInfluxDBSmartResultsCreatePostInfluxDBSmartResults(d postScsiErrorCounterLog.Write.TotalUncorrectedErrors = int64(preScsiAttribute.Value) } } - postDeviceSmartData.ProcessScsiSmartInfo(postScsiGrownDefectList, postScsiErrorCounterLog) + postDeviceSmartData.ProcessScsiSmartInfo(postScsiGrownDefectList, postScsiErrorCounterLog, nil) } else { return fmt.Errorf("Unknown device protocol: %s", preDevice.DeviceProtocol), postDeviceSmartData } diff --git a/webapp/backend/pkg/database/scrutiny_repository_temperature.go b/webapp/backend/pkg/database/scrutiny_repository_temperature.go index 9edec87e..37614eab 100644 --- a/webapp/backend/pkg/database/scrutiny_repository_temperature.go +++ b/webapp/backend/pkg/database/scrutiny_repository_temperature.go @@ -3,18 +3,19 @@ package database import ( "context" "fmt" + "strings" + "time" + "github.com/analogj/scrutiny/webapp/backend/pkg/models/collector" "github.com/analogj/scrutiny/webapp/backend/pkg/models/measurements" influxdb2 "github.com/influxdata/influxdb-client-go/v2" - "strings" - "time" ) -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +// ////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // Temperature Data -//////////////////////////////////////////////////////////////////////////////////////////////////////////////////// -func (sr *scrutinyRepository) SaveSmartTemperature(ctx context.Context, wwn string, deviceProtocol string, collectorSmartData collector.SmartInfo) error { - if len(collectorSmartData.AtaSctTemperatureHistory.Table) > 0 { +// ////////////////////////////////////////////////////////////////////////////////////////////////////////////////// +func (sr *scrutinyRepository) SaveSmartTemperature(ctx context.Context, wwn string, deviceProtocol string, collectorSmartData collector.SmartInfo, retrieveSCTTemperatureHistory bool) error { + if len(collectorSmartData.AtaSctTemperatureHistory.Table) > 0 && retrieveSCTTemperatureHistory { for ndx, temp := range collectorSmartData.AtaSctTemperatureHistory.Table { //temp value may be null, we must skip/ignore them. See #393 @@ -22,9 +23,11 @@ func (sr *scrutinyRepository) SaveSmartTemperature(ctx context.Context, wwn stri continue } - minutesOffset := collectorSmartData.AtaSctTemperatureHistory.LoggingIntervalMinutes * int64(ndx) * 60 + intervalSec := collectorSmartData.AtaSctTemperatureHistory.LoggingIntervalMinutes * 60 + datapointTime := collectorSmartData.LocalTime.TimeT - int64(ndx) * intervalSec + alignedDatapointTime := datapointTime - datapointTime % intervalSec smartTemp := measurements.SmartTemperature{ - Date: time.Unix(collectorSmartData.LocalTime.TimeT-minutesOffset, 0), + Date: time.Unix(alignedDatapointTime, 0), Temp: temp, } @@ -39,23 +42,22 @@ func (sr *scrutinyRepository) SaveSmartTemperature(ctx context.Context, wwn stri return err } } - // also add the current temperature. - } else { + } - smartTemp := measurements.SmartTemperature{ - Date: time.Unix(collectorSmartData.LocalTime.TimeT, 0), - Temp: collectorSmartData.Temperature.Current, - } - tags, fields := smartTemp.Flatten() - tags["device_wwn"] = wwn - p := influxdb2.NewPoint("temp", - tags, - fields, - smartTemp.Date) - return sr.influxWriteApi.WritePoint(ctx, p) + // Even if ata_sct_temperature_history is present, also add current temperature. See #824 + smartTemp := measurements.SmartTemperature{ + Date: time.Unix(collectorSmartData.LocalTime.TimeT, 0), + Temp: collectorSmartData.Temperature.Current, } - return nil + + tags, fields := smartTemp.Flatten() + tags["device_wwn"] = wwn + p := influxdb2.NewPoint("temp", + tags, + fields, + smartTemp.Date) + return sr.influxWriteApi.WritePoint(ctx, p) } func (sr *scrutinyRepository) GetSmartTemperatureHistory(ctx context.Context, durationKey string) (map[string][]measurements.SmartTemperature, error) { @@ -138,13 +140,14 @@ func (sr *scrutinyRepository) aggregateTempQuery(durationKey string) string { for _, nestedDurationKey := range nestedDurationKeys { bucketName := sr.lookupBucketName(nestedDurationKey) durationRange := sr.lookupDuration(nestedDurationKey) + durationResolution := sr.lookupResolution(nestedDurationKey) subQueryNames = append(subQueryNames, fmt.Sprintf(`%sData`, nestedDurationKey)) partialQueryStr = append(partialQueryStr, []string{ fmt.Sprintf(`%sData = from(bucket: "%s")`, nestedDurationKey, bucketName), fmt.Sprintf(`|> range(start: %s, stop: %s)`, durationRange[0], durationRange[1]), `|> filter(fn: (r) => r["_measurement"] == "temp" )`, - `|> aggregateWindow(every: 1h, fn: mean, createEmpty: false)`, + fmt.Sprintf(`|> aggregateWindow(every: %s, fn: mean, createEmpty: false)`, durationResolution), `|> group(columns: ["device_wwn"])`, `|> toInt()`, "", @@ -167,5 +170,6 @@ func (sr *scrutinyRepository) aggregateTempQuery(durationKey string) string { }...) } + return strings.Join(partialQueryStr, "\n") } diff --git a/webapp/backend/pkg/metrics/collector.go b/webapp/backend/pkg/metrics/collector.go new file mode 100644 index 00000000..b5b0e4c3 --- /dev/null +++ b/webapp/backend/pkg/metrics/collector.go @@ -0,0 +1,261 @@ +package metrics + +import ( + "context" + "fmt" + "sync" + "time" + + "github.com/analogj/scrutiny/webapp/backend/pkg/database" + "github.com/analogj/scrutiny/webapp/backend/pkg/models" + "github.com/analogj/scrutiny/webapp/backend/pkg/models/measurements" + metricsModels "github.com/analogj/scrutiny/webapp/backend/pkg/models/metrics" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/collectors" + "github.com/sirupsen/logrus" +) + +// Collector manages Prometheus metrics for all devices +type Collector struct { + mu sync.RWMutex + devices map[string]*metricsModels.DeviceMetricsData // key: wwn + registry *prometheus.Registry + logger *logrus.Entry +} + +// NewCollector creates a new metrics collector +func NewCollector(logger *logrus.Entry) *Collector { + mc := &Collector{ + devices: make(map[string]*metricsModels.DeviceMetricsData), + registry: prometheus.NewRegistry(), + logger: logger, + } + + // Register Go runtime metrics (memory, GC, goroutines, etc.) + mc.registry.MustRegister(collectors.NewGoCollector()) + + // Register custom device metrics collector + mc.registry.MustRegister(mc) + return mc +} + +// UpdateDeviceMetrics updates device metrics (called from UploadDeviceMetrics) +func (mc *Collector) UpdateDeviceMetrics(wwn string, device models.Device, smartData measurements.Smart) { + mc.mu.Lock() + defer mc.mu.Unlock() + + mc.devices[wwn] = &metricsModels.DeviceMetricsData{ + Device: device, + SmartData: smartData, + UpdatedAt: time.Now(), + } + mc.logger.Debugf("Updated metrics for device %s", wwn) +} + +// LoadInitialData loads initial data from database (called at startup) +func (mc *Collector) LoadInitialData(deviceRepo database.DeviceRepo, ctx context.Context) error { + start := time.Now() + mc.logger.Info("Loading initial metrics data from database...") + + // Get device summary + summary, err := deviceRepo.GetSummary(ctx) + if err != nil { + return fmt.Errorf("failed to load device summary: %w", err) + } + + // Concurrently fetch latest SMART data for each device + smartDataMap := make(map[string][]measurements.Smart) + var wg sync.WaitGroup + var mu sync.Mutex + + for wwn := range summary { + wg.Add(1) + go func(w string) { + defer wg.Done() + smarts, err := deviceRepo.GetSmartAttributeHistory(ctx, w, "forever", 1, 0, nil) + if err == nil && len(smarts) > 0 { + mu.Lock() + smartDataMap[w] = smarts + mu.Unlock() + } + }(wwn) + } + + wg.Wait() + + // Load into memory + mc.mu.Lock() + defer mc.mu.Unlock() + + for wwn, deviceSummary := range summary { + if smartResults, ok := smartDataMap[wwn]; ok && len(smartResults) > 0 { + mc.devices[wwn] = &metricsModels.DeviceMetricsData{ + Device: deviceSummary.Device, + SmartData: smartResults[0], + UpdatedAt: time.Now(), + } + } + } + + mc.logger.Infof("Loaded metrics for %d devices in %v", len(mc.devices), time.Since(start)) + return nil +} + +// GetRegistry returns the Prometheus registry +func (mc *Collector) GetRegistry() *prometheus.Registry { + return mc.registry +} + +// Describe implements prometheus.Collector interface +func (mc *Collector) Describe(ch chan<- *prometheus.Desc) { + // Dynamic metrics, no need to pre-describe +} + +// Collect implements prometheus.Collector interface +func (mc *Collector) Collect(ch chan<- prometheus.Metric) { + start := time.Now() + mc.mu.RLock() + defer mc.mu.RUnlock() + + mc.collectDeviceInfo(ch) + mc.collectDeviceCapacity(ch) + mc.collectDeviceStatus(ch) + mc.collectSmartAttributes(ch) + mc.collectSummaryMetrics(ch) + mc.collectStatistics(ch) + + mc.logger.Debugf("Metrics collected in %v for %d devices", time.Since(start), len(mc.devices)) +} + +// collectDeviceInfo generates device information metrics +func (mc *Collector) collectDeviceInfo(ch chan<- prometheus.Metric) { + for wwn, data := range mc.devices { + ch <- prometheus.MustNewConstMetric( + prometheus.NewDesc("scrutiny_device_info", "Device information", + []string{"wwn", "device_name", "model_name", "serial_number", + "firmware", "protocol", "host_id", "form_factor"}, nil), + prometheus.GaugeValue, 1, + wwn, data.Device.DeviceName, data.Device.ModelName, + data.Device.SerialNumber, data.Device.Firmware, + data.Device.DeviceProtocol, data.Device.HostId, data.Device.FormFactor, + ) + } +} + +// collectDeviceCapacity generates device capacity metrics +func (mc *Collector) collectDeviceCapacity(ch chan<- prometheus.Metric) { + for wwn, data := range mc.devices { + if data.Device.Capacity > 0 { + ch <- prometheus.MustNewConstMetric( + prometheus.NewDesc("scrutiny_device_capacity_bytes", "Device capacity in bytes", + []string{"wwn", "device_name", "model_name", "protocol", "host_id"}, nil), + prometheus.GaugeValue, float64(data.Device.Capacity), + wwn, data.Device.DeviceName, data.Device.ModelName, + data.Device.DeviceProtocol, data.Device.HostId, + ) + } + } +} + +// collectDeviceStatus generates device status metrics +func (mc *Collector) collectDeviceStatus(ch chan<- prometheus.Metric) { + for wwn, data := range mc.devices { + ch <- prometheus.MustNewConstMetric( + prometheus.NewDesc("scrutiny_device_status", "Device status (0=passed, 1=failed)", + []string{"wwn", "device_name", "model_name", "protocol", "host_id"}, nil), + prometheus.GaugeValue, float64(data.Device.DeviceStatus), + wwn, data.Device.DeviceName, data.Device.ModelName, + data.Device.DeviceProtocol, data.Device.HostId, + ) + } +} + +// collectSmartAttributes generates SMART attribute metrics +func (mc *Collector) collectSmartAttributes(ch chan<- prometheus.Metric) { + for wwn, data := range mc.devices { + baseLabels := []string{wwn, data.Device.DeviceName, data.Device.ModelName, + data.Device.DeviceProtocol, data.Device.HostId} + + for attrID, attr := range data.SmartData.Attributes { + attrLabels := append(baseLabels, attrID) + flattenedAttrs := attr.Flatten() + + for key, value := range flattenedAttrs { + metricName := SanitizeMetricName(key) + if floatVal, ok := TryParseFloat(value); ok { + ch <- prometheus.MustNewConstMetric( + prometheus.NewDesc(metricName, fmt.Sprintf("SMART attribute %s", key), + []string{"wwn", "device_name", "model_name", "protocol", "host_id", "attribute_id"}, nil), + prometheus.GaugeValue, floatVal, attrLabels..., + ) + } + } + } + } +} + +// collectSummaryMetrics generates summary metrics +func (mc *Collector) collectSummaryMetrics(ch chan<- prometheus.Metric) { + for wwn, data := range mc.devices { + labels := []string{wwn, data.Device.DeviceName, data.Device.ModelName, + data.Device.DeviceProtocol, data.Device.HostId} + + if data.SmartData.Temp > 0 { + ch <- prometheus.MustNewConstMetric( + prometheus.NewDesc("scrutiny_smart_temperature_celsius", + "Device temperature in Celsius", + []string{"wwn", "device_name", "model_name", "protocol", "host_id"}, nil), + prometheus.GaugeValue, float64(data.SmartData.Temp), labels..., + ) + } + + if data.SmartData.PowerOnHours > 0 { + ch <- prometheus.MustNewConstMetric( + prometheus.NewDesc("scrutiny_smart_power_on_hours", "Device power on hours", + []string{"wwn", "device_name", "model_name", "protocol", "host_id"}, nil), + prometheus.GaugeValue, float64(data.SmartData.PowerOnHours), labels..., + ) + } + + if data.SmartData.PowerCycleCount > 0 { + ch <- prometheus.MustNewConstMetric( + prometheus.NewDesc("scrutiny_smart_power_cycle_count", "Device power cycle count", + []string{"wwn", "device_name", "model_name", "protocol", "host_id"}, nil), + prometheus.GaugeValue, float64(data.SmartData.PowerCycleCount), labels..., + ) + } + + timestampMs := float64(data.SmartData.Date.Unix() * 1000) + ch <- prometheus.MustNewConstMetric( + prometheus.NewDesc("scrutiny_smart_collector_timestamp", + "Timestamp of last data collection", + []string{"wwn", "device_name", "model_name", "protocol", "host_id"}, nil), + prometheus.GaugeValue, timestampMs, labels..., + ) + } +} + +// collectStatistics generates statistics metrics +func (mc *Collector) collectStatistics(ch chan<- prometheus.Metric) { + ch <- prometheus.MustNewConstMetric( + prometheus.NewDesc("scrutiny_devices_total", "Total number of monitored devices", nil, nil), + prometheus.GaugeValue, float64(len(mc.devices)), + ) + + protocolCount := make(map[string]int) + for _, data := range mc.devices { + protocol := data.Device.DeviceProtocol + if protocol == "" { + protocol = "unknown" + } + protocolCount[protocol]++ + } + + for protocol, count := range protocolCount { + ch <- prometheus.MustNewConstMetric( + prometheus.NewDesc("scrutiny_devices_by_protocol", "Number of devices by protocol", + []string{"protocol"}, nil), + prometheus.GaugeValue, float64(count), protocol, + ) + } +} diff --git a/webapp/backend/pkg/metrics/utils.go b/webapp/backend/pkg/metrics/utils.go new file mode 100644 index 00000000..3df0efea --- /dev/null +++ b/webapp/backend/pkg/metrics/utils.go @@ -0,0 +1,63 @@ +package metrics + +import ( + "strconv" + "strings" + + "github.com/analogj/scrutiny/webapp/backend/pkg/models/measurements" +) + +// SanitizeMetricName converts a string to a valid Prometheus metric name +// Example: converts "attr.5.raw_value" to "scrutiny_smart_attr_5_raw_value" +func SanitizeMetricName(name string) string { + name = strings.ReplaceAll(name, ".", "_") + name = strings.ReplaceAll(name, "-", "_") + name = strings.ReplaceAll(name, " ", "_") + name = strings.ToLower(name) + + if strings.HasPrefix(name, "attr_") { + return "scrutiny_smart_" + name + } + return name +} + +// TryParseFloat attempts to convert any type to float64 +// Supports: int, int64, float32, float64, string, hexadecimal strings +func TryParseFloat(value interface{}) (float64, bool) { + switch v := value.(type) { + case int: + return float64(v), true + case int64: + return float64(v), true + case float64: + return v, true + case float32: + return float64(v), true + case string: + if f, err := strconv.ParseFloat(v, 64); err == nil { + return f, true + } + // Try parsing hexadecimal + if strings.HasPrefix(v, "0x") || strings.HasPrefix(v, "0X") { + if i, err := strconv.ParseInt(v, 0, 64); err == nil { + return float64(i), true + } + } + } + return 0, false +} + +// SelectLatestSmartResult selects the latest SMART result from a list (by timestamp) +func SelectLatestSmartResult(smartResults []measurements.Smart) *measurements.Smart { + if len(smartResults) == 0 { + return nil + } + + latest := &smartResults[0] + for i := 1; i < len(smartResults); i++ { + if smartResults[i].Date.After(latest.Date) { + latest = &smartResults[i] + } + } + return latest +} diff --git a/webapp/backend/pkg/metrics/utils_test.go b/webapp/backend/pkg/metrics/utils_test.go new file mode 100644 index 00000000..e4f078f4 --- /dev/null +++ b/webapp/backend/pkg/metrics/utils_test.go @@ -0,0 +1,218 @@ +package metrics + +import ( + "testing" + "time" + + "github.com/analogj/scrutiny/webapp/backend/pkg/models/measurements" +) + +func TestSanitizeMetricName(t *testing.T) { + tests := []struct { + name string + input string + expected string + }{ + { + name: "convert dots to underscores", + input: "attr.5.raw_value", + expected: "scrutiny_smart_attr_5_raw_value", + }, + { + name: "convert hyphens to underscores", + input: "attr-5-raw-value", + expected: "scrutiny_smart_attr_5_raw_value", + }, + { + name: "convert spaces to underscores", + input: "attr 5 raw value", + expected: "scrutiny_smart_attr_5_raw_value", + }, + { + name: "convert to lowercase", + input: "Attr.5.Raw_Value", + expected: "scrutiny_smart_attr_5_raw_value", + }, + { + name: "already valid name", + input: "valid_metric_name", + expected: "valid_metric_name", + }, + { + name: "without attr prefix", + input: "some.metric.name", + expected: "some_metric_name", + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := SanitizeMetricName(tt.input) + if result != tt.expected { + t.Errorf("SanitizeMetricName(%q) = %q, want %q", tt.input, result, tt.expected) + } + }) + } +} + +func TestTryParseFloat(t *testing.T) { + tests := []struct { + name string + input interface{} + expected float64 + shouldOk bool + }{ + { + name: "parse int", + input: 42, + expected: 42.0, + shouldOk: true, + }, + { + name: "parse int64", + input: int64(12345), + expected: 12345.0, + shouldOk: true, + }, + { + name: "parse float64", + input: 3.14159, + expected: 3.14159, + shouldOk: true, + }, + { + name: "parse float32", + input: float32(2.71), + expected: float64(float32(2.71)), // Account for float32 precision + shouldOk: true, + }, + { + name: "parse string number", + input: "123.45", + expected: 123.45, + shouldOk: true, + }, + { + name: "parse hexadecimal with 0x", + input: "0x1A", + expected: 26.0, + shouldOk: true, + }, + { + name: "parse hexadecimal with 0X", + input: "0XFF", + expected: 255.0, + shouldOk: true, + }, + { + name: "parse empty string", + input: "", + expected: 0, + shouldOk: false, + }, + { + name: "parse invalid string", + input: "not_a_number", + expected: 0, + shouldOk: false, + }, + { + name: "parse nil", + input: nil, + expected: 0, + shouldOk: false, + }, + { + name: "parse bool", + input: true, + expected: 0, + shouldOk: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result, ok := TryParseFloat(tt.input) + if ok != tt.shouldOk { + t.Errorf("TryParseFloat(%v) ok = %v, want %v", tt.input, ok, tt.shouldOk) + } + if tt.shouldOk && result != tt.expected { + t.Errorf("TryParseFloat(%v) = %v, want %v", tt.input, result, tt.expected) + } + }) + } +} + +func TestSelectLatestSmartResult(t *testing.T) { + now := time.Now() + older := now.Add(-1 * time.Hour) + oldest := now.Add(-2 * time.Hour) + + tests := []struct { + name string + input []measurements.Smart + expected *time.Time + }{ + { + name: "empty list", + input: []measurements.Smart{}, + expected: nil, + }, + { + name: "single result", + input: []measurements.Smart{ + {Date: now}, + }, + expected: &now, + }, + { + name: "multiple results in order", + input: []measurements.Smart{ + {Date: now}, + {Date: older}, + {Date: oldest}, + }, + expected: &now, + }, + { + name: "multiple results out of order", + input: []measurements.Smart{ + {Date: older}, + {Date: now}, + {Date: oldest}, + }, + expected: &now, + }, + { + name: "multiple results reverse order", + input: []measurements.Smart{ + {Date: oldest}, + {Date: older}, + {Date: now}, + }, + expected: &now, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + result := SelectLatestSmartResult(tt.input) + + if tt.expected == nil { + if result != nil { + t.Errorf("SelectLatestSmartResult() = %v, want nil", result) + } + return + } + + if result == nil { + t.Errorf("SelectLatestSmartResult() = nil, want non-nil") + return + } + + if !result.Date.Equal(*tt.expected) { + t.Errorf("SelectLatestSmartResult() date = %v, want %v", result.Date, *tt.expected) + } + }) + } +} diff --git a/webapp/backend/pkg/models/collector/smart.go b/webapp/backend/pkg/models/collector/smart.go index c466ca48..e5190cdd 100644 --- a/webapp/backend/pkg/models/collector/smart.go +++ b/webapp/backend/pkg/models/collector/smart.go @@ -235,6 +235,17 @@ type SmartInfo struct { ScsiVersion string `json:"scsi_version"` ScsiGrownDefectList int64 `json:"scsi_grown_defect_list"` ScsiErrorCounterLog ScsiErrorCounterLog `json:"scsi_error_counter_log"` + + ScsiEnvironmentalReports map[string]ScsiTemperatureData `json:"scsi_environmental_reports"` +} + +type ScsiTemperatureData struct { + ParameterCode int `json:"parameter_code"` + Current int64 `json:"current"` + LifetimeMaximum int64 `json:"lifetime_maximum"` + LifetimeMinimum int64 `json:"lifetime_minimum"` + MaximumSincePowerOn int64 `json:"maximum_since_power_on"` + MinimumSincePowerOn int64 `json:"minimum_since_power_on"` } // Capacity finds the total capacity of the device in bytes, or 0 if unknown. diff --git a/webapp/backend/pkg/models/device.go b/webapp/backend/pkg/models/device.go index a891652e..4702ee80 100644 --- a/webapp/backend/pkg/models/device.go +++ b/webapp/backend/pkg/models/device.go @@ -15,6 +15,7 @@ type DeviceWrapper struct { type Device struct { //GORM attributes, see: http://gorm.io/docs/conventions.html Archived bool `json:"archived"` + Muted bool `json:"muted"` CreatedAt time.Time UpdatedAt time.Time DeletedAt *time.Time diff --git a/webapp/backend/pkg/models/measurements/smart.go b/webapp/backend/pkg/models/measurements/smart.go index 40c13970..621da3d4 100644 --- a/webapp/backend/pkg/models/measurements/smart.go +++ b/webapp/backend/pkg/models/measurements/smart.go @@ -2,13 +2,15 @@ package measurements import ( "fmt" - "github.com/analogj/scrutiny/webapp/backend/pkg" - "github.com/analogj/scrutiny/webapp/backend/pkg/models/collector" - "github.com/analogj/scrutiny/webapp/backend/pkg/thresholds" "log" "strconv" "strings" "time" + + "github.com/analogj/scrutiny/webapp/backend/pkg" + "github.com/analogj/scrutiny/webapp/backend/pkg/config" + "github.com/analogj/scrutiny/webapp/backend/pkg/models/collector" + "github.com/analogj/scrutiny/webapp/backend/pkg/thresholds" ) type Smart struct { @@ -100,8 +102,8 @@ func NewSmartFromInfluxDB(attrs map[string]interface{}) (*Smart, error) { return &sm, nil } -//Parse Collector SMART data results and create Smart object (and associated SmartAtaAttribute entries) -func (sm *Smart) FromCollectorSmartInfo(wwn string, info collector.SmartInfo) error { +// Parse Collector SMART data results and create Smart object (and associated SmartAtaAttribute entries) +func (sm *Smart) FromCollectorSmartInfo(cfg config.Interface, wwn string, info collector.SmartInfo) error { sm.DeviceWWN = wwn sm.Date = time.Unix(info.LocalTime.TimeT, 0) @@ -117,18 +119,18 @@ func (sm *Smart) FromCollectorSmartInfo(wwn string, info collector.SmartInfo) er // process ATA/NVME/SCSI protocol data sm.Attributes = map[string]SmartAttribute{} if sm.DeviceProtocol == pkg.DeviceProtocolAta { - sm.ProcessAtaSmartInfo(info.AtaSmartAttributes.Table) + sm.ProcessAtaSmartInfo(cfg, info.AtaSmartAttributes.Table) } else if sm.DeviceProtocol == pkg.DeviceProtocolNvme { sm.ProcessNvmeSmartInfo(info.NvmeSmartHealthInformationLog) } else if sm.DeviceProtocol == pkg.DeviceProtocolScsi { - sm.ProcessScsiSmartInfo(info.ScsiGrownDefectList, info.ScsiErrorCounterLog) + sm.ProcessScsiSmartInfo(info.ScsiGrownDefectList, info.ScsiErrorCounterLog, info.ScsiEnvironmentalReports) } return nil } -//generate SmartAtaAttribute entries from Scrutiny Collector Smart data. -func (sm *Smart) ProcessAtaSmartInfo(tableItems []collector.AtaSmartAttributesTableItem) { +// generate SmartAtaAttribute entries from Scrutiny Collector Smart data. +func (sm *Smart) ProcessAtaSmartInfo(cfg config.Interface, tableItems []collector.AtaSmartAttributesTableItem) { for _, collectorAttr := range tableItems { attrModel := SmartAtaAttribute{ AttributeId: collectorAttr.ID, @@ -149,13 +151,25 @@ func (sm *Smart) ProcessAtaSmartInfo(tableItems []collector.AtaSmartAttributesTa attrModel.PopulateAttributeStatus() sm.Attributes[strconv.Itoa(collectorAttr.ID)] = &attrModel - if pkg.AttributeStatusHas(attrModel.Status, pkg.AttributeStatusFailedScrutiny) { + var transient bool + + if cfg != nil { + transients := cfg.GetIntSlice("failures.transient.ata") + for i := range transients { + if collectorAttr.ID == transients[i] { + transient = true + break + } + } + } + + if pkg.AttributeStatusHas(attrModel.Status, pkg.AttributeStatusFailedScrutiny) && !transient { sm.Status = pkg.DeviceStatusSet(sm.Status, pkg.DeviceStatusFailedScrutiny) } } } -//generate SmartNvmeAttribute entries from Scrutiny Collector Smart data. +// generate SmartNvmeAttribute entries from Scrutiny Collector Smart data. func (sm *Smart) ProcessNvmeSmartInfo(nvmeSmartHealthInformationLog collector.NvmeSmartHealthInformationLog) { sm.Attributes = map[string]SmartAttribute{ @@ -185,9 +199,11 @@ func (sm *Smart) ProcessNvmeSmartInfo(nvmeSmartHealthInformationLog collector.Nv } } -//generate SmartScsiAttribute entries from Scrutiny Collector Smart data. -func (sm *Smart) ProcessScsiSmartInfo(defectGrownList int64, scsiErrorCounterLog collector.ScsiErrorCounterLog) { +// generate SmartScsiAttribute entries from Scrutiny Collector Smart data. +func (sm *Smart) ProcessScsiSmartInfo(defectGrownList int64, scsiErrorCounterLog collector.ScsiErrorCounterLog, temperature map[string]collector.ScsiTemperatureData) { sm.Attributes = map[string]SmartAttribute{ + "temperature": (&SmartNvmeAttribute{AttributeId: "temperature", Value: getScsiTemperature(temperature), Threshold: -1}).PopulateAttributeStatus(), + "scsi_grown_defect_list": (&SmartScsiAttribute{AttributeId: "scsi_grown_defect_list", Value: defectGrownList, Threshold: 0}).PopulateAttributeStatus(), "read_errors_corrected_by_eccfast": (&SmartScsiAttribute{AttributeId: "read_errors_corrected_by_eccfast", Value: scsiErrorCounterLog.Read.ErrorsCorrectedByEccfast, Threshold: -1}).PopulateAttributeStatus(), "read_errors_corrected_by_eccdelayed": (&SmartScsiAttribute{AttributeId: "read_errors_corrected_by_eccdelayed", Value: scsiErrorCounterLog.Read.ErrorsCorrectedByEccdelayed, Threshold: -1}).PopulateAttributeStatus(), @@ -210,3 +226,12 @@ func (sm *Smart) ProcessScsiSmartInfo(defectGrownList int64, scsiErrorCounterLog } } } + +func getScsiTemperature(s map[string]collector.ScsiTemperatureData) int64 { + temp, ok := s["temperature_1"] + if !ok { + return 0 + } + + return temp.Current +} diff --git a/webapp/backend/pkg/models/measurements/smart_test.go b/webapp/backend/pkg/models/measurements/smart_test.go index ad7b73e9..8eadcfa4 100644 --- a/webapp/backend/pkg/models/measurements/smart_test.go +++ b/webapp/backend/pkg/models/measurements/smart_test.go @@ -2,14 +2,17 @@ package measurements_test import ( "encoding/json" - "github.com/analogj/scrutiny/webapp/backend/pkg" - "github.com/analogj/scrutiny/webapp/backend/pkg/models/collector" - "github.com/analogj/scrutiny/webapp/backend/pkg/models/measurements" - "github.com/stretchr/testify/require" "io/ioutil" "os" "testing" "time" + + "github.com/analogj/scrutiny/webapp/backend/pkg" + mock_config "github.com/analogj/scrutiny/webapp/backend/pkg/config/mock" + "github.com/analogj/scrutiny/webapp/backend/pkg/models/collector" + "github.com/analogj/scrutiny/webapp/backend/pkg/models/measurements" + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/require" ) func TestSmart_Flatten(t *testing.T) { @@ -306,6 +309,11 @@ func TestNewSmartFromInfluxDB_SCSI(t *testing.T) { func TestFromCollectorSmartInfo(t *testing.T) { //setup + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + fakeConfig := mock_config.NewMockInterface(mockCtrl) + fakeConfig.EXPECT().GetIntSlice("failures.transient.ata").Return([]int{195}).AnyTimes() + smartDataFile, err := os.Open("../testdata/smart-ata.json") require.NoError(t, err) defer smartDataFile.Close() @@ -319,7 +327,7 @@ func TestFromCollectorSmartInfo(t *testing.T) { //test smartMdl := measurements.Smart{} - err = smartMdl.FromCollectorSmartInfo("WWN-test", smartJson) + err = smartMdl.FromCollectorSmartInfo(fakeConfig, "WWN-test", smartJson) //assert require.NoError(t, err) @@ -338,6 +346,11 @@ func TestFromCollectorSmartInfo(t *testing.T) { func TestFromCollectorSmartInfo_Fail_Smart(t *testing.T) { //setup + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + fakeConfig := mock_config.NewMockInterface(mockCtrl) + fakeConfig.EXPECT().GetIntSlice("failures.transient.ata").Return([]int{195}).AnyTimes() + smartDataFile, err := os.Open("../testdata/smart-fail.json") require.NoError(t, err) defer smartDataFile.Close() @@ -351,7 +364,7 @@ func TestFromCollectorSmartInfo_Fail_Smart(t *testing.T) { //test smartMdl := measurements.Smart{} - err = smartMdl.FromCollectorSmartInfo("WWN-test", smartJson) + err = smartMdl.FromCollectorSmartInfo(fakeConfig, "WWN-test", smartJson) //assert require.NoError(t, err) @@ -362,6 +375,11 @@ func TestFromCollectorSmartInfo_Fail_Smart(t *testing.T) { func TestFromCollectorSmartInfo_Fail_ScrutinySmart(t *testing.T) { //setup + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + fakeConfig := mock_config.NewMockInterface(mockCtrl) + fakeConfig.EXPECT().GetIntSlice("failures.transient.ata").Return([]int{195}).AnyTimes() + smartDataFile, err := os.Open("../testdata/smart-fail2.json") require.NoError(t, err) defer smartDataFile.Close() @@ -375,7 +393,7 @@ func TestFromCollectorSmartInfo_Fail_ScrutinySmart(t *testing.T) { //test smartMdl := measurements.Smart{} - err = smartMdl.FromCollectorSmartInfo("WWN-test", smartJson) + err = smartMdl.FromCollectorSmartInfo(fakeConfig, "WWN-test", smartJson) //assert require.NoError(t, err) @@ -386,6 +404,11 @@ func TestFromCollectorSmartInfo_Fail_ScrutinySmart(t *testing.T) { func TestFromCollectorSmartInfo_Fail_ScrutinyNonCriticalFailed(t *testing.T) { //setup + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + fakeConfig := mock_config.NewMockInterface(mockCtrl) + fakeConfig.EXPECT().GetIntSlice("failures.transient.ata").Return([]int{195}).AnyTimes() + smartDataFile, err := os.Open("../testdata/smart-ata-failed-scrutiny.json") require.NoError(t, err) defer smartDataFile.Close() @@ -399,7 +422,7 @@ func TestFromCollectorSmartInfo_Fail_ScrutinyNonCriticalFailed(t *testing.T) { //test smartMdl := measurements.Smart{} - err = smartMdl.FromCollectorSmartInfo("WWN-test", smartJson) + err = smartMdl.FromCollectorSmartInfo(fakeConfig, "WWN-test", smartJson) //assert require.NoError(t, err) @@ -419,6 +442,11 @@ func TestFromCollectorSmartInfo_Fail_ScrutinyNonCriticalFailed(t *testing.T) { func TestFromCollectorSmartInfo_NVMe_Fail_Scrutiny(t *testing.T) { //setup + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + fakeConfig := mock_config.NewMockInterface(mockCtrl) + fakeConfig.EXPECT().GetIntSlice("failures.transient.ata").Return([]int{195}).AnyTimes() + smartDataFile, err := os.Open("../testdata/smart-nvme-failed.json") require.NoError(t, err) defer smartDataFile.Close() @@ -432,7 +460,7 @@ func TestFromCollectorSmartInfo_NVMe_Fail_Scrutiny(t *testing.T) { //test smartMdl := measurements.Smart{} - err = smartMdl.FromCollectorSmartInfo("WWN-test", smartJson) + err = smartMdl.FromCollectorSmartInfo(fakeConfig, "WWN-test", smartJson) //assert require.NoError(t, err) @@ -450,6 +478,11 @@ func TestFromCollectorSmartInfo_NVMe_Fail_Scrutiny(t *testing.T) { func TestFromCollectorSmartInfo_Nvme(t *testing.T) { //setup + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + fakeConfig := mock_config.NewMockInterface(mockCtrl) + fakeConfig.EXPECT().GetIntSlice("failures.transient.ata").Return([]int{195}).AnyTimes() + smartDataFile, err := os.Open("../testdata/smart-nvme.json") require.NoError(t, err) defer smartDataFile.Close() @@ -463,7 +496,7 @@ func TestFromCollectorSmartInfo_Nvme(t *testing.T) { //test smartMdl := measurements.Smart{} - err = smartMdl.FromCollectorSmartInfo("WWN-test", smartJson) + err = smartMdl.FromCollectorSmartInfo(fakeConfig, "WWN-test", smartJson) //assert require.NoError(t, err) @@ -477,6 +510,11 @@ func TestFromCollectorSmartInfo_Nvme(t *testing.T) { func TestFromCollectorSmartInfo_Scsi(t *testing.T) { //setup + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + fakeConfig := mock_config.NewMockInterface(mockCtrl) + fakeConfig.EXPECT().GetIntSlice("failures.transient.ata").Return([]int{195}).AnyTimes() + smartDataFile, err := os.Open("../testdata/smart-scsi.json") require.NoError(t, err) defer smartDataFile.Close() @@ -490,7 +528,7 @@ func TestFromCollectorSmartInfo_Scsi(t *testing.T) { //test smartMdl := measurements.Smart{} - err = smartMdl.FromCollectorSmartInfo("WWN-test", smartJson) + err = smartMdl.FromCollectorSmartInfo(fakeConfig, "WWN-test", smartJson) //assert require.NoError(t, err) diff --git a/webapp/backend/pkg/models/metrics/types.go b/webapp/backend/pkg/models/metrics/types.go new file mode 100644 index 00000000..8dd5d9b4 --- /dev/null +++ b/webapp/backend/pkg/models/metrics/types.go @@ -0,0 +1,15 @@ +package metrics + +import ( + "time" + + "github.com/analogj/scrutiny/webapp/backend/pkg/models" + "github.com/analogj/scrutiny/webapp/backend/pkg/models/measurements" +) + +// DeviceMetricsData stores metrics data for a single device +type DeviceMetricsData struct { + Device models.Device `json:"device"` + SmartData measurements.Smart `json:"smart_data"` + UpdatedAt time.Time `json:"updated_at"` +} diff --git a/webapp/backend/pkg/models/settings.go b/webapp/backend/pkg/models/settings.go index d40e3e9c..a08ea76b 100644 --- a/webapp/backend/pkg/models/settings.go +++ b/webapp/backend/pkg/models/settings.go @@ -17,6 +17,10 @@ type Settings struct { LineStroke string `json:"line_stroke" mapstructure:"line_stroke"` PoweredOnHoursUnit string `json:"powered_on_hours_unit" mapstructure:"powered_on_hours_unit"` + Collector struct { + RetrieveSCTHistory bool `json:"retrieve_sct_temperature_history" mapstructure:"retrieve_sct_temperature_history"` + } `json:"collector" mapstructure:"collector"` + Metrics struct { NotifyLevel int `json:"notify_level" mapstructure:"notify_level"` StatusFilterAttributes int `json:"status_filter_attributes" mapstructure:"status_filter_attributes"` diff --git a/webapp/backend/pkg/notify/notify.go b/webapp/backend/pkg/notify/notify.go index bf927ee3..155e782d 100644 --- a/webapp/backend/pkg/notify/notify.go +++ b/webapp/backend/pkg/notify/notify.go @@ -38,6 +38,11 @@ func ShouldNotify(logger logrus.FieldLogger, device models.Device, smartAttrs me return false } + // If the device is muted, skip notification regardless of status + if device.Muted { + return false + } + //TODO: cannot check for warning notifyLevel yet. // setup constants for comparison diff --git a/webapp/backend/pkg/web/handler/get_metrics.go b/webapp/backend/pkg/web/handler/get_metrics.go new file mode 100644 index 00000000..0add2641 --- /dev/null +++ b/webapp/backend/pkg/web/handler/get_metrics.go @@ -0,0 +1,23 @@ +package handler + +import ( + "github.com/analogj/scrutiny/webapp/backend/pkg/metrics" + "github.com/gin-gonic/gin" + "github.com/prometheus/client_golang/prometheus/promhttp" + "github.com/sirupsen/logrus" +) + +// GetMetrics handles Prometheus metrics endpoint +func GetMetrics(c *gin.Context) { + logger := c.MustGet("LOGGER").(*logrus.Entry) + collector, exists := c.MustGet("METRICS_COLLECTOR").(*metrics.Collector) + + if !exists || collector == nil { + logger.Errorln("Metrics collector not found in context") + c.String(500, "Metrics collector not initialized") + return + } + + handler := promhttp.HandlerFor(collector.GetRegistry(), promhttp.HandlerOpts{}) + handler.ServeHTTP(c.Writer, c.Request) +} diff --git a/webapp/backend/pkg/web/handler/mute_device.go b/webapp/backend/pkg/web/handler/mute_device.go new file mode 100644 index 00000000..38c581d1 --- /dev/null +++ b/webapp/backend/pkg/web/handler/mute_device.go @@ -0,0 +1,22 @@ +package handler + +import ( + "github.com/analogj/scrutiny/webapp/backend/pkg/database" + "github.com/gin-gonic/gin" + "github.com/sirupsen/logrus" + "net/http" +) + +func MuteDevice(c *gin.Context) { + logger := c.MustGet("LOGGER").(*logrus.Entry) + deviceRepo := c.MustGet("DEVICE_REPOSITORY").(database.DeviceRepo) + + err := deviceRepo.UpdateDeviceMuted(c, c.Param("wwn"), true) + if err != nil { + logger.Errorln("An error occurred while muting device", err) + c.JSON(http.StatusInternalServerError, gin.H{"success": false}) + return + } + + c.JSON(http.StatusOK, gin.H{"success": true}) +} diff --git a/webapp/backend/pkg/web/handler/unmute_device.go b/webapp/backend/pkg/web/handler/unmute_device.go new file mode 100644 index 00000000..11bb2306 --- /dev/null +++ b/webapp/backend/pkg/web/handler/unmute_device.go @@ -0,0 +1,22 @@ +package handler + +import ( + "github.com/analogj/scrutiny/webapp/backend/pkg/database" + "github.com/gin-gonic/gin" + "github.com/sirupsen/logrus" + "net/http" +) + +func UnmuteDevice(c *gin.Context) { + logger := c.MustGet("LOGGER").(*logrus.Entry) + deviceRepo := c.MustGet("DEVICE_REPOSITORY").(database.DeviceRepo) + + err := deviceRepo.UpdateDeviceMuted(c, c.Param("wwn"), false) + if err != nil { + logger.Errorln("An error occurred while muting device", err) + c.JSON(http.StatusInternalServerError, gin.H{"success": false}) + return + } + + c.JSON(http.StatusOK, gin.H{"success": true}) +} diff --git a/webapp/backend/pkg/web/handler/upload_device_metrics.go b/webapp/backend/pkg/web/handler/upload_device_metrics.go index 08144338..32f33c9b 100644 --- a/webapp/backend/pkg/web/handler/upload_device_metrics.go +++ b/webapp/backend/pkg/web/handler/upload_device_metrics.go @@ -7,6 +7,7 @@ import ( "github.com/analogj/scrutiny/webapp/backend/pkg" "github.com/analogj/scrutiny/webapp/backend/pkg/config" "github.com/analogj/scrutiny/webapp/backend/pkg/database" + "github.com/analogj/scrutiny/webapp/backend/pkg/metrics" "github.com/analogj/scrutiny/webapp/backend/pkg/models/collector" "github.com/analogj/scrutiny/webapp/backend/pkg/notify" "github.com/gin-gonic/gin" @@ -61,7 +62,7 @@ func UploadDeviceMetrics(c *gin.Context) { } // save smart temperature data (ignore failures) - err = deviceRepo.SaveSmartTemperature(c, c.Param("wwn"), updatedDevice.DeviceProtocol, collectorSmartData) + err = deviceRepo.SaveSmartTemperature(c, c.Param("wwn"), updatedDevice.DeviceProtocol, collectorSmartData, appConfig.GetBool(fmt.Sprintf("%s.collector.retrieve_sct_temperature_history", config.DB_USER_SETTINGS_SUBKEY))) if err != nil { logger.Errorln("An error occurred while saving smartctl temp data", err) c.JSON(http.StatusInternalServerError, gin.H{"success": false}) @@ -90,5 +91,12 @@ func UploadDeviceMetrics(c *gin.Context) { _ = liveNotify.Send() //we ignore error message when sending notifications. } + // Update Prometheus metrics (if enabled) + if collectorVal, exists := c.Get("METRICS_COLLECTOR"); exists { + if collector, ok := collectorVal.(*metrics.Collector); ok && collector != nil { + collector.UpdateDeviceMetrics(c.Param("wwn"), updatedDevice, smartData) + } + } + c.JSON(http.StatusOK, gin.H{"success": true}) } diff --git a/webapp/backend/pkg/web/middleware/metrics.go b/webapp/backend/pkg/web/middleware/metrics.go new file mode 100644 index 00000000..373a6ee2 --- /dev/null +++ b/webapp/backend/pkg/web/middleware/metrics.go @@ -0,0 +1,14 @@ +package middleware + +import ( + "github.com/analogj/scrutiny/webapp/backend/pkg/metrics" + "github.com/gin-gonic/gin" +) + +// MetricsMiddleware injects metrics collector into gin context +func MetricsMiddleware(collector *metrics.Collector) gin.HandlerFunc { + return func(c *gin.Context) { + c.Set("METRICS_COLLECTOR", collector) + c.Next() + } +} diff --git a/webapp/backend/pkg/web/server.go b/webapp/backend/pkg/web/server.go index 38383c92..485a6a15 100644 --- a/webapp/backend/pkg/web/server.go +++ b/webapp/backend/pkg/web/server.go @@ -1,22 +1,27 @@ package web import ( + "context" "fmt" + "net/http" + "path/filepath" + "strings" + "github.com/analogj/go-util/utils" "github.com/analogj/scrutiny/webapp/backend/pkg/config" + "github.com/analogj/scrutiny/webapp/backend/pkg/database" "github.com/analogj/scrutiny/webapp/backend/pkg/errors" + "github.com/analogj/scrutiny/webapp/backend/pkg/metrics" "github.com/analogj/scrutiny/webapp/backend/pkg/web/handler" "github.com/analogj/scrutiny/webapp/backend/pkg/web/middleware" "github.com/gin-gonic/gin" "github.com/sirupsen/logrus" - "net/http" - "path/filepath" - "strings" ) type AppEngine struct { - Config config.Interface - Logger *logrus.Entry + Config config.Interface + Logger *logrus.Entry + MetricsCollector *metrics.Collector } func (ae *AppEngine) Setup(logger *logrus.Entry) *gin.Engine { @@ -25,6 +30,17 @@ func (ae *AppEngine) Setup(logger *logrus.Entry) *gin.Engine { r.Use(middleware.LoggerMiddleware(logger)) r.Use(middleware.RepositoryMiddleware(ae.Config, logger)) r.Use(middleware.ConfigMiddleware(ae.Config)) + + // Initialize metrics collector if enabled + if ae.Config.GetBool("web.metrics.enabled") { + if ae.MetricsCollector == nil { + ae.MetricsCollector = metrics.NewCollector(logger) + } + r.Use(middleware.MetricsMiddleware(ae.MetricsCollector)) + logger.Info("Prometheus metrics endpoint enabled") + } else { + logger.Info("Prometheus metrics endpoint disabled") + } r.Use(gin.Recovery()) basePath := ae.Config.GetString("web.listen.basepath") @@ -40,11 +56,19 @@ func (ae *AppEngine) Setup(logger *logrus.Entry) *gin.Engine { api.POST("/devices/register", handler.RegisterDevices) //used by Collector to register new devices and retrieve filtered list api.GET("/summary", handler.GetDevicesSummary) //used by Dashboard api.GET("/summary/temp", handler.GetDevicesSummaryTempHistory) //used by Dashboard (Temperature history dropdown) - api.POST("/device/:wwn/smart", handler.UploadDeviceMetrics) //used by Collector to upload data + + // Prometheus metrics endpoint (only registered if enabled) + if ae.Config.GetBool("web.metrics.enabled") { + api.GET("/metrics", handler.GetMetrics) + } + + api.POST("/device/:wwn/smart", handler.UploadDeviceMetrics) //used by Collector to upload data api.POST("/device/:wwn/selftest", handler.UploadDeviceSelfTests) api.GET("/device/:wwn/details", handler.GetDeviceDetails) //used by Details api.POST("/device/:wwn/archive", handler.ArchiveDevice) //used by UI to archive device api.POST("/device/:wwn/unarchive", handler.UnarchiveDevice) //used by UI to unarchive device + api.POST("/device/:wwn/mute", handler.MuteDevice) //used by UI to mute device + api.POST("/device/:wwn/unmute", handler.UnmuteDevice) //used by UI to unmute device api.DELETE("/device/:wwn", handler.DeleteDevice) //used by UI to delete device api.GET("/settings", handler.GetSettings) //used to get settings @@ -83,5 +107,21 @@ func (ae *AppEngine) Start() error { r := ae.Setup(ae.Logger) + // Load initial metrics data asynchronously at startup (if metrics enabled) + if ae.Config.GetBool("web.metrics.enabled") && ae.MetricsCollector != nil { + go func() { + deviceRepo, err := database.NewScrutinyRepository(ae.Config, ae.Logger) + if err != nil { + ae.Logger.Errorln("Failed to create repository for loading metrics:", err) + return + } + defer deviceRepo.Close() + + if err := ae.MetricsCollector.LoadInitialData(deviceRepo, context.Background()); err != nil { + ae.Logger.Errorln("Failed to load initial metrics data:", err) + } + }() + } + return r.Run(fmt.Sprintf("%s:%s", ae.Config.GetString("web.listen.host"), ae.Config.GetString("web.listen.port"))) } diff --git a/webapp/backend/pkg/web/server_test.go b/webapp/backend/pkg/web/server_test.go index bd2407c7..af576cce 100644 --- a/webapp/backend/pkg/web/server_test.go +++ b/webapp/backend/pkg/web/server_test.go @@ -106,6 +106,8 @@ func (suite *ServerTestSuite) TestHealthRoute() { fakeConfig.EXPECT().GetString("web.influxdb.bucket").Return("metrics").AnyTimes() fakeConfig.EXPECT().GetBool("web.influxdb.tls.insecure_skip_verify").Return(false).AnyTimes() fakeConfig.EXPECT().GetBool("web.influxdb.retention_policy").Return(false).AnyTimes() + + fakeConfig.EXPECT().GetIntSlice("failures.transient.ata").Return([]int{195}).AnyTimes() if _, isGithubActions := os.LookupEnv("GITHUB_ACTIONS"); isGithubActions { // when running test suite in github actions, we run an influxdb service as a sidecar. fakeConfig.EXPECT().GetString("web.influxdb.host").Return("influxdb").AnyTimes() @@ -149,6 +151,7 @@ func (suite *ServerTestSuite) TestRegisterDevicesRoute() { fakeConfig.EXPECT().GetString("web.influxdb.bucket").Return("metrics").AnyTimes() fakeConfig.EXPECT().GetBool("web.influxdb.tls.insecure_skip_verify").Return(false).AnyTimes() fakeConfig.EXPECT().GetBool("web.influxdb.retention_policy").Return(false).AnyTimes() + fakeConfig.EXPECT().GetIntSlice("failures.transient.ata").Return([]int{195}).AnyTimes() if _, isGithubActions := os.LookupEnv("GITHUB_ACTIONS"); isGithubActions { // when running test suite in github actions, we run an influxdb service as a sidecar. fakeConfig.EXPECT().GetString("web.influxdb.host").Return("influxdb").AnyTimes() @@ -191,8 +194,10 @@ func (suite *ServerTestSuite) TestUploadDeviceMetricsRoute() { fakeConfig.EXPECT().GetString("web.influxdb.org").Return("scrutiny").AnyTimes() fakeConfig.EXPECT().GetString("web.influxdb.bucket").Return("metrics").AnyTimes() fakeConfig.EXPECT().GetBool("user.metrics.repeat_notifications").Return(true).AnyTimes() + fakeConfig.EXPECT().GetBool("user.collector.retrieve_sct_temperature_history").Return(true).AnyTimes() fakeConfig.EXPECT().GetBool("web.influxdb.tls.insecure_skip_verify").Return(false).AnyTimes() fakeConfig.EXPECT().GetBool("web.influxdb.retention_policy").Return(false).AnyTimes() + fakeConfig.EXPECT().GetIntSlice("failures.transient.ata").Return([]int{195}).AnyTimes() if _, isGithubActions := os.LookupEnv("GITHUB_ACTIONS"); isGithubActions { // when running test suite in github actions, we run an influxdb service as a sidecar. fakeConfig.EXPECT().GetString("web.influxdb.host").Return("influxdb").AnyTimes() @@ -250,8 +255,10 @@ func (suite *ServerTestSuite) TestPopulateMultiple() { fakeConfig.EXPECT().GetString("web.influxdb.org").Return("scrutiny").AnyTimes() fakeConfig.EXPECT().GetString("web.influxdb.bucket").Return("metrics").AnyTimes() fakeConfig.EXPECT().GetBool("user.metrics.repeat_notifications").Return(true).AnyTimes() + fakeConfig.EXPECT().GetBool("user.collector.retrieve_sct_temperature_history").Return(true).AnyTimes() fakeConfig.EXPECT().GetBool("web.influxdb.tls.insecure_skip_verify").Return(false).AnyTimes() fakeConfig.EXPECT().GetBool("web.influxdb.retention_policy").Return(false).AnyTimes() + fakeConfig.EXPECT().GetIntSlice("failures.transient.ata").Return([]int{195}).AnyTimes() if _, isGithubActions := os.LookupEnv("GITHUB_ACTIONS"); isGithubActions { // when running test suite in github actions, we run an influxdb service as a sidecar. fakeConfig.EXPECT().GetString("web.influxdb.host").Return("influxdb").AnyTimes() @@ -351,6 +358,7 @@ func (suite *ServerTestSuite) TestSendTestNotificationRoute_WebhookFailure() { fakeConfig.EXPECT().GetString("web.influxdb.bucket").Return("metrics").AnyTimes() fakeConfig.EXPECT().GetBool("web.influxdb.tls.insecure_skip_verify").Return(false).AnyTimes() fakeConfig.EXPECT().GetBool("web.influxdb.retention_policy").Return(false).AnyTimes() + fakeConfig.EXPECT().GetIntSlice("failures.transient.ata").Return([]int{195}).AnyTimes() fakeConfig.EXPECT().GetStringSlice("notify.urls").AnyTimes().Return([]string{"https://unroutable.domain.example.asdfghj"}) fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.notify_level", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsNotifyLevelFail)) fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.status_filter_attributes", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsStatusFilterAttributesAll)) @@ -397,6 +405,7 @@ func (suite *ServerTestSuite) TestSendTestNotificationRoute_ScriptFailure() { fakeConfig.EXPECT().GetString("web.influxdb.bucket").Return("metrics").AnyTimes() fakeConfig.EXPECT().GetBool("web.influxdb.tls.insecure_skip_verify").Return(false).AnyTimes() fakeConfig.EXPECT().GetBool("web.influxdb.retention_policy").Return(false).AnyTimes() + fakeConfig.EXPECT().GetIntSlice("failures.transient.ata").Return([]int{195}).AnyTimes() fakeConfig.EXPECT().GetStringSlice("notify.urls").AnyTimes().Return([]string{"script:///missing/path/on/disk"}) fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.notify_level", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsNotifyLevelFail)) fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.status_filter_attributes", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsStatusFilterAttributesAll)) @@ -443,6 +452,7 @@ func (suite *ServerTestSuite) TestSendTestNotificationRoute_ScriptSuccess() { fakeConfig.EXPECT().GetString("web.influxdb.bucket").Return("metrics").AnyTimes() fakeConfig.EXPECT().GetBool("web.influxdb.tls.insecure_skip_verify").Return(false).AnyTimes() fakeConfig.EXPECT().GetBool("web.influxdb.retention_policy").Return(false).AnyTimes() + fakeConfig.EXPECT().GetIntSlice("failures.transient.ata").Return([]int{195}).AnyTimes() fakeConfig.EXPECT().GetStringSlice("notify.urls").AnyTimes().Return([]string{"script:///usr/bin/env"}) fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.notify_level", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsNotifyLevelFail)) fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.status_filter_attributes", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsStatusFilterAttributesAll)) @@ -489,6 +499,7 @@ func (suite *ServerTestSuite) TestSendTestNotificationRoute_ShoutrrrFailure() { fakeConfig.EXPECT().GetString("web.influxdb.bucket").Return("metrics").AnyTimes() fakeConfig.EXPECT().GetBool("web.influxdb.tls.insecure_skip_verify").Return(false).AnyTimes() fakeConfig.EXPECT().GetBool("web.influxdb.retention_policy").Return(false).AnyTimes() + fakeConfig.EXPECT().GetIntSlice("failures.transient.ata").Return([]int{195}).AnyTimes() fakeConfig.EXPECT().GetStringSlice("notify.urls").AnyTimes().Return([]string{"discord://invalidtoken@channel"}) fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.notify_level", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsNotifyLevelFail)) fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.status_filter_attributes", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsStatusFilterAttributesAll)) @@ -533,8 +544,10 @@ func (suite *ServerTestSuite) TestGetDevicesSummaryRoute_Nvme() { fakeConfig.EXPECT().GetString("web.influxdb.org").Return("scrutiny").AnyTimes() fakeConfig.EXPECT().GetString("web.influxdb.bucket").Return("metrics").AnyTimes() fakeConfig.EXPECT().GetBool("user.metrics.repeat_notifications").Return(true).AnyTimes() + fakeConfig.EXPECT().GetBool("user.collector.retrieve_sct_temperature_history").Return(true).AnyTimes() fakeConfig.EXPECT().GetBool("web.influxdb.tls.insecure_skip_verify").Return(false).AnyTimes() fakeConfig.EXPECT().GetBool("web.influxdb.retention_policy").Return(false).AnyTimes() + fakeConfig.EXPECT().GetIntSlice("failures.transient.ata").Return([]int{195}).AnyTimes() fakeConfig.EXPECT().GetStringSlice("notify.urls").AnyTimes().Return([]string{}) fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.notify_level", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsNotifyLevelFail)) fakeConfig.EXPECT().GetInt(fmt.Sprintf("%s.metrics.status_filter_attributes", config.DB_USER_SETTINGS_SUBKEY)).AnyTimes().Return(int(pkg.MetricsStatusFilterAttributesAll)) diff --git a/webapp/frontend/src/app/core/config/app.config.ts b/webapp/frontend/src/app/core/config/app.config.ts index c5a1f6f1..e8069912 100644 --- a/webapp/frontend/src/app/core/config/app.config.ts +++ b/webapp/frontend/src/app/core/config/app.config.ts @@ -54,6 +54,10 @@ export interface AppConfig { line_stroke?: LineStroke; // Settings from Scrutiny API + + collector?: { + retrieve_sct_temperature_history?: boolean + } metrics?: { notify_level?: MetricsNotifyLevel @@ -84,6 +88,10 @@ export const appConfig: AppConfig = { powered_on_hours_unit: 'humanize', line_stroke: 'smooth', + + collector: { + retrieve_sct_temperature_history : true, + }, metrics: { notify_level: MetricsNotifyLevel.Fail, diff --git a/webapp/frontend/src/app/core/models/device-model.ts b/webapp/frontend/src/app/core/models/device-model.ts index bddb776c..9f63d474 100644 --- a/webapp/frontend/src/app/core/models/device-model.ts +++ b/webapp/frontend/src/app/core/models/device-model.ts @@ -1,6 +1,7 @@ // maps to webapp/backend/pkg/models/device.go export interface DeviceModel { archived?: boolean; + muted: boolean; wwn: string; device_name?: string; device_uuid?: string; diff --git a/webapp/frontend/src/app/layout/common/dashboard-device/dashboard-device.component.html b/webapp/frontend/src/app/layout/common/dashboard-device/dashboard-device.component.html index 01ffb5f8..91e05909 100644 --- a/webapp/frontend/src/app/layout/common/dashboard-device/dashboard-device.component.html +++ b/webapp/frontend/src/app/layout/common/dashboard-device/dashboard-device.component.html @@ -17,7 +17,11 @@
{{deviceSummary.device | deviceTitle:config.dashboard_display}} + class="font-bold text-md text-secondary uppercase tracking-wider">{{deviceSummary.device | deviceTitle:config.dashboard_display}}
Last Updated on {{deviceSummary.smart.collector_date | date:'MMMM dd, yyyy - HH:mm' }}
diff --git a/webapp/frontend/src/app/layout/common/dashboard-device/dashboard-device.component.scss b/webapp/frontend/src/app/layout/common/dashboard-device/dashboard-device.component.scss index 60e0fcd2..cc0d4023 100644 --- a/webapp/frontend/src/app/layout/common/dashboard-device/dashboard-device.component.scss +++ b/webapp/frontend/src/app/layout/common/dashboard-device/dashboard-device.component.scss @@ -1,3 +1,21 @@ .text-disabled{ opacity: 0.8; } + +.mat-icon.muted-icon { + /* Treo is messing with the selector a little bit, so we have to assert !importance here */ + display: inline-block !important; + width: 16px; + height: 16px; + min-width: 16px; + min-height: 16px; + + // This gives a slightly better alignment on default zoom. + // Need to figure out a better way. See comment in app/modules/detail/detail.component.scss + vertical-align: middle; +} + +.mat-icon.muted-icon { + width: 16px; + height: 16px; +} diff --git a/webapp/frontend/src/app/layout/common/dashboard-settings/dashboard-settings.component.html b/webapp/frontend/src/app/layout/common/dashboard-settings/dashboard-settings.component.html index 970044cd..8e2590d5 100644 --- a/webapp/frontend/src/app/layout/common/dashboard-settings/dashboard-settings.component.html +++ b/webapp/frontend/src/app/layout/common/dashboard-settings/dashboard-settings.component.html @@ -102,6 +102,16 @@

Scrutiny Settings

+ +
+ + Retrieve SCT Temperature History + + Enabled + Disabled + + +
diff --git a/webapp/frontend/src/app/layout/common/dashboard-settings/dashboard-settings.component.ts b/webapp/frontend/src/app/layout/common/dashboard-settings/dashboard-settings.component.ts index ec65157a..dd76565a 100644 --- a/webapp/frontend/src/app/layout/common/dashboard-settings/dashboard-settings.component.ts +++ b/webapp/frontend/src/app/layout/common/dashboard-settings/dashboard-settings.component.ts @@ -28,6 +28,7 @@ export class DashboardSettingsComponent implements OnInit { poweredOnHoursUnit: string; lineStroke: string; theme: string; + retrieveSCTTemperatureHistory: boolean; statusThreshold: number; statusFilterAttributes: number; repeatNotifications: boolean; @@ -57,6 +58,8 @@ export class DashboardSettingsComponent implements OnInit { this.lineStroke = config.line_stroke; this.theme = config.theme; + this.retrieveSCTTemperatureHistory = config.collector.retrieve_sct_temperature_history; + this.statusFilterAttributes = config.metrics.status_filter_attributes; this.statusThreshold = config.metrics.status_threshold; this.repeatNotifications = config.metrics.repeat_notifications; @@ -74,6 +77,9 @@ export class DashboardSettingsComponent implements OnInit { powered_on_hours_unit: this.poweredOnHoursUnit as DevicePoweredOnUnit, line_stroke: this.lineStroke as LineStroke, theme: this.theme as Theme, + collector: { + retrieve_sct_temperature_history: this.retrieveSCTTemperatureHistory + }, metrics: { status_filter_attributes: this.statusFilterAttributes as MetricsStatusFilterAttributes, status_threshold: this.statusThreshold as MetricsStatusThreshold, diff --git a/webapp/frontend/src/app/layout/common/detail-settings/detail-settings.component.html b/webapp/frontend/src/app/layout/common/detail-settings/detail-settings.component.html index 649148d8..fe597d36 100644 --- a/webapp/frontend/src/app/layout/common/detail-settings/detail-settings.component.html +++ b/webapp/frontend/src/app/layout/common/detail-settings/detail-settings.component.html @@ -15,9 +15,9 @@

Scrutiny Settings

Notifications - - Enabled - Disabled + + Enabled + Disabled
@@ -26,5 +26,5 @@

Scrutiny Settings

- + diff --git a/webapp/frontend/src/app/layout/common/detail-settings/detail-settings.component.ts b/webapp/frontend/src/app/layout/common/detail-settings/detail-settings.component.ts index c3542c74..27f9e0b8 100644 --- a/webapp/frontend/src/app/layout/common/detail-settings/detail-settings.component.ts +++ b/webapp/frontend/src/app/layout/common/detail-settings/detail-settings.component.ts @@ -1,4 +1,5 @@ -import { Component, OnInit } from '@angular/core'; +import { Component, OnInit, Inject } from '@angular/core'; +import { MAT_DIALOG_DATA } from '@angular/material/dialog'; @Component({ selector: 'app-detail-settings', @@ -7,9 +8,14 @@ import { Component, OnInit } from '@angular/core'; }) export class DetailSettingsComponent implements OnInit { - constructor() { } + muted: boolean; - ngOnInit(): void { + constructor( + @Inject(MAT_DIALOG_DATA) public data: { curMuted: boolean } + ) { + this.muted = data.curMuted; } + ngOnInit(): void { + } } diff --git a/webapp/frontend/src/app/modules/dashboard/dashboard.component.html b/webapp/frontend/src/app/modules/dashboard/dashboard.component.html index 872387fc..51b6d547 100644 --- a/webapp/frontend/src/app/modules/dashboard/dashboard.component.html +++ b/webapp/frontend/src/app/modules/dashboard/dashboard.component.html @@ -96,6 +96,7 @@

{{ hostId.key }}

+ diff --git a/webapp/frontend/src/app/modules/dashboard/dashboard.component.ts b/webapp/frontend/src/app/modules/dashboard/dashboard.component.ts index bdd166ce..febf633a 100644 --- a/webapp/frontend/src/app/modules/dashboard/dashboard.component.ts +++ b/webapp/frontend/src/app/modules/dashboard/dashboard.component.ts @@ -269,11 +269,11 @@ export class DashboardComponent implements OnInit, AfterViewInit, OnDestroy } /* - + DURATION_KEY_DAY = "day" DURATION_KEY_WEEK = "week" - DURATION_KEY_MONTH = "month" - DURATION_KEY_YEAR = "year" - DURATION_KEY_FOREVER = "forever" + DURATION_KEY_MONTH = "month" + DURATION_KEY_YEAR = "year" + DURATION_KEY_FOREVER = "forever" */ changeSummaryTempDuration(durationKey: string): void { diff --git a/webapp/frontend/src/app/modules/detail/detail.component.html b/webapp/frontend/src/app/modules/detail/detail.component.html index 13b19c29..88408170 100644 --- a/webapp/frontend/src/app/modules/detail/detail.component.html +++ b/webapp/frontend/src/app/modules/detail/detail.component.html @@ -4,20 +4,25 @@
-

Drive Details - {{device | deviceTitle:config.dashboard_display}}

+

Drive Details - {{device | deviceTitle:config.dashboard_display}}

Dive into S.M.A.R.T data
- +