Skip to content

Commit 6d52356

Browse files
AnalogJAnalogJ
authored
Merge branch 'master' into fix_494
2 parents 5789d7a + affe05e commit 6d52356

File tree

13 files changed

+173
-657
lines changed

13 files changed

+173
-657
lines changed

‎.github/workflows/ci.yaml‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ jobs:
1313
run: |
1414
make binary-frontend-test-coverage
1515
- name: Upload coverage
16-
uses: actions/upload-artifact@v3
16+
uses: actions/upload-artifact@v4
1717
with:
1818
name: coverage
1919
path: ${{ github.workspace }}/webapp/frontend/coverage/lcov.info
@@ -49,7 +49,7 @@ jobs:
4949
run: |
5050
make binary-clean binary-test-coverage
5151
- name: Upload coverage
52-
uses: actions/upload-artifact@v3
52+
uses: actions/upload-artifact@v4
5353
with:
5454
name: coverage
5555
path: ${{ github.workspace }}/coverage.txt
@@ -64,7 +64,7 @@ jobs:
6464
- name: Checkout
6565
uses: actions/checkout@v2
6666
- name: Download coverage reports
67-
uses: actions/download-artifact@v3
67+
uses: actions/download-artifact@v4
6868
with:
6969
name: coverage
7070
- name: Upload coverage reports
@@ -106,7 +106,7 @@ jobs:
106106
run: |
107107
make binary-clean binary-all
108108
- name: Archive
109-
uses: actions/upload-artifact@v2
109+
uses: actions/upload-artifact@v4
110110
with:
111111
name: binaries.zip
112112
path: |

‎.github/workflows/release.yaml‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@ jobs:
6161
with:
6262
version_metadata_path: ${{ github.event.inputs.version_metadata_path }}
6363
- name: Upload workspace
64-
uses: actions/upload-artifact@v3
64+
uses: actions/upload-artifact@v4
6565
with:
6666
name: workspace
6767
path: ${{ github.workspace }}/**/*
@@ -91,7 +91,7 @@ jobs:
9191
- { on: windows-latest, goos: windows, goarch: arm64 }
9292
steps:
9393
- name: Download workspace
94-
uses: actions/download-artifact@v3
94+
uses: actions/download-artifact@v4
9595
with:
9696
name: workspace
9797
- uses: actions/setup-go@v3
@@ -101,7 +101,7 @@ jobs:
101101
run: |
102102
make binary-clean binary-all
103103
- name: Archive
104-
uses: actions/upload-artifact@v2
104+
uses: actions/upload-artifact@v4
105105
with:
106106
name: binaries.zip
107107
path: |
@@ -114,11 +114,11 @@ jobs:
114114
runs-on: ubuntu-latest
115115
steps:
116116
- name: Download workspace
117-
uses: actions/download-artifact@v3
117+
uses: actions/download-artifact@v4
118118
with:
119119
name: workspace
120120
- name: Download binaries
121-
uses: actions/download-artifact@v3
121+
uses: actions/download-artifact@v4
122122
with:
123123
name: binaries.zip
124124
- name: List

‎collector/pkg/collector/metrics.go‎

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -4,17 +4,19 @@ import (
44
"bytes"
55
"encoding/json"
66
"fmt"
7+
"net/url"
8+
"os"
9+
"os/exec"
10+
"strings"
11+
"time"
12+
713
"github.com/analogj/scrutiny/collector/pkg/common/shell"
814
"github.com/analogj/scrutiny/collector/pkg/config"
915
"github.com/analogj/scrutiny/collector/pkg/detect"
1016
"github.com/analogj/scrutiny/collector/pkg/errors"
1117
"github.com/analogj/scrutiny/collector/pkg/models"
1218
"github.com/samber/lo"
1319
"github.com/sirupsen/logrus"
14-
"net/url"
15-
"os"
16-
"os/exec"
17-
"strings"
1820
)
1921

2022
type MetricsCollector struct {
@@ -90,8 +92,9 @@ func (mc *MetricsCollector) Run() error {
9092
//go mc.Collect(&wg, device.WWN, device.DeviceName, device.DeviceType)
9193
mc.Collect(device.WWN, device.DeviceName, device.DeviceType)
9294

93-
// TODO: we may need to sleep for between each call to smartctl -a
94-
//time.Sleep(30 * time.Millisecond)
95+
if mc.config.GetInt("commands.metrics_smartctl_wait") > 0 {
96+
time.Sleep(time.Duration(mc.config.GetInt("commands.metrics_smartctl_wait")) * time.Second)
97+
}
9598
}
9699

97100
//mc.logger.Infoln("Main: Waiting for workers to finish")
@@ -113,7 +116,7 @@ func (mc *MetricsCollector) Validate() error {
113116
return nil
114117
}
115118

116-
//func (mc *MetricsCollector) Collect(wg *sync.WaitGroup, deviceWWN string, deviceName string, deviceType string) {
119+
// func (mc *MetricsCollector) Collect(wg *sync.WaitGroup, deviceWWN string, deviceName string, deviceType string) {
117120
func (mc *MetricsCollector) Collect(deviceWWN string, deviceName string, deviceType string) {
118121
//defer wg.Done()
119122
if len(deviceWWN) == 0 {

‎collector/pkg/config/config.go‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@ import (
2020
type configuration struct {
2121
*viper.Viper
2222

23-
deviceOverrides []models.ScanOverride
23+
deviceOverrides []models.ScanOverride
2424
}
2525

2626
//Viper uses the following precedence order. Each item takes precedence over the item below it:
@@ -47,9 +47,12 @@ func (c *configuration) Init() error {
4747
c.SetDefault("commands.metrics_scan_args", "--scan --json")
4848
c.SetDefault("commands.metrics_info_args", "--info --json")
4949
c.SetDefault("commands.metrics_smart_args", "--xall --json")
50+
c.SetDefault("commands.metrics_smartctl_wait", 0)
5051

5152
//c.SetDefault("collect.short.command", "-a -o on -S on")
5253

54+
c.SetDefault("allow_listed_devices", []string{})
55+
5356
//if you want to load a non-standard location system config file (~/drawbridge.yml), use ReadConfig
5457
c.SetConfigType("yaml")
5558
//c.SetConfigName("drawbridge")
@@ -186,3 +189,18 @@ func (c *configuration) GetCommandMetricsSmartArgs(deviceName string) string {
186189
}
187190
return c.GetString("commands.metrics_smart_args")
188191
}
192+
193+
func (c *configuration) IsAllowlistedDevice(deviceName string) bool {
194+
allowList := c.GetStringSlice("allow_listed_devices")
195+
if len(allowList) == 0 {
196+
return true
197+
}
198+
199+
for _, item := range allowList {
200+
if item == deviceName {
201+
return true
202+
}
203+
}
204+
205+
return false
206+
}

‎collector/pkg/config/config_test.go‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -144,3 +144,29 @@ func TestConfiguration_OverrideDeviceCommands_MetricsInfoArgs(t *testing.T) {
144144
require.Equal(t, "--info --json", testConfig.GetCommandMetricsInfoArgs("/dev/sdb"))
145145
//require.Equal(t, []models.ScanOverride{{Device: "/dev/sda", DeviceType: nil, Commands: {MetricsInfoArgs: "--info --json -T "}}}, scanOverrides)
146146
}
147+
148+
func TestConfiguration_DeviceAllowList(t *testing.T) {
149+
t.Parallel()
150+
151+
t.Run("present", func(t *testing.T) {
152+
testConfig, err := config.Create()
153+
require.NoError(t, err)
154+
155+
require.NoError(t, testConfig.ReadConfig(path.Join("testdata", "allow_listed_devices_present.yaml")))
156+
157+
require.True(t, testConfig.IsAllowlistedDevice("/dev/sda"), "/dev/sda should be allow listed")
158+
require.False(t, testConfig.IsAllowlistedDevice("/dev/sdc"), "/dev/sda should not be allow listed")
159+
})
160+
161+
t.Run("missing", func(t *testing.T) {
162+
testConfig, err := config.Create()
163+
require.NoError(t, err)
164+
165+
// Really just any other config where the key is full missing
166+
require.NoError(t, testConfig.ReadConfig(path.Join("testdata", "override_device_commands.yaml")))
167+
168+
// Anything should be allow listed if the key isnt there
169+
require.True(t, testConfig.IsAllowlistedDevice("/dev/sda"), "/dev/sda should be allow listed")
170+
require.True(t, testConfig.IsAllowlistedDevice("/dev/sdc"), "/dev/sda should be allow listed")
171+
})
172+
}

‎collector/pkg/config/interface.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,4 +25,6 @@ type Interface interface {
2525
GetDeviceOverrides() []models.ScanOverride
2626
GetCommandMetricsInfoArgs(deviceName string) string
2727
GetCommandMetricsSmartArgs(deviceName string) string
28+
29+
IsAllowlistedDevice(deviceName string) bool
2830
}

‎collector/pkg/config/mock/mock_config.go‎

Lines changed: 14 additions & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎collector/pkg/config/testdata/allow_listed_devices_present.yaml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
allow_listed_devices:
2+
- /dev/sda
3+
- /dev/sdb

‎collector/pkg/detect/detect.go‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -124,6 +124,11 @@ func (d *Detect) TransformDetectedDevices(detectedDeviceConns models.Scan) []mod
124124

125125
deviceFile := strings.ToLower(scannedDevice.Name)
126126

127+
// If the user has defined a device allow list, and this device isnt there, then ignore it
128+
if !d.Config.IsAllowlistedDevice(deviceFile) {
129+
continue
130+
}
131+
127132
detectedDevice := models.Device{
128133
HostId: d.Config.GetString("host.id"),
129134
DeviceType: scannedDevice.Type,

‎collector/pkg/detect/detect_test.go‎

Lines changed: 48 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ func TestDetect_SmartctlScan(t *testing.T) {
2424
fakeConfig.EXPECT().GetDeviceOverrides().AnyTimes().Return([]models.ScanOverride{})
2525
fakeConfig.EXPECT().GetString("commands.metrics_smartctl_bin").AnyTimes().Return("smartctl")
2626
fakeConfig.EXPECT().GetString("commands.metrics_scan_args").AnyTimes().Return("--scan --json")
27+
fakeConfig.EXPECT().IsAllowlistedDevice(gomock.Any()).AnyTimes().Return(true)
2728

2829
fakeShell := mock_shell.NewMockInterface(mockCtrl)
2930
testScanResults, err := os.ReadFile("testdata/smartctl_scan_simple.json")
@@ -53,6 +54,7 @@ func TestDetect_SmartctlScan_Megaraid(t *testing.T) {
5354
fakeConfig.EXPECT().GetDeviceOverrides().AnyTimes().Return([]models.ScanOverride{})
5455
fakeConfig.EXPECT().GetString("commands.metrics_smartctl_bin").AnyTimes().Return("smartctl")
5556
fakeConfig.EXPECT().GetString("commands.metrics_scan_args").AnyTimes().Return("--scan --json")
57+
fakeConfig.EXPECT().IsAllowlistedDevice(gomock.Any()).AnyTimes().Return(true)
5658

5759
fakeShell := mock_shell.NewMockInterface(mockCtrl)
5860
testScanResults, err := os.ReadFile("testdata/smartctl_scan_megaraid.json")
@@ -85,6 +87,7 @@ func TestDetect_SmartctlScan_Nvme(t *testing.T) {
8587
fakeConfig.EXPECT().GetDeviceOverrides().AnyTimes().Return([]models.ScanOverride{})
8688
fakeConfig.EXPECT().GetString("commands.metrics_smartctl_bin").AnyTimes().Return("smartctl")
8789
fakeConfig.EXPECT().GetString("commands.metrics_scan_args").AnyTimes().Return("--scan --json")
90+
fakeConfig.EXPECT().IsAllowlistedDevice(gomock.Any()).AnyTimes().Return(true)
8891

8992
fakeShell := mock_shell.NewMockInterface(mockCtrl)
9093
testScanResults, err := os.ReadFile("testdata/smartctl_scan_nvme.json")
@@ -116,6 +119,7 @@ func TestDetect_TransformDetectedDevices_Empty(t *testing.T) {
116119
fakeConfig.EXPECT().GetDeviceOverrides().AnyTimes().Return([]models.ScanOverride{})
117120
fakeConfig.EXPECT().GetString("commands.metrics_smartctl_bin").AnyTimes().Return("smartctl")
118121
fakeConfig.EXPECT().GetString("commands.metrics_scan_args").AnyTimes().Return("--scan --json")
122+
fakeConfig.EXPECT().IsAllowlistedDevice(gomock.Any()).AnyTimes().Return(true)
119123

120124
detectedDevices := models.Scan{
121125
Devices: []models.ScanDevice{
@@ -149,6 +153,7 @@ func TestDetect_TransformDetectedDevices_Ignore(t *testing.T) {
149153
fakeConfig.EXPECT().GetDeviceOverrides().AnyTimes().Return([]models.ScanOverride{{Device: "/dev/sda", DeviceType: nil, Ignore: true}})
150154
fakeConfig.EXPECT().GetString("commands.metrics_smartctl_bin").AnyTimes().Return("smartctl")
151155
fakeConfig.EXPECT().GetString("commands.metrics_scan_args").AnyTimes().Return("--scan --json")
156+
fakeConfig.EXPECT().IsAllowlistedDevice(gomock.Any()).AnyTimes().Return(true)
152157

153158
detectedDevices := models.Scan{
154159
Devices: []models.ScanDevice{
@@ -180,6 +185,7 @@ func TestDetect_TransformDetectedDevices_Raid(t *testing.T) {
180185
fakeConfig.EXPECT().GetString("host.id").AnyTimes().Return("")
181186
fakeConfig.EXPECT().GetString("commands.metrics_smartctl_bin").AnyTimes().Return("smartctl")
182187
fakeConfig.EXPECT().GetString("commands.metrics_scan_args").AnyTimes().Return("--scan --json")
188+
fakeConfig.EXPECT().IsAllowlistedDevice(gomock.Any()).AnyTimes().Return(true)
183189
fakeConfig.EXPECT().GetDeviceOverrides().AnyTimes().Return([]models.ScanOverride{
184190
{
185191
Device: "/dev/bus/0",
@@ -223,6 +229,7 @@ func TestDetect_TransformDetectedDevices_Simple(t *testing.T) {
223229
fakeConfig.EXPECT().GetString("commands.metrics_smartctl_bin").AnyTimes().Return("smartctl")
224230
fakeConfig.EXPECT().GetString("commands.metrics_scan_args").AnyTimes().Return("--scan --json")
225231
fakeConfig.EXPECT().GetDeviceOverrides().AnyTimes().Return([]models.ScanOverride{{Device: "/dev/sda", DeviceType: []string{"sat+megaraid"}}})
232+
fakeConfig.EXPECT().IsAllowlistedDevice(gomock.Any()).AnyTimes().Return(true)
226233
detectedDevices := models.Scan{
227234
Devices: []models.ScanDevice{
228235
{
@@ -256,6 +263,7 @@ func TestDetect_TransformDetectedDevices_WithoutDeviceTypeOverride(t *testing.T)
256263
fakeConfig.EXPECT().GetString("commands.metrics_smartctl_bin").AnyTimes().Return("smartctl")
257264
fakeConfig.EXPECT().GetString("commands.metrics_scan_args").AnyTimes().Return("--scan --json")
258265
fakeConfig.EXPECT().GetDeviceOverrides().AnyTimes().Return([]models.ScanOverride{{Device: "/dev/sda"}})
266+
fakeConfig.EXPECT().IsAllowlistedDevice(gomock.Any()).AnyTimes().Return(true)
259267
detectedDevices := models.Scan{
260268
Devices: []models.ScanDevice{
261269
{
@@ -302,6 +310,46 @@ func TestDetect_TransformDetectedDevices_WhenDeviceNotDetected(t *testing.T) {
302310
require.Equal(t, "ata", transformedDevices[0].DeviceType)
303311
}
304312

313+
func TestDetect_TransformDetectedDevices_AllowListFilters(t *testing.T) {
314+
mockCtrl := gomock.NewController(t)
315+
defer mockCtrl.Finish()
316+
317+
fakeConfig := mock_config.NewMockInterface(mockCtrl)
318+
fakeConfig.EXPECT().GetString("host.id").AnyTimes().Return("")
319+
fakeConfig.EXPECT().GetString("commands.metrics_smartctl_bin").AnyTimes().Return("smartctl")
320+
fakeConfig.EXPECT().GetString("commands.metrics_scan_args").AnyTimes().Return("--scan --json")
321+
fakeConfig.EXPECT().GetDeviceOverrides().AnyTimes().Return([]models.ScanOverride{{Device: "/dev/sda", DeviceType: []string{"sat+megaraid"}}})
322+
fakeConfig.EXPECT().IsAllowlistedDevice("/dev/sda").Return(true)
323+
fakeConfig.EXPECT().IsAllowlistedDevice("/dev/sdb").Return(false)
324+
detectedDevices := models.Scan{
325+
Devices: []models.ScanDevice{
326+
{
327+
Name: "/dev/sda",
328+
InfoName: "/dev/sda",
329+
Protocol: "ata",
330+
Type: "ata",
331+
},
332+
{
333+
Name: "/dev/sdb",
334+
InfoName: "/dev/sdb",
335+
Protocol: "ata",
336+
Type: "ata",
337+
},
338+
},
339+
}
340+
341+
d := detect.Detect{
342+
Config: fakeConfig,
343+
}
344+
345+
// test
346+
transformedDevices := d.TransformDetectedDevices(detectedDevices)
347+
348+
// assert
349+
require.Equal(t, 1, len(transformedDevices))
350+
require.Equal(t, "sda", transformedDevices[0].DeviceName)
351+
}
352+
305353
func TestDetect_SmartCtlInfo(t *testing.T) {
306354
t.Run("should report nvme info", func(t *testing.T) {
307355
ctrl := gomock.NewController(t)

0 commit comments

Comments
 (0)