Skip to content

Commit 997aa73

Browse files
jeffmahoneyjeffmahoney
authored
Merge pull request SUSE#63 from jeffmahoney/sensor-base-0.6.7
Sensor base 0.6.7
2 parents 5aa226e + ef182c6 commit 997aa73

File tree

19 files changed

+3128
-157
lines changed

19 files changed

+3128
-157
lines changed

‎.github/workflows/go.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ jobs:
4141
run: |
4242
go get -v -t -d ./...
4343
sudo apt-get update
44-
sudo apt-get install mingw-w64-x86-64-dev gcc-mingw-w64-x86-64 gcc-mingw-w64 libsystemd-dev clang-12 llvm libelf-dev git linux-tools-common make libzstd-dev
44+
sudo apt-get install mingw-w64-x86-64-dev gcc-mingw-w64-x86-64 gcc-mingw-w64 libsystemd-dev clang-12 llvm libelf-dev git linux-tools-common make libzstd-dev linux-tools-common
4545
4646
- name: Use Node.js v16
4747
uses: actions/setup-node@v1

‎.github/workflows/linux.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@ jobs:
2525
run: |
2626
go get -v -t -d ./...
2727
sudo apt-get update
28-
sudo apt-get install libsystemd-dev clang-12 llvm libelf-dev git linux-tools-common make libzstd-dev
28+
sudo apt-get install libsystemd-dev clang-12 llvm libelf-dev git linux-tools-common make libzstd-dev linux-tools-common
2929
3030
- name: Build GUI-less binary
3131
run: |

‎Makefile‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ LIBBPF_DIR := $(LIBBPFGO_DIR)/libbpf
1616
LIBBPF_OUTPUT := $(LIBBPFGO_DIR)/output
1717
LIBBPF_LIB := $(LIBBPF_OUTPUT)/libbpf.a
1818
GIT := git
19+
BPFTOOL := bpftool
1920
EXTRA_TAGS += linuxbpf libbpfgo_static
2021
else
2122
$(error Cannot build BPF objects without clang installed. Install clang or build with BUILD_LIBBPFGO=0.)
@@ -76,7 +77,11 @@ $(LIBBPFGO_DIR): always-check
7677
echo "INFO: updating submodule 'libbpfgo'"
7778
$(GIT) submodule update --init --recursive $@
7879

79-
$(LIBBPF_LIB): $(LIBBPFGO_DIR)
80+
$(LIBBPFGO_DIR)/vmlinux.h: $(LIBBFGO_DIR)
81+
mkdir -p $(LIBBPF_OUTPUT)
82+
$(BPFTOOL) btf dump file /sys/kernel/btf/vmlinux format c > ./third_party/libbpfgo/output/vmlinux.h
83+
84+
$(LIBBPF_LIB): $(LIBBPFGO_DIR) $(LIBBPFGO_DIR)/vmlinux.h
8085
make -C $(LIBBPFGO_DIR) libbpfgo-static
8186

8287
%.bpf.o: %.bpf.c $(LIBBPF_LIB)

‎artifacts/definitions/LogScale/Events/Clients.yaml‎

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,76 @@
1+
name: LogScale.Events.Clients
2+
description: |
3+
This server side event monitoring artifact will watch a selection of client
4+
monitoring artifacts for new events and push those to a LogScale (formerly
5+
Humio) ingestion endpoint
6+
7+
NOTE: You must ensure you are collecting these artifacts from the
8+
clients by adding them to the "Client Events" GUI.
9+
10+
type: SERVER_EVENT
11+
12+
parameters:
13+
- name: ingestApiBase
14+
description: API Base Url for LogScale server
15+
type: string
16+
default: https://cloud.community.humio.com/api
17+
- name: ingestToken
18+
description: Ingest token for API
19+
type: string
20+
- name: tagFields
21+
description: Comma-separated list of field names to use as tags in the message; Can be renamed with <oldname>=<newname>.
22+
default:
23+
type: string
24+
- name: numThreads
25+
description: Number of threads to start up to post events
26+
type: int
27+
default: 1
28+
- name: httpTimeout
29+
description: Timeout (in seconds) for http connection attempts
30+
type: int
31+
default: 10
32+
- name: batchingTimeoutMs
33+
description: Timeout (in ms) to batch events prior to sending
34+
type: int
35+
default: 30000
36+
- name: eventBatchSize
37+
description: Count of events to batch prior to sending
38+
type: int
39+
default: 2000
40+
- name: statsInterval
41+
description: Interval to post statistics to log (in seconds, 0 to disable)
42+
type: int
43+
default: 600
44+
- name: debug
45+
description: Enable verbose logging
46+
type: bool
47+
default: false
48+
- name: Artifacts
49+
type: artifactset
50+
artifact_type: CLIENT_EVENT
51+
description: Client artifacts to monitor
52+
53+
sources:
54+
- query: |
55+
LET artifacts_to_watch = SELECT Artifact FROM Artifacts
56+
WHERE log(message="Uploading artifact " + Artifact + " to LogScale")
57+
58+
LET events = SELECT * FROM foreach(
59+
row=artifacts_to_watch,
60+
async=TRUE, // Required for event queries in foreach()
61+
query={
62+
SELECT *, Artifact, timestamp(epoch=now()) AS timestamp
63+
FROM watch_monitoring(artifact=Artifact)
64+
})
65+
66+
SELECT * FROM logscale_upload(
67+
query=events,
68+
apibaseurl=ingestApiBase,
69+
ingest_token=ingestToken,
70+
threads=numThreads,
71+
tag_fields=split(string=tagFields, sep=","),
72+
batching_timeout_ms=batchingTimeoutMs,
73+
event_batch_size=eventBatchSize,
74+
http_timeout=httpTimeout,
75+
debug=debug,
76+
stats_interval=statsInterval)

‎artifacts/definitions/LogScale/Flows/Upload.yaml‎

Lines changed: 81 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,81 @@
1+
name: LogScale.Flows.Upload
2+
description: |
3+
This server side event monitoring artifact waits for new artifacts
4+
to be collected from endpoints and automatically posts those to a
5+
LogScale (formerly Humio) ingestion endpoint.
6+
7+
type: SERVER_EVENT
8+
9+
parameters:
10+
- name: ingestApiBase
11+
description: API Base Url for LogScale server
12+
type: string
13+
default: https://cloud.community.humio.com/api
14+
- name: ingestToken
15+
description: Ingest token for API
16+
type: string
17+
- name: tagFields
18+
description: Comma-separated list of field names to use as tags in the message; Can be renamed with <oldname>=<newname>.
19+
default:
20+
type: string
21+
- name: numThreads
22+
description: Number of threads to start up to post events
23+
type: int
24+
default: 1
25+
- name: httpTimeout
26+
description: Timeout (in seconds) for http connection attempts
27+
type: int
28+
default: 10
29+
- name: batchingTimeoutMs
30+
description: Timeout to batch events prior to sending
31+
type: int
32+
default: 30000
33+
- name: eventBatchSize
34+
description: Count of events to batch prior to sending
35+
type: int
36+
default: 2000
37+
- name: statsInterval
38+
description: Interval to post statistics to log (in seconds, 0 to disable)
39+
type: int
40+
default: 600
41+
- name: debug
42+
description: Enable verbose logging
43+
type: bool
44+
default: false
45+
- name: ArtifactNameRegex
46+
default: .
47+
type: regex
48+
description: Only upload these artifacts to elastic
49+
50+
sources:
51+
- query: |
52+
LET completions = SELECT * FROM watch_monitoring(
53+
artifact="System.Flow.Completion")
54+
WHERE Flow.artifacts_with_results =~ ArtifactNameRegex
55+
56+
LET documents = SELECT * FROM foreach(row=completions,
57+
query={
58+
SELECT * FROM foreach(
59+
row=Flow.artifacts_with_results,
60+
query={
61+
SELECT *, _value AS Artifact,
62+
timestamp(epoch=now()) AS timestamp,
63+
ClientId, Flow.session_id AS FlowId
64+
FROM source(
65+
client_id=ClientId,
66+
flow_id=Flow.session_id,
67+
artifact=_value)
68+
})
69+
})
70+
71+
SELECT * FROM logscale_upload(
72+
query=documents,
73+
apibaseurl=ingestApiBase,
74+
ingest_token=ingestToken,
75+
threads=numThreads,
76+
tag_fields=split(string=tagFields, sep=","),
77+
batching_timeout_ms=batchingTimeoutMs,
78+
event_batch_size=eventBatchSize,
79+
http_timeout=httpTimeout,
80+
debug=debug,
81+
stats_interval=statsInterval)

‎file_store/directory/buffer.go‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -204,6 +204,12 @@ func (self *FileBasedRingBuffer) _Truncate() {
204204
_, _ = self.fd.WriteAt(serialized, 0)
205205
}
206206

207+
func (self *FileBasedRingBuffer) PendingSize() int64 {
208+
self.mu.Lock()
209+
defer self.mu.Unlock()
210+
return self.header.WritePointer - self.header.ReadPointer
211+
}
212+
207213
func (self *FileBasedRingBuffer) Reset() {
208214
self.mu.Lock()
209215
defer self.mu.Unlock()

‎file_store/directory/listener.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -195,6 +195,10 @@ func (self *Listener) Close() {
195195
}
196196
}
197197

198+
func (self *Listener) FileBufferSize() int64 {
199+
return self.file_buffer.PendingSize()
200+
}
201+
198202
func (self *Listener) Debug() *ordereddict.Dict {
199203
self.mu.Lock()
200204
defer self.mu.Unlock()

0 commit comments

Comments
 (0)