diff --git a/hosts/glyph/services/alloy.nix b/hosts/glyph/services/alloy.nix new file mode 100644 index 00000000..9f77a165 --- /dev/null +++ b/hosts/glyph/services/alloy.nix @@ -0,0 +1,18 @@ +_: { + services.alloy.enable = true; + + environment.etc."alloy/config.alloy".text = '' + loki.source.journal "systemd" { + forward_to = [loki.write.local.receiver] + } + + loki.write "local" { + endpoint { + url = "http://localhost:3100/loki/api/v1/push" + } + external_labels = { + host = "glyph", + } + } + ''; +} diff --git a/hosts/glyph/services/default.nix b/hosts/glyph/services/default.nix index 59bc0e8b..30148950 100644 --- a/hosts/glyph/services/default.nix +++ b/hosts/glyph/services/default.nix @@ -4,12 +4,14 @@ ... }: { imports = [ + ./alloy.nix ./attic.nix ./db.nix ./avahi.nix ./dns.nix ./filebrowser.nix ./jellyfin.nix + ./loki.nix ./nfs.nix ./open-terminal.nix ./open-webui.nix @@ -21,7 +23,6 @@ users.users.mu.extraGroups = ["media"]; users.users.${config.services.transmission.user}.extraGroups = ["media"]; - services.cockpit.enable = true; services.openssh.enable = true; services.roon-server = { enable = true; diff --git a/hosts/glyph/services/loki.nix b/hosts/glyph/services/loki.nix new file mode 100644 index 00000000..0ed2d8fd --- /dev/null +++ b/hosts/glyph/services/loki.nix @@ -0,0 +1,44 @@ +_: { + services.loki = { + enable = true; + configuration = { + server.http_listen_port = 3100; + auth_enabled = false; + + common = { + path_prefix = "/var/lib/loki"; + replication_factor = 1; + ring.kvstore.store = "inmemory"; + }; + + schema_config.configs = [ + { + from = "2024-01-01"; + store = "tsdb"; + object_store = "filesystem"; + schema = "v13"; + index = { + prefix = "index_"; + period = "24h"; + }; + } + ]; + + storage_config.filesystem.directory = "/var/lib/loki/chunks"; + + limits_config = { + retention_period = "30d"; + ingestion_burst_size_mb = 16; + ingestion_rate_mb = 8; + }; + + compactor = { + working_directory = "/var/lib/loki/compactor"; + compaction_interval = "10m"; + retention_enabled = true; + retention_delete_delay = "2h"; + delete_request_store = "filesystem"; + }; + }; + }; +} diff --git a/hosts/spore/services/alloy.nix b/hosts/spore/services/alloy.nix new file mode 100644 index 00000000..bd288806 --- /dev/null +++ b/hosts/spore/services/alloy.nix @@ -0,0 +1,18 @@ +_: { + services.alloy.enable = true; + + environment.etc."alloy/config.alloy".text = '' + loki.source.journal "systemd" { + forward_to = [loki.write.remote.receiver] + } + + loki.write "remote" { + endpoint { + url = "http://glyph.rove-duck.ts.net:3100/loki/api/v1/push" + } + external_labels = { + host = "spore", + } + } + ''; +} diff --git a/hosts/spore/services/dashboards/logs.json b/hosts/spore/services/dashboards/logs.json new file mode 100644 index 00000000..554e5e0f --- /dev/null +++ b/hosts/spore/services/dashboards/logs.json @@ -0,0 +1,224 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { "type": "grafana", "uid": "-- Grafana --" }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": false, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "links": [], + "panels": [ + { + "datasource": { "type": "loki", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "fixed", "fixedColor": "text" } + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 8, "x": 0, "y": 0 }, + "id": 1, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["count"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "title": "Total Log Lines", + "type": "stat", + "targets": [ + { + "expr": "sum(count_over_time({host=~\"$host\"} |~ \"$search\" [$__auto]))", + "refId": "A" + } + ] + }, + { + "datasource": { "type": "loki", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "fixed", "fixedColor": "red" } + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 8, "x": 8, "y": 0 }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["count"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "title": "Error Lines", + "type": "stat", + "targets": [ + { + "expr": "sum(count_over_time({host=~\"$host\"} |~ \"$search\" |~ \"(?i)error|fatal|panic\" [$__auto]))", + "refId": "A" + } + ] + }, + { + "datasource": { "type": "loki", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "fixed", "fixedColor": "yellow" } + }, + "overrides": [] + }, + "gridPos": { "h": 4, "w": 8, "x": 16, "y": 0 }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": ["count"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "title": "Warning Lines", + "type": "stat", + "targets": [ + { + "expr": "sum(count_over_time({host=~\"$host\"} |~ \"$search\" |~ \"(?i)warn\" [$__auto]))", + "refId": "A" + } + ] + }, + { + "datasource": { "type": "loki", "uid": "${datasource}" }, + "fieldConfig": { + "defaults": { + "color": { "mode": "palette-classic" }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 50, + "gradientMode": "none", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { "type": "linear" }, + "showPoints": "never", + "spanNulls": false, + "stacking": { "group": "A", "mode": "normal" }, + "thresholdsStyle": { "mode": "off" } + } + }, + "overrides": [] + }, + "gridPos": { "h": 6, "w": 24, "x": 0, "y": 4 }, + "id": 4, + "options": { + "legend": { "calcs": [], "displayMode": "list", "placement": "bottom" }, + "tooltip": { "mode": "multi", "sort": "desc" } + }, + "title": "Log Volume", + "type": "timeseries", + "targets": [ + { + "expr": "sum by (host) (count_over_time({host=~\"$host\"} |~ \"$search\" [$__auto]))", + "legendFormat": "{{host}}", + "refId": "A" + } + ] + }, + { + "datasource": { "type": "loki", "uid": "${datasource}" }, + "gridPos": { "h": 18, "w": 24, "x": 0, "y": 10 }, + "id": 5, + "options": { + "dedupStrategy": "none", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": true, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": true + }, + "title": "Logs", + "type": "logs", + "targets": [ + { + "expr": "{host=~\"$host\"} |~ \"$search\"", + "refId": "A" + } + ] + } + ], + "schemaVersion": 39, + "tags": ["logs", "loki"], + "templating": { + "list": [ + { + "current": {}, + "hide": 0, + "includeAll": true, + "label": "Host", + "multi": true, + "name": "host", + "options": [], + "query": "label_values({}, host)", + "refresh": 2, + "regex": "", + "type": "query" + }, + { + "current": { "selected": false, "text": "", "value": "" }, + "hide": 0, + "label": "Search", + "name": "search", + "options": [ + { "selected": true, "text": "", "value": "" } + ], + "query": "", + "type": "textbox" + }, + { + "current": {}, + "hide": 0, + "includeAll": false, + "label": "Datasource", + "multi": false, + "name": "datasource", + "options": [], + "query": "loki", + "refresh": 1, + "regex": "", + "type": "datasource" + } + ] + }, + "time": { "from": "now-1h", "to": "now" }, + "title": "Log Explorer", + "uid": "log-explorer" +} diff --git a/hosts/spore/services/default.nix b/hosts/spore/services/default.nix index 114b2c36..e21d5a00 100644 --- a/hosts/spore/services/default.nix +++ b/hosts/spore/services/default.nix @@ -4,6 +4,7 @@ ... }: { imports = [ + ./alloy.nix ./grafana.nix ./homepage-dashboard.nix ./mastodon.nix diff --git a/hosts/spore/services/grafana.nix b/hosts/spore/services/grafana.nix index 2c9b06d4..b385036c 100644 --- a/hosts/spore/services/grafana.nix +++ b/hosts/spore/services/grafana.nix @@ -73,6 +73,12 @@ isDefault = true; editable = false; } + { + name = "Loki"; + type = "loki"; + url = "http://glyph.rove-duck.ts.net:3100"; + editable = false; + } ]; }; }; diff --git a/hosts/zeta/services/alloy.nix b/hosts/zeta/services/alloy.nix new file mode 100644 index 00000000..02f7479e --- /dev/null +++ b/hosts/zeta/services/alloy.nix @@ -0,0 +1,18 @@ +_: { + services.alloy.enable = true; + + environment.etc."alloy/config.alloy".text = '' + loki.source.journal "systemd" { + forward_to = [loki.write.remote.receiver] + } + + loki.write "remote" { + endpoint { + url = "http://glyph.rove-duck.ts.net:3100/loki/api/v1/push" + } + external_labels = { + host = "zeta", + } + } + ''; +} diff --git a/hosts/zeta/services/default.nix b/hosts/zeta/services/default.nix index 9fb999b7..f9a7cac5 100644 --- a/hosts/zeta/services/default.nix +++ b/hosts/zeta/services/default.nix @@ -4,6 +4,7 @@ ... }: { imports = [ + ./alloy.nix ./home-assistant.nix ./homebridge.nix ./znc.nix