feat: add Loki log aggregation with Alloy journal forwarding

hosts/glyph/services/alloy.nix

@@ -0,0 +1,18 @@
+_: {
+  services.alloy.enable = true;
+
+  environment.etc."alloy/config.alloy".text = ''
+    loki.source.journal "systemd" {
+      forward_to = [loki.write.local.receiver]
+    }
+
+    loki.write "local" {
+      endpoint {
+        url = "http://localhost:3100/loki/api/v1/push"
+      }
+      external_labels = {
+        host = "glyph",
+      }
+    }
+  '';
+}

hosts/glyph/services/default.nix

@@ -4,12 +4,14 @@
   ...
 }: {
   imports = [
+    ./alloy.nix
     ./attic.nix
     ./db.nix
     ./avahi.nix
     ./dns.nix
     ./filebrowser.nix
     ./jellyfin.nix
+    ./loki.nix
     ./nfs.nix
     ./open-terminal.nix
     ./open-webui.nix
@@ -21,7 +23,6 @@
   users.users.mu.extraGroups = ["media"];
   users.users.${config.services.transmission.user}.extraGroups = ["media"];
 
-  services.cockpit.enable = true;
   services.openssh.enable = true;
   services.roon-server = {
     enable = true;

hosts/glyph/services/loki.nix

@@ -0,0 +1,44 @@
+_: {
+  services.loki = {
+    enable = true;
+    configuration = {
+      server.http_listen_port = 3100;
+      auth_enabled = false;
+
+      common = {
+        path_prefix = "/var/lib/loki";
+        replication_factor = 1;
+        ring.kvstore.store = "inmemory";
+      };
+
+      schema_config.configs = [
+        {
+          from = "2024-01-01";
+          store = "tsdb";
+          object_store = "filesystem";
+          schema = "v13";
+          index = {
+            prefix = "index_";
+            period = "24h";
+          };
+        }
+      ];
+
+      storage_config.filesystem.directory = "/var/lib/loki/chunks";
+
+      limits_config = {
+        retention_period = "30d";
+        ingestion_burst_size_mb = 16;
+        ingestion_rate_mb = 8;
+      };
+
+      compactor = {
+        working_directory = "/var/lib/loki/compactor";
+        compaction_interval = "10m";
+        retention_enabled = true;
+        retention_delete_delay = "2h";
+        delete_request_store = "filesystem";
+      };
+    };
+  };
+}

hosts/spore/services/alloy.nix

@@ -0,0 +1,18 @@
+_: {
+  services.alloy.enable = true;
+
+  environment.etc."alloy/config.alloy".text = ''
+    loki.source.journal "systemd" {
+      forward_to = [loki.write.remote.receiver]
+    }
+
+    loki.write "remote" {
+      endpoint {
+        url = "http://glyph.rove-duck.ts.net:3100/loki/api/v1/push"
+      }
+      external_labels = {
+        host = "spore",
+      }
+    }
+  '';
+}

hosts/spore/services/dashboards/logs.json

@@ -0,0 +1,224 @@
+{
+  "annotations": {
+    "list": [
+      {
+        "builtIn": 1,
+        "datasource": { "type": "grafana", "uid": "-- Grafana --" },
+        "enable": true,
+        "hide": true,
+        "iconColor": "rgba(0, 211, 255, 1)",
+        "name": "Annotations & Alerts",
+        "type": "dashboard"
+      }
+    ]
+  },
+  "editable": false,
+  "fiscalYearStartMonth": 0,
+  "graphTooltip": 1,
+  "links": [],
+  "panels": [
+    {
+      "datasource": { "type": "loki", "uid": "${datasource}" },
+      "fieldConfig": {
+        "defaults": {
+          "color": { "mode": "fixed", "fixedColor": "text" }
+        },
+        "overrides": []
+      },
+      "gridPos": { "h": 4, "w": 8, "x": 0, "y": 0 },
+      "id": 1,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "none",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": ["count"],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "title": "Total Log Lines",
+      "type": "stat",
+      "targets": [
+        {
+          "expr": "sum(count_over_time({host=~\"$host\"} |~ \"$search\" [$__auto]))",
+          "refId": "A"
+        }
+      ]
+    },
+    {
+      "datasource": { "type": "loki", "uid": "${datasource}" },
+      "fieldConfig": {
+        "defaults": {
+          "color": { "mode": "fixed", "fixedColor": "red" }
+        },
+        "overrides": []
+      },
+      "gridPos": { "h": 4, "w": 8, "x": 8, "y": 0 },
+      "id": 2,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "none",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": ["count"],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "title": "Error Lines",
+      "type": "stat",
+      "targets": [
+        {
+          "expr": "sum(count_over_time({host=~\"$host\"} |~ \"$search\" |~ \"(?i)error|fatal|panic\" [$__auto]))",
+          "refId": "A"
+        }
+      ]
+    },
+    {
+      "datasource": { "type": "loki", "uid": "${datasource}" },
+      "fieldConfig": {
+        "defaults": {
+          "color": { "mode": "fixed", "fixedColor": "yellow" }
+        },
+        "overrides": []
+      },
+      "gridPos": { "h": 4, "w": 8, "x": 16, "y": 0 },
+      "id": 3,
+      "options": {
+        "colorMode": "value",
+        "graphMode": "none",
+        "justifyMode": "auto",
+        "orientation": "auto",
+        "reduceOptions": {
+          "calcs": ["count"],
+          "fields": "",
+          "values": false
+        },
+        "textMode": "auto"
+      },
+      "title": "Warning Lines",
+      "type": "stat",
+      "targets": [
+        {
+          "expr": "sum(count_over_time({host=~\"$host\"} |~ \"$search\" |~ \"(?i)warn\" [$__auto]))",
+          "refId": "A"
+        }
+      ]
+    },
+    {
+      "datasource": { "type": "loki", "uid": "${datasource}" },
+      "fieldConfig": {
+        "defaults": {
+          "color": { "mode": "palette-classic" },
+          "custom": {
+            "axisBorderShow": false,
+            "axisCenteredZero": false,
+            "axisLabel": "",
+            "axisPlacement": "auto",
+            "barAlignment": 0,
+            "drawStyle": "bars",
+            "fillOpacity": 50,
+            "gradientMode": "none",
+            "lineWidth": 1,
+            "pointSize": 5,
+            "scaleDistribution": { "type": "linear" },
+            "showPoints": "never",
+            "spanNulls": false,
+            "stacking": { "group": "A", "mode": "normal" },
+            "thresholdsStyle": { "mode": "off" }
+          }
+        },
+        "overrides": []
+      },
+      "gridPos": { "h": 6, "w": 24, "x": 0, "y": 4 },
+      "id": 4,
+      "options": {
+        "legend": { "calcs": [], "displayMode": "list", "placement": "bottom" },
+        "tooltip": { "mode": "multi", "sort": "desc" }
+      },
+      "title": "Log Volume",
+      "type": "timeseries",
+      "targets": [
+        {
+          "expr": "sum by (host) (count_over_time({host=~\"$host\"} |~ \"$search\" [$__auto]))",
+          "legendFormat": "{{host}}",
+          "refId": "A"
+        }
+      ]
+    },
+    {
+      "datasource": { "type": "loki", "uid": "${datasource}" },
+      "gridPos": { "h": 18, "w": 24, "x": 0, "y": 10 },
+      "id": 5,
+      "options": {
+        "dedupStrategy": "none",
+        "enableLogDetails": true,
+        "prettifyLogMessage": false,
+        "showCommonLabels": false,
+        "showLabels": true,
+        "showTime": true,
+        "sortOrder": "Descending",
+        "wrapLogMessage": true
+      },
+      "title": "Logs",
+      "type": "logs",
+      "targets": [
+        {
+          "expr": "{host=~\"$host\"} |~ \"$search\"",
+          "refId": "A"
+        }
+      ]
+    }
+  ],
+  "schemaVersion": 39,
+  "tags": ["logs", "loki"],
+  "templating": {
+    "list": [
+      {
+        "current": {},
+        "hide": 0,
+        "includeAll": true,
+        "label": "Host",
+        "multi": true,
+        "name": "host",
+        "options": [],
+        "query": "label_values({}, host)",
+        "refresh": 2,
+        "regex": "",
+        "type": "query"
+      },
+      {
+        "current": { "selected": false, "text": "", "value": "" },
+        "hide": 0,
+        "label": "Search",
+        "name": "search",
+        "options": [
+          { "selected": true, "text": "", "value": "" }
+        ],
+        "query": "",
+        "type": "textbox"
+      },
+      {
+        "current": {},
+        "hide": 0,
+        "includeAll": false,
+        "label": "Datasource",
+        "multi": false,
+        "name": "datasource",
+        "options": [],
+        "query": "loki",
+        "refresh": 1,
+        "regex": "",
+        "type": "datasource"
+      }
+    ]
+  },
+  "time": { "from": "now-1h", "to": "now" },
+  "title": "Log Explorer",
+  "uid": "log-explorer"
+}

hosts/spore/services/default.nix

@@ -4,6 +4,7 @@
   ...
 }: {
   imports = [
+    ./alloy.nix
     ./grafana.nix
     ./homepage-dashboard.nix
     ./mastodon.nix

hosts/spore/services/grafana.nix

@@ -73,6 +73,12 @@
           isDefault = true;
           editable = false;
         }
+        {
+          name = "Loki";
+          type = "loki";
+          url = "http://glyph.rove-duck.ts.net:3100";
+          editable = false;
+        }
       ];
     };
   };

hosts/zeta/services/alloy.nix

@@ -0,0 +1,18 @@
+_: {
+  services.alloy.enable = true;
+
+  environment.etc."alloy/config.alloy".text = ''
+    loki.source.journal "systemd" {
+      forward_to = [loki.write.remote.receiver]
+    }
+
+    loki.write "remote" {
+      endpoint {
+        url = "http://glyph.rove-duck.ts.net:3100/loki/api/v1/push"
+      }
+      external_labels = {
+        host = "zeta",
+      }
+    }
+  '';
+}

hosts/zeta/services/default.nix

@@ -4,6 +4,7 @@
   ...
 }: {
   imports = [
+    ./alloy.nix
     ./home-assistant.nix
     ./homebridge.nix
     ./znc.nix