Merge pull request #1871 from input-output-hk/jpraynaud/1826-mithril-nodes-footprint

Monitor Mithril nodes footprint

Author: jpraynaud

CHANGELOG.md

@@ -9,6 +9,8 @@ As a minor extension, we have adopted a slightly different versioning convention
 
 ## Mithril Distribution [XXXX] - UNRELEASED
 
+- Support for Mithril nodes footprint support in Prometheus monitoring in infrastructure
+
 - **UNSTABLE** Cardano stake distribution certification:
 
   - Implement the signable and artifact builders for the signed entity type `CardanoStakeDistribution`

mithril-infra/assets/docker/docker-compose-monitoring.yaml

@@ -52,6 +52,44 @@ services:
       options:
         tag: "{{.ImageName}}|{{.Name}}|{{.ImageFullID}}|{{.FullID}}"
 
+  process-exporter:
+    image: ncabatoff/process-exporter:latest
+    container_name: process-exporter
+    profiles:
+      - tools
+      - monitoring
+      - all
+    restart: always
+    ports:
+      - "9256:9256"
+    volumes:
+      - /proc:/host/proc:ro
+      - ./prometheus/process_exporter/process.yml:/config/process.yml:ro
+    command:
+      - "-procfs=/host/proc"
+      - "-config.path=/config/process.yml"
+    logging:
+      driver: "${LOGGING_DRIVER}"
+      options:
+        tag: "{{.ImageName}}|{{.Name}}|{{.ImageFullID}}|{{.FullID}}"
+
+  disk-usage-exporter:
+    container_name: disk-usage-exporter
+    restart: always
+    build:
+      context: .
+      dockerfile: ./prometheus/disk_usage_exporter/Dockerfile
+      args:
+        DISK_USAGE_EXPORTER_VERSION: 0.6.0
+    profiles:
+      - all
+    ports:
+      - "9995:9995"
+    volumes:
+      - ./prometheus/disk_usage_exporter/disk_usage_exporter.yml:/config/disk_usage_exporter.yml:ro
+      - ../data/${NETWORK}/:/data
+    command: ["run", "--config", "/config/disk_usage_exporter.yml"]
+
   loki:
     image: grafana/loki:2.8.0
     container_name: loki

mithril-infra/assets/docker/prometheus/disk_usage_exporter/Dockerfile

@@ -0,0 +1,29 @@
+ARG DOCKER_IMAGE_FROM=debian:11-slim
+FROM $DOCKER_IMAGE_FROM
+
+# Create appuser
+RUN adduser --no-create-home --disabled-password appuser
+
+# Precreate workdir
+RUN mkdir -p /app/bin
+
+# Upgrade
+RUN apt-get update -y && apt-get install -y libssl-dev ca-certificates wget && apt-get clean && rm -rf /var/lib/apt/lists/*
+
+#Workdir
+WORKDIR /app/
+
+## Download Disk usage exporter
+ARG DISK_USAGE_EXPORTER_VERSION=0.6.0
+ARG DISK_USAGE_EXPORTER_URL=https://github.com/dundee/disk_usage_exporter/releases/download/v$DISK_USAGE_EXPORTER_VERSION/disk_usage_exporter_linux_386.tgz
+RUN wget $DISK_USAGE_EXPORTER_URL \
+    && tar xzf disk_usage_exporter_linux_386.tgz \
+    && chmod u+x disk_usage_exporter_linux_386 \
+    && mv disk_usage_exporter_linux_386 /app/bin/disk_usage_exporter \
+    && rm -f disk_usage_exporter_linux_386.tgz
+
+# Use an unprivileged user
+USER appuser
+
+# Run the executable
+ENTRYPOINT ["/app/bin/disk_usage_exporter"]

mithril-infra/assets/docker/prometheus/disk_usage_exporter/disk_usage_exporter-base.yml

@@ -0,0 +1,2 @@
+multi-paths:
+  /data/mithril-aggregator/mithril: 1

mithril-infra/assets/docker/prometheus/process_exporter/process.yml

@@ -0,0 +1,8 @@
+process_names:
+  - comm:
+      - cardano-node
+      - mithril-aggregator
+      - mithril-aggrega
+      - mithril-client
+      - mithril-signer
+      - mithril-relay

mithril-infra/assets/docker/prometheus/prometheus-base.yml

@@ -19,6 +19,14 @@ scrape_configs:
     static_configs:
       - targets: ["node-exporter:9100"]
 
+  - job_name: "process-exporter"
+    static_configs:
+      - targets: ["process-exporter:9256"]
+
+  - job_name: "disk-usage-exporter"
+    static_configs:
+      - targets: ["disk-usage-exporter:9995"]
+
   - job_name: "reverse-proxy"
     static_configs:
       - targets: ["reverse-proxy:8080"]

mithril-infra/assets/infra.version

@@ -1 +1 @@
-0.2.27
+0.2.28

mithril-infra/mithril.monitoring.tf

@@ -75,12 +75,24 @@ cat >> /home/curry/docker/promtail/promtail-config.yml << EOF
   - url: https://${var.loki_ingest_username}:${var.loki_ingest_password}@${var.loki_ingest_host}/loki/api/v1/push
 EOF
 fi
+EOT
+      ,
+      <<-EOT
+set -e
+# Setup disk usage exporter configuration
+cp /home/curry/docker/prometheus/disk_usage_exporter/disk_usage_exporter-base.yml /home/curry/docker/prometheus/disk_usage_exporter/disk_usage_exporter.yml
+# Setup disk usage exporter targets for Mithril signers
+MITHRIL_SIGNER_NODES=$(docker ps --format='{{.Names}},' | grep "mithril-signer" | grep -v "www" | grep -v "relay" | sort | tr -d '\n\t\r ' | sed 's/.$//' | tr ',' ' ')
+for MITHRIL_SIGNER_NODE in $MITHRIL_SIGNER_NODES; do
+    echo "  /data/$MITHRIL_SIGNER_NODE/mithril: 1" >> /home/curry/docker/prometheus/disk_usage_exporter/disk_usage_exporter.yml 
+done
 EOT
     ]
   }
 
   provisioner "remote-exec" {
     inline = [
+      "export NETWORK=${var.cardano_network}",
       "export LOGGING_DRIVER='${var.mithril_container_logging_driver}'",
       "export PROMETHEUS_HOST=${local.prometheus_host}",
       "export PROMETHEUS_AUTH_USER_PASSWORD=$(htpasswd -nb ${var.prometheus_auth_username} ${var.prometheus_auth_password})",