feat: remove absence alerts. make common alert file

changelogs/fragments/461.yml

@@ -0,0 +1,4 @@
+---
+minor_changes:
+ - prometheus - Remove unnecessary absence alerts. The general ExporterDown metric can cover these scenarios
+ - prometheus - Moved the ExporterDown alert to its own common alerts file and have it be enabled by default (no .example extension on the file name)

prometheus/common/alert-rules.d/crunchy-alert-rules-common.yml

@@ -0,0 +1,21 @@
+###
+#
+# Copyright © 2017-2025 Crunchy Data Solutions, Inc. All Rights Reserved.
+#
+###
+
+groups:
+- name: alert-rules
+  rules:
+
+########## COMMON RULES ##########
+  - alert: ExporterDown
+    expr: avg_over_time(up[5m]) < 0.5
+    for: 10s
+    labels:
+      service: system
+      severity: critical
+      severity_num: 300
+    annotations:
+      description: 'Metrics exporter service for {{ $labels.job }} running on {{ $labels.instance }} has been down at least 50% of the time for the last 5 minutes. Service may be flapping or down.'
+      summary: 'Prometheus Exporter Service Down'

prometheus/common/alert-rules.d/crunchy-alert-rules-etcd.yml.example

@@ -56,36 +56,3 @@ groups:
 #      severity_num: 300
 #    annotations:
 #      description: 'The expected minimum count of etcd nodes was not found. Current count {{ $value }}'
-
-# Absence alerts must be configured per named job, otherwise there's no way to know which job is down
-# Below is are some examples using the leader metric for a targets called "etcd#" for a 3 node etcd cluster
-
-#  - alert: ETCDAbsent_etcd1
-#    expr: absent(etcd_server_has_leader{job="ip11_etcd1"})
-#    for: 10s
-#    labels:
-#      service: etcd
-#      severity: critical
-#      severity_num: 300
-#    annotations:
-#      description: 'Leader metric is absent from target {{ $labels.job }}. Check that etcd is running on target host.'
-
-#  - alert: ETCDAbsent_etcd2
-#    expr: absent(etcd_server_has_leader{job="ip21_etcd2"})
-#    for: 10s
-#    labels:
-#      service: etcd
-#      severity: critical
-#      severity_num: 300
-#    annotations:
-#      description: 'Leader metric is absent from target {{ $labels.job }}. Check that etcd is running on target host.'
-
-#  - alert: ETCDAbsent_etcd3
-#    expr: absent(etcd_server_has_leader{job="ip31_etcd3"})
-#    for: 10s
-#    labels:
-#      service: etcd
-#      severity: critical
-#      severity_num: 300
-#    annotations:
-#      description: 'Leader metric is absent from target {{ $labels.job }}. Check that etcd is running on target host.'

prometheus/common/alert-rules.d/crunchy-alert-rules-pg.yml.example

@@ -164,18 +164,6 @@ groups:
 #      summary: '{{ $labels.job }} has changed from replica to primary'
 
 
-## Absence alerts must be configured per named job, otherwise there's no way to know which job is down
-## Below is an example for a target job called "Prod"
-#  - alert: PGConnectionAbsent_Prod
-#    expr: absent(ccp_connection_stats_max_connections{job="Prod"})
-#    for: 10s
-#    labels:
-#      service: postgresql
-#      severity: critical
-#      severity_num: 300
-#    annotations:
-#      description: 'Connection metric is absent from target (Prod). Check that postgres_exporter can connect to PostgreSQL.'
-
 
 ## Optional monitor for changes to pg_settings (postgresql.conf) system catalog.
 ## A similar metric is available for monitoring pg_hba.conf. See ccp_hba_settings_checksum.