tide: add more metrics

Author: Prucek

pkg/tide/tide.go

@@ -216,6 +216,15 @@ var (
 
 		// Per controller
 		syncHeartbeat *prometheus.CounterVec
+
+		// Retesting metrics
+		retests             *prometheus.CounterVec
+		retestsByAction     *prometheus.CounterVec
+		batchFailures       *prometheus.CounterVec
+		poolMissingPRs      *prometheus.GaugeVec
+		poolPendingPRs      *prometheus.GaugeVec
+		poolSuccessfulPRs   *prometheus.GaugeVec
+		poolBatchPendingPRs *prometheus.GaugeVec
 	}{
 		pooledPRs: prometheus.NewGaugeVec(prometheus.GaugeOpts{
 			Name: "pooledprs",
@@ -280,6 +289,63 @@ var (
 		}, []string{
 			"controller",
 		}),
+		retests: prometheus.NewCounterVec(prometheus.CounterOpts{
+			Name: "tide_retests_total",
+			Help: "Total number of test retriggers by org, repo, and branch. Incremented when Tide triggers tests for PRs that need retesting.",
+		}, []string{
+			"org",
+			"repo",
+			"branch",
+		}),
+		retestsByAction: prometheus.NewCounterVec(prometheus.CounterOpts{
+			Name: "tide_retests_by_action_total",
+			Help: "Total number of retests by action type (TRIGGER for serial, TRIGGER_BATCH for batch). Helps identify whether batch or serial testing is causing more retests.",
+		}, []string{
+			"org",
+			"repo",
+			"branch",
+			"action",
+		}),
+		batchFailures: prometheus.NewCounterVec(prometheus.CounterOpts{
+			Name: "tide_batch_failures_total",
+			Help: "Total number of times a batch test completes but PRs move back to missing state (indicating batch failure). This is expensive as all PRs in the batch need retesting.",
+		}, []string{
+			"org",
+			"repo",
+			"branch",
+		}),
+		poolMissingPRs: prometheus.NewGaugeVec(prometheus.GaugeOpts{
+			Name: "tide_pool_missing_prs",
+			Help: "Number of PRs with missing or failed tests in each pool. High values indicate testing bottlenecks.",
+		}, []string{
+			"org",
+			"repo",
+			"branch",
+		}),
+		poolPendingPRs: prometheus.NewGaugeVec(prometheus.GaugeOpts{
+			Name: "tide_pool_pending_prs",
+			Help: "Number of PRs with pending tests in each pool.",
+		}, []string{
+			"org",
+			"repo",
+			"branch",
+		}),
+		poolSuccessfulPRs: prometheus.NewGaugeVec(prometheus.GaugeOpts{
+			Name: "tide_pool_successful_prs",
+			Help: "Number of PRs with all tests passing in each pool.",
+		}, []string{
+			"org",
+			"repo",
+			"branch",
+		}),
+		poolBatchPendingPRs: prometheus.NewGaugeVec(prometheus.GaugeOpts{
+			Name: "tide_pool_batch_pending_prs",
+			Help: "Number of PRs in a pending batch test in each pool.",
+		}, []string{
+			"org",
+			"repo",
+			"branch",
+		}),
 	}
 )
 
@@ -292,6 +358,13 @@ func init() {
 	prometheus.MustRegister(tideMetrics.syncHeartbeat)
 	prometheus.MustRegister(tideMetrics.poolErrors)
 	prometheus.MustRegister(tideMetrics.queryResults)
+	prometheus.MustRegister(tideMetrics.retests)
+	prometheus.MustRegister(tideMetrics.retestsByAction)
+	prometheus.MustRegister(tideMetrics.batchFailures)
+	prometheus.MustRegister(tideMetrics.poolMissingPRs)
+	prometheus.MustRegister(tideMetrics.poolPendingPRs)
+	prometheus.MustRegister(tideMetrics.poolSuccessfulPRs)
+	prometheus.MustRegister(tideMetrics.poolBatchPendingPRs)
 }
 
 type manager interface {
@@ -1705,8 +1778,22 @@ func (c *syncController) syncSubpool(sp subpool, blocks []blockers.Blocker) (Poo
 		"action":  string(act),
 		"targets": prNumbers(targets),
 	}).Info("Subpool synced.")
+
 	tideMetrics.pooledPRs.WithLabelValues(sp.org, sp.repo, sp.branch).Set(float64(len(sp.prs)))
 	tideMetrics.updateTime.WithLabelValues(sp.org, sp.repo, sp.branch).Set(float64(time.Now().Unix()))
+	tideMetrics.poolMissingPRs.WithLabelValues(sp.org, sp.repo, sp.branch).Set(float64(len(missings)))
+	tideMetrics.poolPendingPRs.WithLabelValues(sp.org, sp.repo, sp.branch).Set(float64(len(pendings)))
+	tideMetrics.poolSuccessfulPRs.WithLabelValues(sp.org, sp.repo, sp.branch).Set(float64(len(successes)))
+	tideMetrics.poolBatchPendingPRs.WithLabelValues(sp.org, sp.repo, sp.branch).Set(float64(len(batchPending)))
+	if act == Trigger || act == TriggerBatch {
+		tideMetrics.retests.WithLabelValues(sp.org, sp.repo, sp.branch).Add(float64(len(targets)))
+		tideMetrics.retestsByAction.WithLabelValues(sp.org, sp.repo, sp.branch, string(act)).Add(float64(len(targets)))
+	}
+	if len(batchPending) == 0 && len(missings) > 0 && len(pendings) == 0 {
+		if act == Trigger {
+			tideMetrics.batchFailures.WithLabelValues(sp.org, sp.repo, sp.branch).Inc()
+		}
+	}
 	return Pool{
 			Org:    sp.org,
 			Repo:   sp.repo,