feat(inhibit): add inhibition metrics (#4629)

Add metrics for inhibitor:
- alertmanager_inhibitor_source_alerts_cache_items
- alertmanager_inhibitor_source_alerts_index_items
- alertmanager_inhibitor_mutes_duration_seconds

Add metrics for inhibition rules:
- alertmanager_inhibit_rule_source_alerts_cache_items
- alertmanager_inhibit_rule_source_alerts_index_items
- alertmanager_inhibit_rule_matches_duration_seconds
- alertmanager_inhibit_rule_mutes_duration_seconds

Other changes:
- Add debug logs for duplicate inhibition rule names
- Add Len() method to store.Alerts struct
- Add Len() method to inhibit.index struct
- update docs

Signed-off-by: Siavash Safi <[email protected]>

Author: siavashs

cmd/alertmanager/main.go

@@ -104,7 +104,9 @@ var (
 		prometheus.GaugeOpts{
 			Name: "alertmanager_inhibition_rules",
 			Help: "Number of configured inhibition rules.",
-		})
+		},
+	)
+
 	promslogConfig = promslog.Config{}
 )
 
@@ -408,6 +410,7 @@ func run() int {
 	)
 
 	dispMetrics := dispatch.NewDispatcherMetrics(false, prometheus.DefaultRegisterer)
+	inhibitMetrics := inhibit.NewInhibitorMetrics(prometheus.DefaultRegisterer)
 	pipelineBuilder := notify.NewPipelineBuilder(prometheus.DefaultRegisterer, ff)
 	configLogger := logger.With("component", "configuration")
 	configCoordinator := config.NewCoordinator(
@@ -462,7 +465,7 @@ func run() int {
 		inhibitor.Stop()
 		disp.Stop()
 
-		inhibitor = inhibit.NewInhibitor(alerts, conf.InhibitRules, marker, logger)
+		inhibitor = inhibit.NewInhibitor(alerts, conf.InhibitRules, marker, logger, inhibitMetrics)
 		silencer := silence.NewSilencer(silences, marker, logger)
 
 		// An interface value that holds a nil concrete value is non-nil.

docs/configuration.md

@@ -442,6 +442,7 @@ to reason about and does not trigger this special case.
 
 ```yaml
 # Optional name of the inhibition rule.
+# Duplicate names are allowed but will affect the per-rule metrics.
 name: <string>
 
 # DEPRECATED: Use target_matchers below.

go.mod

@@ -35,6 +35,7 @@ require (
 	github.com/oklog/run v1.2.0
 	github.com/oklog/ulid v1.3.1
 	github.com/prometheus/client_golang v1.23.2
+	github.com/prometheus/client_model v0.6.2
 	github.com/prometheus/common v0.67.1
 	github.com/prometheus/exporter-toolkit v0.14.1
 	github.com/prometheus/sigv4 v0.2.1
@@ -104,7 +105,6 @@ require (
 	github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect
 	github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/prometheus/client_model v0.6.2 // indirect
 	github.com/prometheus/procfs v0.16.1 // indirect
 	github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 // indirect
 	github.com/xhit/go-str2duration/v2 v2.1.0 // indirect

inhibit/index.go

@@ -55,3 +55,10 @@ func (c *index) Delete(key model.Fingerprint) {
 
 	delete(c.items, key)
 }
+
+func (c *index) Len() int {
+	c.mtx.RLock()
+	defer c.mtx.RUnlock()
+
+	return len(c.items)
+}

inhibit/inhibit.go

@@ -20,6 +20,7 @@ import (
 	"time"
 
 	"github.com/oklog/run"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/common/model"
 
 	"github.com/prometheus/alertmanager/config"
@@ -33,26 +34,39 @@ import (
 // currently active alerts and a set of inhibition rules. It implements the
 // Muter interface.
 type Inhibitor struct {
-	alerts provider.Alerts
-	rules  []*InhibitRule
-	marker types.AlertMarker
-	logger *slog.Logger
+	alerts  provider.Alerts
+	rules   []*InhibitRule
+	marker  types.AlertMarker
+	logger  *slog.Logger
+	metrics *InhibitorMetrics
 
 	mtx    sync.RWMutex
 	cancel func()
 }
 
 // NewInhibitor returns a new Inhibitor.
-func NewInhibitor(ap provider.Alerts, rs []config.InhibitRule, mk types.AlertMarker, logger *slog.Logger) *Inhibitor {
+func NewInhibitor(ap provider.Alerts, rs []config.InhibitRule, mk types.AlertMarker, logger *slog.Logger, metrics *InhibitorMetrics) *Inhibitor {
 	ih := &Inhibitor{
-		alerts: ap,
-		marker: mk,
-		logger: logger,
+		alerts:  ap,
+		marker:  mk,
+		logger:  logger,
+		metrics: metrics,
 	}
-	for _, cr := range rs {
-		r := NewInhibitRule(cr)
+
+	ruleNames := make(map[string]struct{})
+	for i, cr := range rs {
+		if _, ok := ruleNames[cr.Name]; ok {
+			ih.logger.Debug("duplicate inhibition rule name", "index", i, "name", cr.Name)
+		}
+
+		r := NewInhibitRule(cr, NewRuleMetrics(cr.Name, metrics))
 		ih.rules = append(ih.rules, r)
+
+		if cr.Name != "" {
+			ruleNames[cr.Name] = struct{}{}
+		}
 	}
+
 	return ih
 }
 
@@ -70,16 +84,30 @@ func (ih *Inhibitor) run(ctx context.Context) {
 				continue
 			}
 			// Update the inhibition rules' cache.
+			cachedSum := 0
+			indexedSum := 0
 			for _, r := range ih.rules {
 				if r.SourceMatchers.Matches(a.Labels) {
 					if err := r.scache.Set(a); err != nil {
 						ih.logger.Error("error on set alert", "err", err)
 						continue
 					}
-
 					r.updateIndex(a)
+
+					cached := r.scache.Len()
+					indexed := r.sindex.Len()
+
+					if r.Name != "" {
+						r.metrics.sourceAlertsCacheItems.With(prometheus.Labels{"rule": r.Name}).Set(float64(cached))
+						r.metrics.sourceAlertsIndexItems.With(prometheus.Labels{"rule": r.Name}).Set(float64(indexed))
+					}
+
+					cachedSum += cached
+					indexedSum += indexed
 				}
 			}
+			ih.metrics.sourceAlertsCacheItems.Set(float64(cachedSum))
+			ih.metrics.sourceAlertsIndexItems.Set(float64(indexedSum))
 		}
 	}
 }
@@ -128,21 +156,29 @@ func (ih *Inhibitor) Stop() {
 // Mutes returns true iff the given label set is muted. It implements the Muter
 // interface.
 func (ih *Inhibitor) Mutes(lset model.LabelSet) bool {
+	start := time.Now()
 	fp := lset.Fingerprint()
 
 	for _, r := range ih.rules {
+		ruleStart := time.Now()
 		if !r.TargetMatchers.Matches(lset) {
 			// If target side of rule doesn't match, we don't need to look any further.
+			r.metrics.matchesDuration.With(prometheus.Labels{"rule": r.Name, "matched": "false"}).Observe(time.Since(ruleStart).Seconds())
 			continue
 		}
+		r.metrics.matchesDuration.With(prometheus.Labels{"rule": r.Name, "matched": "true"}).Observe(time.Since(ruleStart).Seconds())
 		// If we are here, the target side matches. If the source side matches, too, we
 		// need to exclude inhibiting alerts for which the same is true.
 		if inhibitedByFP, eq := r.hasEqual(lset, r.SourceMatchers.Matches(lset)); eq {
 			ih.marker.SetInhibited(fp, inhibitedByFP.String())
+			ih.metrics.mutesDuration.With(prometheus.Labels{"muted": "true"}).Observe(time.Since(start).Seconds())
+			r.metrics.mutesDuration.With(prometheus.Labels{"rule": r.Name, "muted": "true"}).Observe(time.Since(ruleStart).Seconds())
 			return true
 		}
+		r.metrics.mutesDuration.With(prometheus.Labels{"rule": r.Name, "muted": "false"}).Observe(time.Since(ruleStart).Seconds())
 	}
 	ih.marker.SetInhibited(fp)
+	ih.metrics.mutesDuration.With(prometheus.Labels{"muted": "false"}).Observe(time.Since(start).Seconds())
 
 	return false
 }
@@ -173,14 +209,17 @@ type InhibitRule struct {
 	// The index items might overwrite eachother if multiple source alerts have exact equal labels.
 	// Overwrites only happen if the new source alert has bigger EndsAt value.
 	sindex *index
+
+	metrics *RuleMetrics
 }
 
 // NewInhibitRule returns a new InhibitRule based on a configuration definition.
-func NewInhibitRule(cr config.InhibitRule) *InhibitRule {
+func NewInhibitRule(cr config.InhibitRule, metrics *RuleMetrics) *InhibitRule {
 	var (
 		sourcem labels.Matchers
 		targetm labels.Matchers
 	)
+
 	// cr.SourceMatch will be deprecated. This for loop appends regex matchers.
 	for ln, lv := range cr.SourceMatch {
 		matcher, err := labels.NewMatcher(labels.MatchEqual, ln, lv)
@@ -235,6 +274,7 @@ func NewInhibitRule(cr config.InhibitRule) *InhibitRule {
 		Equal:          equal,
 		scache:         store.NewAlerts(),
 		sindex:         newIndex(),
+		metrics:        metrics,
 	}
 
 	rule.scache.SetGCCallback(rule.gcCallback)
@@ -310,6 +350,10 @@ func (r *InhibitRule) gcCallback(alerts []types.Alert) {
 		fp := r.fingerprintEquals(a.Labels)
 		r.sindex.Delete(fp)
 	}
+	if r.Name != "" {
+		r.metrics.sourceAlertsCacheItems.With(prometheus.Labels{"rule": r.Name}).Set(float64(r.scache.Len()))
+		r.metrics.sourceAlertsIndexItems.With(prometheus.Labels{"rule": r.Name}).Set(float64(r.sindex.Len()))
+	}
 }
 
 // hasEqual checks whether the source cache contains alerts matching the equal

inhibit/inhibit_bench_test.go

@@ -198,7 +198,7 @@ func benchmarkMutes(b *testing.B, opts benchmarkOptions) {
 		}
 	}
 
-	ih := NewInhibitor(s, rules, m, promslog.NewNopLogger())
+	ih := NewInhibitor(s, rules, m, promslog.NewNopLogger(), NewInhibitorMetrics(r))
 	defer ih.Stop()
 	go ih.Run()
 

inhibit/inhibit_test.go

@@ -125,9 +125,10 @@ func TestInhibitRuleHasEqual(t *testing.T) {
 	for _, c := range cases {
 		t.Run(c.name, func(t *testing.T) {
 			r := &InhibitRule{
-				Equal:  map[model.LabelName]struct{}{},
-				scache: store.NewAlerts(),
-				sindex: newIndex(),
+				Equal:   map[model.LabelName]struct{}{},
+				scache:  store.NewAlerts(),
+				sindex:  newIndex(),
+				metrics: NewRuleMetrics("test", NewInhibitorMetrics(prometheus.NewRegistry())),
 			}
 			for _, ln := range c.equal {
 				r.Equal[ln] = struct{}{}
@@ -159,7 +160,7 @@ func TestInhibitRuleMatches(t *testing.T) {
 	}
 
 	m := types.NewMarker(prometheus.NewRegistry())
-	ih := NewInhibitor(nil, []config.InhibitRule{rule1, rule2}, m, nopLogger)
+	ih := NewInhibitor(nil, []config.InhibitRule{rule1, rule2}, m, nopLogger, NewInhibitorMetrics(prometheus.NewRegistry()))
 	now := time.Now()
 	// Active alert that matches the source filter of rule1.
 	sourceAlert1 := &types.Alert{
@@ -260,7 +261,7 @@ func TestInhibitRuleMatchers(t *testing.T) {
 	}
 
 	m := types.NewMarker(prometheus.NewRegistry())
-	ih := NewInhibitor(nil, []config.InhibitRule{rule1, rule2}, m, nopLogger)
+	ih := NewInhibitor(nil, []config.InhibitRule{rule1, rule2}, m, nopLogger, NewInhibitorMetrics(prometheus.NewRegistry()))
 	now := time.Now()
 	// Active alert that matches the source filter of rule1.
 	sourceAlert1 := &types.Alert{
@@ -369,8 +370,8 @@ func TestInhibitRuleName(t *testing.T) {
 		Equal: []string{"instance"},
 	}
 
-	rule1 := NewInhibitRule(config1)
-	rule2 := NewInhibitRule(config2)
+	rule1 := NewInhibitRule(config1, nil)
+	rule2 := NewInhibitRule(config2, nil)
 
 	require.Equal(t, "test-rule", rule1.Name, "Expected named rule to have adopt name from config")
 	require.Empty(t, rule2.Name, "Expected unnamed rule to have empty name")
@@ -498,7 +499,7 @@ func TestInhibit(t *testing.T) {
 	} {
 		ap := newFakeAlerts(tc.alerts)
 		mk := types.NewMarker(prometheus.NewRegistry())
-		inhibitor := NewInhibitor(ap, []config.InhibitRule{inhibitRule()}, mk, nopLogger)
+		inhibitor := NewInhibitor(ap, []config.InhibitRule{inhibitRule()}, mk, nopLogger, NewInhibitorMetrics(prometheus.NewRegistry()))
 
 		go func() {
 			for ap.finished != nil {