Skip unavailable metrics instead of killing the test process (#1305)

When a Prometheus query returns an empty vector (e.g. because a
monitored pod was OOMKilled), the metrics gatherer retries for 5
minutes then calls Fatalf, which terminates the entire test run
via os.Exit(1). This is disproportionate for a transient condition.

- Replace Fatalf with Infof + continue so the current gathering
  cycle skips the unavailable metric and the next cycle retries
- Guard avg() against division by zero when sampleCount is 0
- Add tests for empty vector, zero-sample avg, and ComputeResults
  with no samples

Fixes #1304

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Signed-off-by: Víctor M. Múgica <vmugicag@redhat.com>
Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
Co-authored-by: Rajiv Senthilnathan <rajivnathan@gmail.com>

Author: 3 people

setup/metrics/gather.go

@@ -43,6 +43,9 @@ type aggregateResult struct {
 }
 
 func (r aggregateResult) avg() float64 {
+	if r.sampleCount == 0 {
+		return 0
+	}
 	return r.sum / float64(r.sampleCount)
 }
 
@@ -109,7 +112,8 @@ func (g *Gatherer) StartGathering() chan struct{} {
 					return metricsErr == nil, nil
 				})
 				if err != nil {
-					g.term.Fatalf(metricsErr, "metrics error")
+					g.term.Infof("⚠ Skipping metric %q: %s (will retry on next gathering interval)", q.Name(), metricsErr)
+					continue
 				}
 			}
 		}, g.queryInterval, stop)

setup/metrics/gather_test.go

@@ -2,6 +2,7 @@ package metrics
 
 import (
 	"fmt"
+	"io"
 	"reflect"
 	"testing"
 
@@ -17,6 +18,18 @@ func TestExecuteQueryAndProcessResult(t *testing.T) {
 	var testTime = model.Now()
 
 	tests := []testcase{
+		{
+			query: testQuery{
+				name: "empty vector",
+				sample: queryResult{
+					val: model.Vector{},
+				},
+			},
+			exp: expected{
+				err:       "metrics value could not be retrieved for query empty vector",
+				resultLen: 0,
+			},
+		},
 		{
 			query: testQuery{
 				name: "first sample",
@@ -190,3 +203,45 @@ func (q testQuery) Execute() (model.Value, prometheus.Warnings, error) {
 func (q testQuery) ResultType() string {
 	return "memory"
 }
+
+func TestAvgZeroSamples(t *testing.T) {
+	// given
+	r := aggregateResult{sampleCount: 0, sum: 0, max: 0}
+
+	// when
+	result := r.avg()
+
+	// then
+	require.InDelta(t, float64(0), result, 0.01)
+}
+
+func TestComputeResultsZeroSamples(t *testing.T) {
+	// given
+	q := testQuery{name: "zero sample metric"}
+	g := &Gatherer{
+		mqueries: []queries.Query{q},
+		results:  map[string]aggregateResult{"zero sample metric": {}},
+		term:     &noopTerminal{},
+	}
+
+	// when
+	tuples := g.ComputeResults()
+
+	// then
+	require.Len(t, tuples, 2)
+	require.Equal(t, "Average zero sample metric (MB)", tuples[0][0])
+	require.Equal(t, "0.00", tuples[0][1])
+	require.Equal(t, "Max zero sample metric (MB)", tuples[1][0])
+	require.Equal(t, "0.00", tuples[1][1])
+}
+
+type noopTerminal struct{}
+
+func (t *noopTerminal) InOrStdin() io.Reader                              { return nil }
+func (t *noopTerminal) OutOrStdout() io.Writer                            { return io.Discard }
+func (t *noopTerminal) Debugf(msg string, args ...interface{})            {}
+func (t *noopTerminal) Infof(msg string, args ...interface{})             {}
+func (t *noopTerminal) Errorf(err error, msg string, args ...interface{}) {}
+func (t *noopTerminal) Fatalf(err error, msg string, args ...interface{}) { panic("unexpected Fatalf") }
+func (t *noopTerminal) PromptBoolf(msg string, args ...interface{}) bool  { return false }
+func (t *noopTerminal) AddPreFatalExitHook(func())                        {}