Merge pull request #30143 from dgoodwin/drop-etcd-took-too-long-test

OCPBUGS-52968: Drop the etcd should not log excessive took too long messages test

Author: sdodson

pkg/monitortests/etcd/legacyetcdmonitortests/etcd.go

@@ -80,45 +80,6 @@ func testEtcdShouldNotLogDroppedRaftMessages(events monitorapi.Intervals) []*jun
 	return []*junitapi.JUnitTestCase{failure, success}
 }
 
-// etcdTookTooLongMaxRatePerFourHours is the max rate of messages allowed over a 4-hour period.
-// This replaces the fixed limit approach with a rate-based approach.
-// Virtually all jobs log these messages at some point, we're just interested in the ones that do so excessively.
-const etcdTookTooLongMaxRatePerFourHours = 12000
-
-func testEtcdDoesNotLogExcessiveTookTooLongMessages(events monitorapi.Intervals, startTime time.Time) []*junitapi.JUnitTestCase {
-	const testName = "[sig-etcd] etcd should not log excessive took too long messages"
-	success := &junitapi.JUnitTestCase{Name: testName}
-
-	counter := 0
-	for _, event := range events {
-		if event.Source == monitorapi.SourceEtcdLog &&
-			strings.Contains(event.Message.HumanMessage, "took too long") {
-			counter++
-		}
-	}
-
-	maxAllowedCount := calculateRateBasedLimit(startTime, etcdTookTooLongMaxRatePerFourHours)
-	actualDuration := time.Since(startTime)
-
-	if counter <= maxAllowedCount {
-		return []*junitapi.JUnitTestCase{success}
-	}
-
-	msg := fmt.Sprintf("Etcd logged %d 'took too long' messages in %v, exceeding the rate-based limit of %d "+
-		"(based on max rate of %d messages per 4 hours). This is a strong indicator that etcd was very unhealthy "+
-		"throughout the run. This can cause sporadic e2e failures and disruption and typically indicates faster "+
-		"disks are needed. These log message intervals are included in spyglass chart artifacts and can be used "+
-		"to correlate with disruption and failed tests.",
-		counter, actualDuration.Round(time.Minute), maxAllowedCount, etcdTookTooLongMaxRatePerFourHours)
-	failure := &junitapi.JUnitTestCase{
-		Name: testName,
-		FailureOutput: &junitapi.FailureOutput{
-			Output: msg,
-		},
-	}
-	return []*junitapi.JUnitTestCase{failure}
-}
-
 // etcdOverloadedNetworkMaxRatePerFourHours uses the same rate-based approach for overloaded network messages.
 // We use the same rate limit as the "took too long" messages since both indicate severe etcd health issues.
 //

pkg/monitortests/etcd/legacyetcdmonitortests/monitortest.go

@@ -71,7 +71,6 @@ func (w *legacyMonitorTests) EvaluateTestsFromConstructedIntervals(ctx context.C
 	junits = append(junits, testEtcdShouldNotLogSlowFdataSyncs(finalIntervals)...)
 	junits = append(junits, testEtcdShouldNotLogDroppedRaftMessages(finalIntervals)...)
 	junits = append(junits, testOperatorStatusChanged(finalIntervals)...)
-	junits = append(junits, testEtcdDoesNotLogExcessiveTookTooLongMessages(finalIntervals, w.startTime)...)
 	junits = append(junits, testEtcdDoesNotLogExcessiveOverloadedNetworkMessages(finalIntervals, w.startTime)...)
 
 	return junits, nil