Merge pull request #30226 from neisw/trt-2246-agent-disruption

trt-2246: lower parallelism based on worker nodes

Author: openshift-merge-bot[bot]

pkg/test/ginkgo/cmd_runsuite.go

@@ -316,19 +316,33 @@ func (o *GinkgoRunSuiteOptions) Run(suite *TestSuite, clusterConfig *clusterdisc
 		}
 	}
 
-	parallelism := o.Parallelism
-	if parallelism == 0 {
-		parallelism = suite.Parallelism
-	}
-
-	if parallelism == 0 {
-		parallelism = 10
-	}
+	// start with suite value which should be based on a 3 worker node cluster
+	parallelism := suite.Parallelism
+	logrus.Infof("Suite defined parallelism %d", parallelism)
 
 	// adjust based on the number of workers
 	totalNodes, workerNodes, err := getClusterNodeCounts(ctx, restConfig)
 	if err != nil {
 		logrus.Errorf("Failed to get cluster node counts: %v", err)
+	} else {
+		// default to 1/3 the defined parallelism value per worker but use the min of that
+		// and the current parallelism value so we don't increase parallelism
+		if workerNodes > 0 && parallelism > 0 {
+			workerParallelism := max(1, parallelism/3) * workerNodes
+			logrus.Infof("Parallelism based on worker node count: %d", workerParallelism)
+			parallelism = min(parallelism, workerParallelism)
+		}
+	}
+
+	// if 0 set our min value
+	if parallelism <= 0 {
+		parallelism = 10
+	}
+
+	// if explicitly set then use the specified value
+	if o.Parallelism > 0 {
+		parallelism = o.Parallelism
+		logrus.Infof("Using specified parallelism value: %d", parallelism)
 	}
 
 	logrus.Infof("Total nodes: %d, Worker nodes: %d, Parallelism: %d", totalNodes, workerNodes, parallelism)

pkg/test/ginkgo/test_suite.go

@@ -98,7 +98,7 @@ var (
 	Stable ClusterStabilityDuringTest = "Stable"
 	// TODO only bring this back if we have some reason to collect Upgrade specific information.  I can't think of reason.
 	// TODO please contact @deads2k for vetting if you think you found something
-	//Upgrade    ClusterStabilityDuringTest = "Upgrade"
+	// Upgrade    ClusterStabilityDuringTest = "Upgrade"
 	// Disruptive means that the suite is expected to induce outages to the cluster.
 	Disruptive ClusterStabilityDuringTest = "Disruptive"
 )
@@ -119,7 +119,9 @@ type TestSuite struct {
 
 	// The number of times to execute each test in this suite.
 	Count int `json:"count,omitempty"`
-	// The maximum parallelism of this suite.
+	// The maximum parallelism of this suite based on a minimum of 3 workers.
+	// Otherwise, parallelism is automatically scaled down using 1/3 the parallelism value x the number of workers.
+	// Jobs can explicitly set the parallelism value via max-parallel-tests
 	Parallelism int `json:"parallelism,omitempty"`
 	// The number of flakes that may occur before this test is marked as a failure.
 	MaximumAllowedFlakes int `json:"maximumAllowedFlakes,omitempty"`

pkg/testsuites/standard_suites.go

@@ -147,6 +147,7 @@ var staticSuites = []ginkgo.TestSuite{
 			withExcludedTestsFilter(withStandardEarlyOrLateTests("name.contains('[Suite:openshift/conformance/serial')")),
 		},
 		TestTimeout: 40 * time.Minute,
+		Parallelism: 1,
 	},
 	{
 		Name: "openshift/disruptive",