clientv3: implement exponential backoff mechanism

The current clientv3 backoff behavior is to do a flat backoff with
jitter. Having a too low backoff wait time can amplify cascading
failures as client requests can be retried many times with a low backoff
between each request. Operators of large etcd clusters can increase the
backoff wait time, but for large clusters that wait time needs to be
quite large in order to safely protect the cluster from a large number
of clients retrying. A very high backoff time means that retries in a
non cascading failure will have a larger wait time than needed.
A better solution to handle both cascading failures as well as having
lower retry times in non cascading failures is to implement exponential
backoff within the etcd clients.

This commit implements the mechanism for exponential backoff in clients
with two new parameters:
1. BackoffExponent: configures exponential backoff factor. For example a
   BackoffExponent of 2.0 doubles the backoff time between each retry.
   The default value of BackoffExponent is 1.0 which disables
   exponential backoff for reverse compatibility.
2. BackoffMaxWaitBetween: configures the max wait time when performing
   exponential backoff. The default value is 5 seconds.

Signed-off-by: Elias Carter <[email protected]>

Author: elias-dbx

client/v3/client.go

@@ -264,10 +264,20 @@ func (c *Client) dialSetupOpts(creds grpccredentials.TransportCredentials, dopts
 		backoffJitterFraction = c.cfg.BackoffJitterFraction
 	}
 
+	backoffExponent := defaultBackoffExponent
+	if c.cfg.BackoffExponent > 0 {
+		backoffExponent = c.cfg.BackoffExponent
+	}
+
+	backoffMaxWaitBetween := defaultBackoffMaxWaitBetween
+	if c.cfg.BackoffMaxWaitBetween > 0 {
+		backoffMaxWaitBetween = c.cfg.BackoffMaxWaitBetween
+	}
+
 	// Interceptor retry and backoff.
 	// TODO: Replace all of clientv3/retry.go with RetryPolicy:
 	// https://github.com/grpc/grpc-proto/blob/cdd9ed5c3d3f87aef62f373b93361cf7bddc620d/grpc/service_config/service_config.proto#L130
-	rrBackoff := withBackoff(c.roundRobinQuorumBackoff(backoffWaitBetween, backoffJitterFraction))
+	rrBackoff := withBackoff(c.roundRobinQuorumBackoff(backoffWaitBetween, backoffJitterFraction, backoffExponent, backoffMaxWaitBetween))
 	opts = append(opts,
 		// Disable stream retry by default since go-grpc-middleware/retry does not support client streams.
 		// Streams that are safe to retry are enabled individually.
@@ -502,14 +512,22 @@ func newClient(cfg *Config) (*Client, error) {
 
 // roundRobinQuorumBackoff retries against quorum between each backoff.
 // This is intended for use with a round robin load balancer.
-func (c *Client) roundRobinQuorumBackoff(waitBetween time.Duration, jitterFraction float64) backoffFunc {
+func (c *Client) roundRobinQuorumBackoff(waitBetween time.Duration, jitterFraction float64, backoffExponent float64, maxWaitBetween time.Duration) backoffFunc {
 	return func(attempt uint) time.Duration {
 		// after each round robin across quorum, backoff for our wait between duration
 		n := uint(len(c.Endpoints()))
 		quorum := (n/2 + 1)
 		if attempt%quorum == 0 {
-			c.lg.Debug("backoff", zap.Uint("attempt", attempt), zap.Uint("quorum", quorum), zap.Duration("waitBetween", waitBetween), zap.Float64("jitterFraction", jitterFraction))
-			return jitterUp(waitBetween, jitterFraction)
+			c.lg.Debug(
+				"backoff",
+				zap.Uint("attempt", attempt),
+				zap.Uint("quorum", quorum),
+				zap.Duration("waitBetween", waitBetween),
+				zap.Float64("jitterFraction", jitterFraction),
+				zap.Float64("backoffExponent", backoffExponent),
+				zap.Duration("maxWaitBetween", maxWaitBetween),
+			)
+			return jitterUp(expBackoff(attempt, backoffExponent, waitBetween, maxWaitBetween), jitterFraction)
 		}
 		c.lg.Debug("backoff skipped", zap.Uint("attempt", attempt), zap.Uint("quorum", quorum))
 		return 0

client/v3/client_test.go

@@ -194,6 +194,46 @@ func TestBackoffJitterFraction(t *testing.T) {
 	require.InDelta(t, backoffJitterFraction, c.cfg.BackoffJitterFraction, 0.01)
 }
 
+func TestBackoffExponent(t *testing.T) {
+	backoffExponent := float64(2.0)
+	cfg := Config{
+		Endpoints:       []string{"127.0.0.1:12345"},
+		BackoffExponent: backoffExponent,
+	}
+	c, err := NewClient(t, cfg)
+	require.NoError(t, err)
+	require.NotNil(t, c)
+	defer c.Close()
+
+	require.InDelta(t, backoffExponent, c.cfg.BackoffExponent, 0.01)
+
+	backoffExponent = float64(1.0)
+	cfg = Config{
+		Endpoints:       []string{"127.0.0.1:12345"},
+		BackoffExponent: backoffExponent,
+	}
+	c, err = NewClient(t, cfg)
+	require.NoError(t, err)
+	require.NotNil(t, c)
+	defer c.Close()
+
+	require.InDelta(t, backoffExponent, c.cfg.BackoffExponent, 0.01)
+}
+
+func TestMaxBackoff(t *testing.T) {
+	backoffMaxWaitBetween := 100 * time.Millisecond
+	cfg := Config{
+		Endpoints:             []string{"127.0.0.1:12345"},
+		BackoffMaxWaitBetween: backoffMaxWaitBetween,
+	}
+	c, err := NewClient(t, cfg)
+	require.NoError(t, err)
+	require.NotNil(t, c)
+	defer c.Close()
+
+	require.Equal(t, backoffMaxWaitBetween, c.cfg.BackoffMaxWaitBetween)
+}
+
 func TestIsHaltErr(t *testing.T) {
 	assert.Truef(t,
 		isHaltErr(t.Context(), errors.New("etcdserver: some etcdserver error")),

client/v3/config.go

@@ -102,6 +102,12 @@ type Config struct {
 	// BackoffJitterFraction is the jitter fraction to randomize backoff wait time.
 	BackoffJitterFraction float64 `json:"backoff-jitter-fraction"`
 
+	// BackoffExponent is the exponential backoff factor for retries.
+	BackoffExponent float64 `json:"backoff-exponent"`
+
+	// BackoffMaxWaitBetween is the max wait time before retrying an RPC after exponential backoff.
+	BackoffMaxWaitBetween time.Duration `json:"backoff-max-wait-between"`
+
 	// TODO: support custom balancer picker
 }
 

client/v3/options.go

@@ -54,6 +54,12 @@ var (
 
 	// client-side retry backoff default jitter fraction.
 	defaultBackoffJitterFraction = 0.10
+
+	// client-side retry backoff exponential factor. Default of 1.0 which is no exponential backoff.
+	defaultBackoffExponent = 1.0
+
+	// client-side retry backoff exponential max wait between requests.
+	defaultBackoffMaxWaitBetween = 5 * time.Second
 )
 
 // defaultCallOpts defines a list of default "gRPC.CallOption".

client/v3/utils.go

@@ -15,6 +15,7 @@
 package clientv3
 
 import (
+	"math"
 	"math/rand"
 	"time"
 )
@@ -29,3 +30,13 @@ func jitterUp(duration time.Duration, jitter float64) time.Duration {
 	multiplier := jitter * (rand.Float64()*2 - 1)
 	return time.Duration(float64(duration) * (1 + multiplier))
 }
+
+// expBackoff returns an exponential backoff duration.
+//
+// This will calculate exponential backoff based upon generation and exponent. The backoff is within [minDelay, maxDelay].
+// For example, an exponent of 2.0 will double the backoff duration every subsequent generation. A generation of 0 will
+// return minDelay.
+func expBackoff(generation uint, exponent float64, minDelay, maxDelay time.Duration) time.Duration {
+	delay := math.Min(math.Pow(exponent, float64(generation))*float64(minDelay), float64(maxDelay))
+	return time.Duration(delay)
+}

client/v3/utils_test.go

@@ -0,0 +1,56 @@
+// Copyright 2025 The etcd Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package clientv3
+
+import (
+	"fmt"
+	"math"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestExpBackoff(t *testing.T) {
+	testCases := []struct {
+		generation      uint
+		exponent        float64
+		minDelay        time.Duration
+		maxDelay        time.Duration
+		expectedBackoff time.Duration
+	}{
+		// exponential backoff with 2.0 exponent
+		{generation: 0, exponent: 2.0, minDelay: 100 * time.Millisecond, maxDelay: 500 * time.Millisecond, expectedBackoff: 100 * time.Millisecond},
+		{generation: 1, exponent: 2.0, minDelay: 100 * time.Millisecond, maxDelay: 500 * time.Millisecond, expectedBackoff: 200 * time.Millisecond},
+		{generation: 2, exponent: 2.0, minDelay: 100 * time.Millisecond, maxDelay: 500 * time.Millisecond, expectedBackoff: 400 * time.Millisecond},
+		{generation: 3, exponent: 2.0, minDelay: 100 * time.Millisecond, maxDelay: 500 * time.Millisecond, expectedBackoff: 500 * time.Millisecond},
+		{generation: math.MaxUint, exponent: 2.0, minDelay: 100 * time.Millisecond, maxDelay: 500 * time.Millisecond, expectedBackoff: 500 * time.Millisecond},
+
+		// exponential backoff with 1.0 exponent
+		{generation: 0, exponent: 1.0, minDelay: 100 * time.Millisecond, maxDelay: 500 * time.Millisecond, expectedBackoff: 100 * time.Millisecond},
+		{generation: 1, exponent: 1.0, minDelay: 100 * time.Millisecond, maxDelay: 500 * time.Millisecond, expectedBackoff: 100 * time.Millisecond},
+		{generation: 2, exponent: 1.0, minDelay: 100 * time.Millisecond, maxDelay: 500 * time.Millisecond, expectedBackoff: 100 * time.Millisecond},
+		{generation: 3, exponent: 1.0, minDelay: 100 * time.Millisecond, maxDelay: 500 * time.Millisecond, expectedBackoff: 100 * time.Millisecond},
+		{generation: math.MaxUint, exponent: 1.0, minDelay: 100 * time.Millisecond, maxDelay: 500 * time.Millisecond, expectedBackoff: 100 * time.Millisecond},
+	}
+
+	for _, testCase := range testCases {
+		testName := fmt.Sprintf("%+v", testCase)
+		t.Run(testName, func(t *testing.T) {
+			backoff := expBackoff(testCase.generation, testCase.exponent, testCase.minDelay, testCase.maxDelay)
+			require.InDelta(t, backoff, testCase.expectedBackoff, 0.01)
+		})
+	}
+}