Add rand to sparse requests for rejoin memberlist cluster (#4747)

Signed-off-by: Daniel Blando <[email protected]>

Author: danielblando

CHANGELOG.md

@@ -28,6 +28,7 @@
 * [ENHANCEMENT] Blocks storage: Add `-blocks-storage.azure.http.*`, `-alertmanager-storage.azure.http.*`, and `-ruler-storage.azure.http.*` to configure the Azure storage client. #4581
 * [ENHANCEMENT] Optimise memberlist receive path when used as a backing store for rings with a large number of members. #4601
 * [ENHANCEMENT] Add length and limit to labelNameTooLongError and labelValueTooLongError #4595
+* [ENHANCEMENT] Add jitter to rejoinInterval. #4747
 * [BUGFIX] AlertManager: remove stale template files. #4495
 * [BUGFIX] Distributor: fix bug in query-exemplar where some results would get dropped. #4583
 * [BUGFIX] Update Thanos dependency: compactor tracing support, azure blocks storage memory fix. #4585

pkg/ring/kv/memberlist/memberlist_client.go

@@ -9,6 +9,7 @@ import (
 	"flag"
 	"fmt"
 	"math"
+	mathrand "math/rand"
 	"strings"
 	"sync"
 	"time"
@@ -462,6 +463,12 @@ func (m *KV) running(ctx context.Context) error {
 	var tickerChan <-chan time.Time
 	if m.cfg.RejoinInterval > 0 && len(m.cfg.JoinMembers) > 0 {
 		t := time.NewTicker(m.cfg.RejoinInterval)
+		t.Stop()
+		time.AfterFunc(time.Duration(uint64(mathrand.Int63())%uint64(m.cfg.RejoinInterval)), func() {
+			m.rejoinMemberlist(ctx)
+
+			t.Reset(m.cfg.RejoinInterval)
+		})
 		defer t.Stop()
 
 		tickerChan = t.C
@@ -470,22 +477,25 @@ func (m *KV) running(ctx context.Context) error {
 	for {
 		select {
 		case <-tickerChan:
-			members := m.discoverMembers(ctx, m.cfg.JoinMembers)
-
-			reached, err := m.memberlist.Join(members)
-			if err == nil {
-				level.Info(m.logger).Log("msg", "re-joined memberlist cluster", "reached_nodes", reached)
-			} else {
-				// Don't report error from rejoin, otherwise KV service would be stopped completely.
-				level.Warn(m.logger).Log("msg", "re-joining memberlist cluster failed", "err", err)
-			}
-
+			m.rejoinMemberlist(ctx)
 		case <-ctx.Done():
 			return nil
 		}
 	}
 }
 
+func (m *KV) rejoinMemberlist(ctx context.Context) {
+	members := m.discoverMembers(ctx, m.cfg.JoinMembers)
+
+	reached, err := m.memberlist.Join(members)
+	if err == nil {
+		level.Info(m.logger).Log("msg", "re-joined memberlist cluster", "reached_nodes", reached)
+	} else {
+		// Don't report error from rejoin, otherwise KV service would be stopped completely.
+		level.Warn(m.logger).Log("msg", "re-joining memberlist cluster failed", "err", err)
+	}
+}
+
 // GetCodec returns codec for given ID or nil.
 func (m *KV) GetCodec(codecID string) codec.Codec {
 	return m.codecs[codecID]