Add limit time for gossip on ring (#4750)

Signed-off-by: Daniel Blando <[email protected]>

Author: danielblando

CHANGELOG.md

@@ -40,6 +40,7 @@
 * [BUGFIX] Distributor: Fix race condition on `/series` introduced by #4683. #4716
 * [BUGFIX] Ruler: Fixed leaking notifiers after users are removed #4718
 * [BUGFIX] Distributor: Fix a memory leak in distributor due to the cluster label. #4739
+* [BUGFIX] Memberlist: Avoid clock skew by limiting the timestamp accepted on gossip. #
 * [ENHANCEMENT] Compactor: uploading blocks no compaction marks to the global location and introduce a new metric #4729
   * `cortex_bucket_blocks_marked_for_no_compaction_count`: Total number of blocks marked for no compaction in the bucket.
 * [ENHANCEMENT] Querier: Reduce the number of series that are kept in memory while streaming from ingesters. #4745

pkg/ring/merge_test.go

@@ -128,6 +128,14 @@ func TestMerge(t *testing.T) {
 		}
 	}
 
+	futureRing := func() *Desc {
+		return &Desc{
+			Ingesters: map[string]InstanceDesc{
+				"Ing 1": {Addr: "addr1", Timestamp: time.Now().Add(31 * time.Minute).Unix(), State: ACTIVE, Tokens: []uint32{30, 40, 50}},
+			},
+		}
+	}
+
 	expectedFirstSecondThirdFourthMerge := func() *Desc {
 		return &Desc{
 			Ingesters: map[string]InstanceDesc{
@@ -181,6 +189,12 @@ func TestMerge(t *testing.T) {
 			},
 		}, ch)
 	}
+
+	{
+		out, err := firstRing().Merge(futureRing(), false)
+		assert.Empty(t, out)
+		assert.ErrorContains(t, err, "ingester Ing 1 timestamp in the future")
+	}
 }
 
 func TestTokensTakeover(t *testing.T) {

pkg/ring/model.go

@@ -209,7 +209,12 @@ func (d *Desc) mergeWithTime(mergeable memberlist.Mergeable, localCAS bool, now
 	var updated []string
 	tokensChanged := false
 
+	maxFutureLimit := now.Add(30 * time.Minute).Unix()
 	for name, oing := range otherIngesterMap {
+		if oing.Timestamp > maxFutureLimit {
+			return nil, fmt.Errorf("ingester %s timestamp in the future, expected max of %d, got %d", name, maxFutureLimit, oing.Timestamp)
+		}
+
 		ting := thisIngesterMap[name]
 		// ting.Timestamp will be 0, if there was no such ingester in our version
 		if oing.Timestamp > ting.Timestamp {