Fix ring tokens sorting regression introduced in #3601 (#3815) (#3816)

Signed-off-by: Marco Pracucci <[email protected]>

Author: pracucci

pkg/ring/lifecycler.go

@@ -337,6 +337,12 @@ func (i *Lifecycler) ClaimTokensFor(ctx context.Context, ingesterID string) erro
 			// update timestamp to give gossiping client a chance register ring change.
 			ing := ringDesc.Ingesters[i.ID]
 			ing.Timestamp = time.Now().Unix()
+
+			// Tokens of the leaving ingester may have been generated by an older version of Cortex which
+			// doesn't guarantee sorted tokens, so we enforce sorting here.
+			sort.Sort(tokens)
+			ing.Tokens = tokens
+
 			ringDesc.Ingesters[i.ID] = ing
 			return ringDesc, true, nil
 		}

pkg/ring/model.go

@@ -406,7 +406,13 @@ func (d *Desc) getTokensInfo() map[uint32]instanceInfo {
 func (d *Desc) GetTokens() []uint32 {
 	instances := make([][]uint32, 0, len(d.Ingesters))
 	for _, instance := range d.Ingesters {
-		instances = append(instances, instance.Tokens)
+		// Tokens may not be sorted for an older version of Cortex which, so we enforce sorting here.
+		tokens := instance.Tokens
+		if !sort.IsSorted(Tokens(tokens)) {
+			sort.Sort(Tokens(tokens))
+		}
+
+		instances = append(instances, tokens)
 	}
 
 	return MergeTokens(instances)
@@ -417,7 +423,13 @@ func (d *Desc) GetTokens() []uint32 {
 func (d *Desc) getTokensByZone() map[string][]uint32 {
 	zones := map[string][][]uint32{}
 	for _, instance := range d.Ingesters {
-		zones[instance.Zone] = append(zones[instance.Zone], instance.Tokens)
+		// Tokens may not be sorted for an older version of Cortex which, so we enforce sorting here.
+		tokens := instance.Tokens
+		if !sort.IsSorted(Tokens(tokens)) {
+			sort.Sort(Tokens(tokens))
+		}
+
+		zones[instance.Zone] = append(zones[instance.Zone], tokens)
 	}
 
 	// Merge tokens per zone.

pkg/ring/tokens.go

@@ -78,6 +78,13 @@ func LoadTokensFromFile(tokenFilePath string) (Tokens, error) {
 	}
 	var t Tokens
 	err = t.Unmarshal(b)
+
+	// Tokens may have been written to file by an older version of Cortex which
+	// doesn't guarantee sorted tokens, so we enforce sorting here.
+	if !sort.IsSorted(t) {
+		sort.Sort(t)
+	}
+
 	return t, err
 }
 
@@ -92,7 +99,7 @@ func (t *Tokens) Unmarshal(b []byte) error {
 	if err := json.Unmarshal(b, &tj); err != nil {
 		return err
 	}
-	*t = Tokens(tj.Tokens)
+	*t = tj.Tokens
 	return nil
 }
 

pkg/ring/tokens_test.go

@@ -1,7 +1,10 @@
 package ring
 
 import (
+	"io/ioutil"
 	"math/rand"
+	"os"
+	"path/filepath"
 	"testing"
 
 	"github.com/stretchr/testify/assert"
@@ -55,3 +58,20 @@ func TestTokens_Equals(t *testing.T) {
 		assert.Equal(t, c.expected, c.second.Equals(c.first))
 	}
 }
+
+func TestLoadTokensFromFile_ShouldGuaranteeSortedTokens(t *testing.T) {
+	tmpDir, err := ioutil.TempDir("", "test-tokens")
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		os.RemoveAll(tmpDir)
+	})
+
+	// Store tokens to file.
+	orig := Tokens{1, 5, 3}
+	require.NoError(t, orig.StoreToFile(filepath.Join(tmpDir, "tokens")))
+
+	// Read back and ensure they're sorted.
+	actual, err := LoadTokensFromFile(filepath.Join(tmpDir, "tokens"))
+	require.NoError(t, err)
+	assert.Equal(t, Tokens{1, 3, 5}, actual)
+}