Add index stats info to bucket index (cortexproject#5506)

* add index stats info to bucket index

Signed-off-by: Ben Ye <[email protected]>

* update changelog

Signed-off-by: Ben Ye <[email protected]>

---------

Signed-off-by: Ben Ye <[email protected]>

Author: yeya24

CHANGELOG.md

@@ -17,6 +17,7 @@
 * [CHANGE] Store Gateway: Rename `cortex_bucket_store_cached_series_fetch_duration_seconds` to `cortex_bucket_store_series_fetch_duration_seconds` and `cortex_bucket_store_cached_postings_fetch_duration_seconds` to `cortex_bucket_store_postings_fetch_duration_seconds`. Add new metric `cortex_bucket_store_chunks_fetch_duration_seconds`. #5448
 * [CHANGE] Store Gateway: Remove `idle_timeout`, `max_conn_age`, `pool_size`, `min_idle_conns` fields for Redis index cache and caching bucket. #5448
 * [CHANGE] Store Gateway: Add flag `-store-gateway.sharding-ring.zone-stable-shuffle-sharding` to enable store gateway to use zone stable shuffle sharding. #5489
+* [CHANGE] Bucket Index: Add `series_max_size` and `chunk_max_size` to bucket index. #5489
 * [FEATURE] Store Gateway: Add `max_downloaded_bytes_per_request` to limit max bytes to download per store gateway request.
 * [FEATURE] Added 2 flags `-alertmanager.alertmanager-client.grpc-max-send-msg-size` and ` -alertmanager.alertmanager-client.grpc-max-recv-msg-size` to configure alert manager grpc client message size limits. #5338
 * [FEATURE] Query Frontend: Add `cortex_rejected_queries_total` metric for throttled queries. #5356

pkg/storage/tsdb/bucketindex/index.go

@@ -80,6 +80,10 @@ type Block struct {
 	SegmentsFormat string `json:"segments_format,omitempty"`
 	SegmentsNum    int    `json:"segments_num,omitempty"`
 
+	// Max size in bytes of series and chunk in the block.
+	SeriesMaxSize int64 `json:"series_max_size,omitempty"`
+	ChunkMaxSize  int64 `json:"chunk_max_size,omitempty"`
+
 	// UploadedAt is a unix timestamp (seconds precision) of when the block has been completed to be uploaded
 	// to the storage.
 	UploadedAt int64 `json:"uploaded_at"`
@@ -113,6 +117,10 @@ func (m *Block) ThanosMeta(userID string) *metadata.Meta {
 				cortex_tsdb.TenantIDExternalLabel: userID,
 			},
 			SegmentFiles: m.thanosMetaSegmentFiles(),
+			IndexStats: metadata.IndexStats{
+				SeriesMaxSize: m.SeriesMaxSize,
+				ChunkMaxSize:  m.ChunkMaxSize,
+			},
 		},
 	}
 }
@@ -143,6 +151,8 @@ func BlockFromThanosMeta(meta metadata.Meta) *Block {
 		MaxTime:        meta.MaxTime,
 		SegmentsFormat: segmentsFormat,
 		SegmentsNum:    segmentsNum,
+		SeriesMaxSize:  meta.Thanos.IndexStats.SeriesMaxSize,
+		ChunkMaxSize:   meta.Thanos.IndexStats.ChunkMaxSize,
 	}
 }
 

pkg/storage/tsdb/bucketindex/index_test.go

@@ -201,6 +201,37 @@ func TestBlockFromThanosMeta(t *testing.T) {
 				SegmentsNum:    3,
 			},
 		},
+		"meta.json with Files and Index Stats": {
+			meta: metadata.Meta{
+				BlockMeta: tsdb.BlockMeta{
+					ULID:    blockID,
+					MinTime: 10,
+					MaxTime: 20,
+				},
+				Thanos: metadata.Thanos{
+					Files: []metadata.File{
+						{RelPath: "index"},
+						{RelPath: "chunks/000001"},
+						{RelPath: "chunks/000002"},
+						{RelPath: "chunks/000003"},
+						{RelPath: "tombstone"},
+					},
+					IndexStats: metadata.IndexStats{
+						SeriesMaxSize: 1000,
+						ChunkMaxSize:  1000,
+					},
+				},
+			},
+			expected: Block{
+				ID:             blockID,
+				MinTime:        10,
+				MaxTime:        20,
+				SegmentsFormat: SegmentsFormat1Based6Digits,
+				SegmentsNum:    3,
+				SeriesMaxSize:  1000,
+				ChunkMaxSize:   1000,
+			},
+		},
 	}
 
 	for testName, testData := range tests {
@@ -314,6 +345,35 @@ func TestBlock_ThanosMeta(t *testing.T) {
 				},
 			},
 		},
+		"block with index stats": {
+			block: Block{
+				ID:             blockID,
+				MinTime:        10,
+				MaxTime:        20,
+				SegmentsFormat: SegmentsFormatUnknown,
+				SegmentsNum:    0,
+				SeriesMaxSize:  1000,
+				ChunkMaxSize:   500,
+			},
+			expected: &metadata.Meta{
+				BlockMeta: tsdb.BlockMeta{
+					ULID:    blockID,
+					MinTime: 10,
+					MaxTime: 20,
+					Version: metadata.TSDBVersion1,
+				},
+				Thanos: metadata.Thanos{
+					Version: metadata.ThanosVersion1,
+					Labels: map[string]string{
+						"__org_id__": userID,
+					},
+					IndexStats: metadata.IndexStats{
+						SeriesMaxSize: 1000,
+						ChunkMaxSize:  500,
+					},
+				},
+			},
+		},
 	}
 
 	for testName, testData := range tests {