bucket-index-enabled by default

AGENTS.md

@@ -23,9 +23,7 @@ make ./cmd/cortex/.uptodate    # Build Cortex Docker image for integration tests
 ### Unit Tests
 
 ```bash
-go test ./...                                    # Run all unit tests
-go test ./pkg/distributor/...                    # Run tests for specific package
-go test -tags "netgo slicelabels" ./...          # Run with build tags (matches CI)
+go test -timeout 2400s -tags "netgo slicelabels" ./...          # Run tests with CI configuration
 ```
 
 ### Integration Tests

CHANGELOG.md

@@ -1,6 +1,7 @@
 # Changelog
 
 ## master / unreleased
+* [CHANGE] Blocks storage: Bucket index is now enabled by default. Disabling the bucket index (`-blocks-storage.bucket-store.bucket-index.enabled=false`) is not recommended for production. #7259
 * [CHANGE] Users Scanner: Rename user index update configuration. #7180
   * Flag: Renamed `-*.users-scanner.user-index.cleanup-interval` to `-*.users-scanner.user-index.update-interval`.
   * Config: Renamed `clean_up_interval` to `update_interval` within the `users_scanner` configuration block..

docs/blocks-storage/bucket-index.md

@@ -7,7 +7,7 @@ slug: bucket-index
 
 The bucket index is a **per-tenant file containing the list of blocks and block deletion marks** in the storage. The bucket index itself is stored in the backend object storage, is periodically updated by the compactor, and used by queriers, store-gateways and rulers to discover blocks in the storage.
 
-The bucket index usage is **optional** and can be enabled via `-blocks-storage.bucket-store.bucket-index.enabled=true` (or its respective YAML config option).
+The bucket index is **enabled by default**. Disabling it via `-blocks-storage.bucket-store.bucket-index.enabled=false` is not recommended for production environments.
 
 ## Benefits
 
@@ -34,7 +34,7 @@ The `bucket-index.json.gz` contains:
 
 The [compactor](./compactor.md) periodically scans the bucket and uploads an updated bucket index to the storage. The frequency at which the bucket index is updated can be configured via `-compactor.cleanup-interval`.
 
-Despite using the bucket index is optional, the index itself is built and updated by the compactor even if `-blocks-storage.bucket-store.bucket-index.enabled` has **not** been enabled. This is intentional, so that once a Cortex cluster operator decides to enable the bucket index in a live cluster, the bucket index for any tenant is already existing and query results consistency is guaranteed. The overhead introduced by keeping the bucket index updated is expected to be non significative.
+The index itself is built and updated by the compactor even if `-blocks-storage.bucket-store.bucket-index.enabled` has **not** been enabled. This is intentional, so that once a Cortex cluster operator decides to enable the bucket index in a live cluster, the bucket index for any tenant is already existing and query results consistency is guaranteed. The overhead introduced by keeping the bucket index updated is expected to be non significative.
 
 ## How it's used by the querier
 

docs/blocks-storage/querier.md

@@ -1716,9 +1716,10 @@ blocks_storage:
 
     bucket_index:
       # True to enable querier and store-gateway to discover blocks in the
-      # storage via bucket index instead of bucket scanning.
+      # storage via bucket index instead of bucket scanning. Disabling the
+      # bucket index is not recommended for production.
       # CLI flag: -blocks-storage.bucket-store.bucket-index.enabled
-      [enabled: <boolean> | default = false]
+      [enabled: <boolean> | default = true]
 
       # How frequently a bucket index, which previously failed to load, should
       # be tried to load again. This option is used only by querier.

docs/blocks-storage/store-gateway.md

@@ -1782,9 +1782,10 @@ blocks_storage:
 
     bucket_index:
       # True to enable querier and store-gateway to discover blocks in the
-      # storage via bucket index instead of bucket scanning.
+      # storage via bucket index instead of bucket scanning. Disabling the
+      # bucket index is not recommended for production.
       # CLI flag: -blocks-storage.bucket-store.bucket-index.enabled
-      [enabled: <boolean> | default = false]
+      [enabled: <boolean> | default = true]
 
       # How frequently a bucket index, which previously failed to load, should
       # be tried to load again. This option is used only by querier.

docs/configuration/config-file-reference.md

@@ -2394,9 +2394,10 @@ bucket_store:
 
   bucket_index:
     # True to enable querier and store-gateway to discover blocks in the storage
-    # via bucket index instead of bucket scanning.
+    # via bucket index instead of bucket scanning. Disabling the bucket index is
+    # not recommended for production.
     # CLI flag: -blocks-storage.bucket-store.bucket-index.enabled
-    [enabled: <boolean> | default = false]
+    [enabled: <boolean> | default = true]
 
     # How frequently a bucket index, which previously failed to load, should be
     # tried to load again. This option is used only by querier.

docs/configuration/single-process-config-blocks-gossip-1.yaml

@@ -105,3 +105,5 @@ alertmanager_storage:
   local:
     # Make sure file exist
     path: /tmp/cortex/alerts
+compactor:
+  cleanup_interval: 10s

docs/configuration/single-process-config-blocks-gossip-2.yaml

@@ -104,3 +104,5 @@ alertmanager_storage:
   local:
     # Make sure file exist
     path: /tmp/cortex/alerts
+compactor:
+  cleanup_interval: 10s

docs/configuration/v1-guarantees.md

@@ -60,9 +60,7 @@ Currently experimental features are:
   - Accept Prometheus remote write 2.0 request (`-distributor.remote-writev2-enabled=true`)
 - Tenant Deletion in Purger, for blocks storage.
 - Query-frontend: query stats tracking (`-frontend.query-stats-enabled`)
-- Blocks storage bucket index
-  - The bucket index support in the querier and store-gateway (enabled via `-blocks-storage.bucket-store.bucket-index.enabled=true`) is experimental
-  - The block deletion marks migration support in the compactor (`-compactor.block-deletion-marks-migration-enabled`) is temporarily and will be removed in future versions
+- The block deletion marks migration support in the compactor (`-compactor.block-deletion-marks-migration-enabled`) is temporarily and will be removed in future versions
 - Blocks storage user index
 - Querier: tenant federation
   - `-tenant-federation.enabled`

integration/querier_microservices_mode_test.go

@@ -30,237 +30,132 @@ func TestQuerierWithBlocksStorageRunningInMicroservicesMode(t *testing.T) {
 		indexCacheBackend      string
 		chunkCacheBackend      string
 		parquetLabelsCache     string
-		bucketIndexEnabled     bool
 	}{
 		// tsdb bucket storage
-		"[TSDB] blocks sharding disabled, memcached index cache": {
-			bucketStorageType:      "tsdb",
-			blocksShardingStrategy: "",
-			indexCacheBackend:      tsdb.IndexCacheBackendMemcached,
-			chunkCacheBackend:      tsdb.CacheBackendMemcached,
-		},
-		"[TSDB] blocks sharding disabled, multilevel index cache (inmemory, memcached)": {
-			bucketStorageType:      "tsdb",
-			blocksShardingStrategy: "",
-			indexCacheBackend:      fmt.Sprintf("%v,%v", tsdb.IndexCacheBackendInMemory, tsdb.IndexCacheBackendMemcached),
-			chunkCacheBackend:      tsdb.CacheBackendMemcached,
-		},
-		"[TSDB] blocks sharding disabled, redis index cache": {
-			bucketStorageType:      "tsdb",
-			blocksShardingStrategy: "",
-			indexCacheBackend:      tsdb.IndexCacheBackendRedis,
-			chunkCacheBackend:      tsdb.CacheBackendRedis,
-		},
-		"[TSDB] blocks sharding disabled, multilevel index cache (inmemory, redis)": {
-			bucketStorageType:      "tsdb",
-			blocksShardingStrategy: "",
-			indexCacheBackend:      fmt.Sprintf("%v,%v", tsdb.IndexCacheBackendInMemory, tsdb.IndexCacheBackendRedis),
-			chunkCacheBackend:      tsdb.CacheBackendRedis,
-		},
-		"[TSDB] blocks default sharding, inmemory index cache": {
-			bucketStorageType:      "tsdb",
-			blocksShardingStrategy: "default",
-			indexCacheBackend:      tsdb.IndexCacheBackendInMemory,
-		},
-		"[TSDB] blocks default sharding, memcached index cache": {
-			bucketStorageType:      "tsdb",
-			blocksShardingStrategy: "default",
-			indexCacheBackend:      tsdb.IndexCacheBackendMemcached,
-			chunkCacheBackend:      tsdb.CacheBackendMemcached,
-		},
-		"[TSDB] blocks shuffle sharding, memcached index cache": {
-			bucketStorageType:      "tsdb",
-			blocksShardingStrategy: "shuffle-sharding",
-			tenantShardSize:        1,
-			indexCacheBackend:      tsdb.IndexCacheBackendMemcached,
-			chunkCacheBackend:      tsdb.CacheBackendMemcached,
-		},
 		"[TSDB] blocks default sharding, inmemory index cache, bucket index enabled": {
 			bucketStorageType:      "tsdb",
 			blocksShardingStrategy: "default",
 			indexCacheBackend:      tsdb.IndexCacheBackendInMemory,
-			bucketIndexEnabled:     true,
 		},
 		"[TSDB] blocks shuffle sharding, memcached index cache, bucket index enabled": {
 			bucketStorageType:      "tsdb",
 			blocksShardingStrategy: "shuffle-sharding",
 			tenantShardSize:        1,
 			indexCacheBackend:      tsdb.IndexCacheBackendInMemory,
-			bucketIndexEnabled:     true,
 		},
 		"[TSDB] blocks default sharding, redis index cache, bucket index enabled": {
 			bucketStorageType:      "tsdb",
 			blocksShardingStrategy: "default",
 			indexCacheBackend:      tsdb.IndexCacheBackendRedis,
 			chunkCacheBackend:      tsdb.CacheBackendRedis,
-			bucketIndexEnabled:     true,
 		},
 		"[TSDB] blocks shuffle sharding, redis index cache, bucket index enabled": {
 			bucketStorageType:      "tsdb",
 			blocksShardingStrategy: "shuffle-sharding",
 			tenantShardSize:        1,
 			indexCacheBackend:      tsdb.IndexCacheBackendRedis,
 			chunkCacheBackend:      tsdb.CacheBackendRedis,
-			bucketIndexEnabled:     true,
 		},
 		"[TSDB] blocks sharding disabled, in-memory chunk cache": {
 			bucketStorageType:      "tsdb",
 			blocksShardingStrategy: "",
 			indexCacheBackend:      tsdb.IndexCacheBackendRedis,
 			chunkCacheBackend:      tsdb.CacheBackendInMemory,
-			bucketIndexEnabled:     true,
 		},
 		"[TSDB] blocks default sharding, in-memory chunk cache": {
 			bucketStorageType:      "tsdb",
 			blocksShardingStrategy: "default",
 			indexCacheBackend:      tsdb.IndexCacheBackendRedis,
 			chunkCacheBackend:      tsdb.CacheBackendInMemory,
-			bucketIndexEnabled:     true,
 		},
 		"[TSDB] blocks shuffle sharding, in-memory chunk cache": {
 			bucketStorageType:      "tsdb",
 			blocksShardingStrategy: "shuffle-sharding",
 			tenantShardSize:        1,
 			indexCacheBackend:      tsdb.IndexCacheBackendRedis,
 			chunkCacheBackend:      tsdb.CacheBackendInMemory,
-			bucketIndexEnabled:     true,
 		},
 		"[TSDB] block sharding disabled, multi-level chunk cache": {
 			bucketStorageType:      "tsdb",
 			blocksShardingStrategy: "",
 			indexCacheBackend:      tsdb.IndexCacheBackendRedis,
 			chunkCacheBackend:      fmt.Sprintf("%v,%v,%v", tsdb.CacheBackendInMemory, tsdb.CacheBackendMemcached, tsdb.CacheBackendRedis),
-			bucketIndexEnabled:     true,
 		},
 		"[TSDB] block default sharding, multi-level chunk cache": {
 			bucketStorageType:      "tsdb",
 			blocksShardingStrategy: "default",
 			indexCacheBackend:      tsdb.IndexCacheBackendRedis,
 			chunkCacheBackend:      fmt.Sprintf("%v,%v,%v", tsdb.CacheBackendInMemory, tsdb.CacheBackendMemcached, tsdb.CacheBackendRedis),
-			bucketIndexEnabled:     true,
 		},
 		"[TSDB] block shuffle sharding, multi-level chunk cache": {
 			bucketStorageType:      "tsdb",
 			blocksShardingStrategy: "shuffle-sharding",
 			tenantShardSize:        1,
 			indexCacheBackend:      tsdb.IndexCacheBackendRedis,
 			chunkCacheBackend:      fmt.Sprintf("%v,%v,%v", tsdb.CacheBackendInMemory, tsdb.CacheBackendMemcached, tsdb.CacheBackendRedis),
-			bucketIndexEnabled:     true,
 		},
 		//parquet bucket storage
-		"[Parquet] blocks sharding disabled, memcached parquet label cache, memcached chunks cache": {
-			bucketStorageType:      "parquet",
-			blocksShardingStrategy: "",
-			parquetLabelsCache:     tsdb.CacheBackendMemcached,
-			chunkCacheBackend:      tsdb.CacheBackendMemcached,
-		},
-		"[Parquet] blocks sharding disabled, multilevel parquet label cache (inmemory, memcached)": {
-			bucketStorageType:      "parquet",
-			blocksShardingStrategy: "",
-			parquetLabelsCache:     fmt.Sprintf("%v,%v", tsdb.CacheBackendInMemory, tsdb.CacheBackendMemcached),
-			chunkCacheBackend:      tsdb.CacheBackendMemcached,
-		},
-		"[Parquet] blocks sharding disabled, redis parquet label cache, redis chunks cache": {
-			bucketStorageType:      "parquet",
-			blocksShardingStrategy: "",
-			parquetLabelsCache:     tsdb.CacheBackendRedis,
-			chunkCacheBackend:      tsdb.CacheBackendRedis,
-		},
-		"[Parquet] blocks sharding disabled, multilevel parquet label cache cache (inmemory, redis), redis chunks cache": {
-			bucketStorageType:      "parquet",
-			blocksShardingStrategy: "",
-			parquetLabelsCache:     fmt.Sprintf("%v,%v", tsdb.CacheBackendInMemory, tsdb.CacheBackendRedis),
-			chunkCacheBackend:      tsdb.CacheBackendRedis,
-		},
-		"[Parquet] blocks default sharding, inmemory parquet label cache": {
-			bucketStorageType:      "parquet",
-			blocksShardingStrategy: "default",
-			parquetLabelsCache:     tsdb.CacheBackendInMemory,
-		},
-		"[Parquet] blocks default sharding, memcached parquet label cache, memcached chunks cache": {
-			bucketStorageType:      "parquet",
-			blocksShardingStrategy: "default",
-			parquetLabelsCache:     tsdb.CacheBackendMemcached,
-			chunkCacheBackend:      tsdb.CacheBackendMemcached,
-		},
-		"[Parquet] blocks shuffle sharding, memcached parquet label cache, memcached chunks cache": {
-			bucketStorageType:      "parquet",
-			blocksShardingStrategy: "shuffle-sharding",
-			tenantShardSize:        1,
-			parquetLabelsCache:     tsdb.CacheBackendMemcached,
-			chunkCacheBackend:      tsdb.CacheBackendMemcached,
-		},
 		"[Parquet] blocks default sharding, inmemory parquet label cache, bucket index enabled": {
 			bucketStorageType:      "parquet",
 			blocksShardingStrategy: "default",
 			parquetLabelsCache:     tsdb.CacheBackendInMemory,
-			bucketIndexEnabled:     true,
 		},
 		"[Parquet] blocks shuffle sharding, memcached parquet label cache, bucket index enabled": {
 			bucketStorageType:      "parquet",
 			blocksShardingStrategy: "shuffle-sharding",
 			tenantShardSize:        1,
 			parquetLabelsCache:     tsdb.CacheBackendInMemory,
-			bucketIndexEnabled:     true,
 		},
 		"[Parquet] blocks default sharding, redis parquet label cache, redis chunks cache, bucket index enabled": {
 			bucketStorageType:      "parquet",
 			blocksShardingStrategy: "default",
 			parquetLabelsCache:     tsdb.CacheBackendRedis,
 			chunkCacheBackend:      tsdb.CacheBackendRedis,
-			bucketIndexEnabled:     true,
 		},
 		"[Parquet] blocks shuffle sharding, redis parquet label cache, redis chunks cache, bucket index enabled": {
 			bucketStorageType:      "parquet",
 			blocksShardingStrategy: "shuffle-sharding",
 			tenantShardSize:        1,
 			parquetLabelsCache:     tsdb.CacheBackendRedis,
 			chunkCacheBackend:      tsdb.CacheBackendRedis,
-			bucketIndexEnabled:     true,
 		},
 		"[Parquet] blocks sharding disabled, redis parquet label cache, in-memory chunks cache, bucket index enabled": {
 			bucketStorageType:      "parquet",
 			blocksShardingStrategy: "",
 			parquetLabelsCache:     tsdb.CacheBackendRedis,
 			chunkCacheBackend:      tsdb.CacheBackendInMemory,
-			bucketIndexEnabled:     true,
 		},
 		"[Parquet] blocks default sharding, redis parquet label cache, in-memory chunk cache": {
 			bucketStorageType:      "parquet",
 			blocksShardingStrategy: "default",
 			parquetLabelsCache:     tsdb.CacheBackendRedis,
 			chunkCacheBackend:      tsdb.CacheBackendInMemory,
-			bucketIndexEnabled:     true,
 		},
 		"[Parquet] blocks shuffle sharding, redis parquet label cache, in-memory chunk cache, bucket index enabled": {
 			bucketStorageType:      "parquet",
 			blocksShardingStrategy: "shuffle-sharding",
 			tenantShardSize:        1,
 			parquetLabelsCache:     tsdb.CacheBackendRedis,
 			chunkCacheBackend:      tsdb.CacheBackendInMemory,
-			bucketIndexEnabled:     true,
 		},
 		"[Parquet] block sharding disabled, redis parquet label cache, multi-level chunk cache (in-memory, memcached, redis), bucket index enabled": {
 			bucketStorageType:      "parquet",
 			blocksShardingStrategy: "",
 			parquetLabelsCache:     tsdb.CacheBackendRedis,
 			chunkCacheBackend:      fmt.Sprintf("%v,%v,%v", tsdb.CacheBackendInMemory, tsdb.CacheBackendMemcached, tsdb.CacheBackendRedis),
-			bucketIndexEnabled:     true,
 		},
 		"[Parquet] block default sharding, redis parquet label cache, multi-level chunk cache (in-memory, memcached, redis), bucket index enabled": {
 			bucketStorageType:      "parquet",
 			blocksShardingStrategy: "default",
 			parquetLabelsCache:     tsdb.CacheBackendRedis,
 			chunkCacheBackend:      fmt.Sprintf("%v,%v,%v", tsdb.CacheBackendInMemory, tsdb.CacheBackendMemcached, tsdb.CacheBackendRedis),
-			bucketIndexEnabled:     true,
 		},
 		"[Parquet] block shuffle sharding, redis parquet label cache, multi-level chunk cache ((in-memory, memcached, redis), bucket index enabled)": {
 			bucketStorageType:      "parquet",
 			blocksShardingStrategy: "shuffle-sharding",
 			tenantShardSize:        1,
 			parquetLabelsCache:     tsdb.CacheBackendRedis,
 			chunkCacheBackend:      fmt.Sprintf("%v,%v,%v", tsdb.CacheBackendInMemory, tsdb.CacheBackendMemcached, tsdb.CacheBackendRedis),
-			bucketIndexEnabled:     true,
 		},
 	}
 
@@ -287,7 +182,6 @@ func TestQuerierWithBlocksStorageRunningInMicroservicesMode(t *testing.T) {
 					"-store-gateway.sharding-strategy":                  testCfg.blocksShardingStrategy,
 					"-store-gateway.tenant-shard-size":                  fmt.Sprintf("%d", testCfg.tenantShardSize),
 					"-querier.thanos-engine":                            strconv.FormatBool(thanosEngine),
-					"-blocks-storage.bucket-store.bucket-index.enabled": strconv.FormatBool(testCfg.bucketIndexEnabled),
 					"-blocks-storage.bucket-store.bucket-store-type":    testCfg.bucketStorageType,
 				})
 
@@ -399,12 +293,6 @@ func TestQuerierWithBlocksStorageRunningInMicroservicesMode(t *testing.T) {
 				require.NoError(t, ingester.WaitSumMetrics(e2e.Equals(3), "cortex_ingester_memory_series_created_total"))
 				require.NoError(t, ingester.WaitSumMetrics(e2e.Equals(2), "cortex_ingester_memory_series_removed_total"))
 
-				if testCfg.bucketIndexEnabled {
-					// Start the compactor to have the bucket index created before querying.
-					compactor := e2ecortex.NewCompactor("compactor", consul.NetworkHTTPEndpoint(), flags, "")
-					require.NoError(t, s.StartAndWaitReady(compactor))
-				}
-
 				switch testCfg.bucketStorageType {
 				case "tsdb":
 					// Wait until the store-gateway has synched the new uploaded blocks. When sharding is enabled
@@ -426,10 +314,6 @@ func TestQuerierWithBlocksStorageRunningInMicroservicesMode(t *testing.T) {
 						require.NoError(t, storeGateways.WaitSumMetrics(e2e.Equals(float64(1*storeGateways.NumInstances())), "cortex_bucket_stores_tenants_synced"))
 					}
 
-					if !testCfg.bucketIndexEnabled {
-						// Wait until the querier has discovered the uploaded blocks.
-						require.NoError(t, querier.WaitSumMetricsWithOptions(e2e.Equals(2), []string{"cortex_blocks_meta_synced"}, e2e.WaitMissingMetrics))
-					}
 				case "parquet":
 					// Wait until the parquet-converter convert blocks
 					require.NoError(t, parquetConverter.WaitSumMetricsWithOptions(e2e.Equals(float64(2)), []string{"cortex_parquet_converter_blocks_converted_total"}, e2e.WaitMissingMetrics))