diff --git a/CHANGELOG.md b/CHANGELOG.md index a2032c41cbd..eceb710eb85 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -116,6 +116,7 @@ * [BUGFIX] Alertmanager: Tighten per-tenant config validation to reject additional file-based settings. #7767 * [BUGFIX] Querier: Fix panic (`index out of range [-1]`) in the active request tracker when truncating a `match[]`/`query` value made entirely of invalid UTF-8 continuation bytes. The backwards scan for a rune boundary now stops at index 0 instead of underflowing. #7743 * [BUGFIX] Config: Fix CSV-list flags/YAML fields (e.g. `-compactor.enabled-tenants`) treating an explicitly empty string as a one-element list containing an empty tenant name instead of an empty list. #7714 +* [BUGFIX] Tenant Federation: Fix regex tenant federation dropping tenants when `-blocks-storage.users-scanner.cache-ttl` is set. The regex resolver sorted the user list returned by the users scanner in place, corrupting the scanner cache and progressively losing tenants on every sync until the cache expired. #7812 ## 1.21.1 2026-06-04 diff --git a/pkg/querier/tenantfederation/regex_resolver.go b/pkg/querier/tenantfederation/regex_resolver.go index 9fb2bc33aed..7d60d7dc7d6 100644 --- a/pkg/querier/tenantfederation/regex_resolver.go +++ b/pkg/querier/tenantfederation/regex_resolver.go @@ -128,7 +128,9 @@ func (r *RegexResolver) running(ctx context.Context) error { continue } - newUsers := append(active, deleting...) + newUsers := make([]string, 0, len(active)+len(deleting)) + newUsers = append(newUsers, active...) + newUsers = append(newUsers, deleting...) sort.Strings(newUsers) r.Lock() diff --git a/pkg/querier/tenantfederation/regex_resolver_test.go b/pkg/querier/tenantfederation/regex_resolver_test.go index 92285e835eb..542b37b202b 100644 --- a/pkg/querier/tenantfederation/regex_resolver_test.go +++ b/pkg/querier/tenantfederation/regex_resolver_test.go @@ -266,6 +266,59 @@ func Test_RegexResolver_Cache(t *testing.T) { } } +func Test_RegexResolver_DoesNotMutateScannerResult(t *testing.T) { + reg := prometheus.NewRegistry() + + activeTenants := []string{"b-1", "b-2", "b-3"} + deletingTenant := "a-del" + expected := []string{"a-del", "b-1", "b-2", "b-3"} + + bucketClient := &bucket.ClientMock{} + bucketClient.MockIter("", append([]string{deletingTenant}, activeTenants...), nil) + bucketClient.MockIter("__markers__", []string{"__markers__/" + deletingTenant + "/"}, nil) + bucketClient.MockExists(users.GetGlobalDeletionMarkPath(deletingTenant), true, nil) + for _, tenant := range activeTenants { + bucketClient.MockExists(users.GetGlobalDeletionMarkPath(tenant), false, nil) + bucketClient.MockExists(users.GetLocalDeletionMarkPath(tenant), false, nil) + } + + bucketClientFactory := func(ctx context.Context) (objstore.InstrumentedBucket, error) { + return bucketClient, nil + } + + // Caching makes the users scanner return the very same slices on every scan, so + // mutating them in place would corrupt the result of all the following scans. + usersScannerConfig := users.UsersScannerConfig{Strategy: users.UserScanStrategyList, CacheTTL: time.Hour} + tenantFederationConfig := Config{UserSyncInterval: 50 * time.Millisecond, MaxTenant: 0, RegexCacheSize: 10} + regexResolver, err := NewRegexResolver(usersScannerConfig, tenantFederationConfig, reg, bucketClientFactory, log.NewNopLogger()) + require.NoError(t, err) + + require.NoError(t, services.StartAndAwaitRunning(context.Background(), regexResolver)) + defer services.StopAndAwaitTerminated(context.Background(), regexResolver) //nolint:errcheck + + // Wait for the first scan. + test.Poll(t, 10*time.Second, true, func() any { + return testutil.ToFloat64(regexResolver.lastUpdateUserRun) > 0 + }) + firstRun := testutil.ToFloat64(regexResolver.lastUpdateUserRun) + + // Wait for several more scans. The corruption only shows up from the second scan + // on, once the scanner starts returning the previously mutated slices. + test.Poll(t, 10*time.Second, true, func() any { + return testutil.ToFloat64(regexResolver.lastUpdateUserRun) > firstRun + }) + + regexResolver.RLock() + knownUsers := append([]string(nil), regexResolver.knownUsers...) + regexResolver.RUnlock() + require.Equal(t, expected, knownUsers) + + ctx := user.InjectOrgID(context.Background(), "b-.+") + orgIDs, err := regexResolver.TenantIDs(ctx) + require.NoError(t, err) + require.Equal(t, activeTenants, orgIDs) +} + func Test_RegexResolver_CacheInvalidation(t *testing.T) { reg := prometheus.NewRegistry() initialTenants := []string{"user-1", "user-2"}