From 923311cc0ec148e66411d878b879abe1b24e9130 Mon Sep 17 00:00:00 2001 From: Rudrakh Panigrahi Date: Sat, 16 May 2026 17:30:46 +0530 Subject: [PATCH] feat: implement is_negative_hits in rate limit descriptor Signed-off-by: Rudrakh Panigrahi --- go.mod | 4 +- go.sum | 8 +- src/limiter/base_limiter.go | 17 +++- src/memcached/cache_impl.go | 43 ++++++++--- src/memcached/client.go | 1 + src/memcached/stats_collecting_client.go | 19 +++++ src/redis/fixed_cache_impl.go | 65 ++++++++++------ src/stats/manager.go | 1 + src/stats/manager_impl.go | 1 + src/utils/utilities.go | 14 +++- test/common/common.go | 11 +++ test/integration/integration_test.go | 46 +++++++++++ test/limiter/base_limiter_test.go | 43 ++++++++--- test/memcached/cache_impl_test.go | 98 ++++++++++++++++++++++++ test/mocks/memcached/client.go | 15 ++++ test/mocks/stats/manager.go | 1 + test/redis/fixed_cache_impl_test.go | 87 +++++++++++++++++++++ 17 files changed, 420 insertions(+), 54 deletions(-) diff --git a/go.mod b/go.mod index 03a1f6cc9..64b513412 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/bradfitz/gomemcache v0.0.0-20230905024940-24af94b03874 github.com/coocood/freecache v1.2.4 github.com/envoyproxy/go-control-plane v0.14.0 - github.com/envoyproxy/go-control-plane/envoy v1.36.0 + github.com/envoyproxy/go-control-plane/envoy v1.37.1-0.20260409083702-98966259b99a github.com/envoyproxy/go-control-plane/ratelimit v0.1.1-0.20260131204543-4ca8b9cded3e github.com/go-kit/log v0.2.1 github.com/golang/mock v1.6.0 @@ -50,7 +50,7 @@ require ( github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/cncf/xds/go v0.0.0-20251210132809-ee656c7534f5 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/envoyproxy/protoc-gen-validate v1.3.0 // indirect + github.com/envoyproxy/protoc-gen-validate v1.3.3 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/go-logfmt/logfmt v0.6.0 // indirect github.com/go-logr/logr v1.4.3 // indirect diff --git a/go.sum b/go.sum index b20cd0d75..b48e95135 100644 --- a/go.sum +++ b/go.sum @@ -35,13 +35,13 @@ github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.m github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= github.com/envoyproxy/go-control-plane v0.14.0 h1:hbG2kr4RuFj222B6+7T83thSPqLjwBIfQawTkC++2HA= github.com/envoyproxy/go-control-plane v0.14.0/go.mod h1:NcS5X47pLl/hfqxU70yPwL9ZMkUlwlKxtAohpi2wBEU= -github.com/envoyproxy/go-control-plane/envoy v1.36.0 h1:yg/JjO5E7ubRyKX3m07GF3reDNEnfOboJ0QySbH736g= -github.com/envoyproxy/go-control-plane/envoy v1.36.0/go.mod h1:ty89S1YCCVruQAm9OtKeEkQLTb+Lkz0k8v9W0Oxsv98= +github.com/envoyproxy/go-control-plane/envoy v1.37.1-0.20260409083702-98966259b99a h1:qrP4J6AWJ9yd6CINhPMRL/MbFXNiV7qimRsCDTOV0a0= +github.com/envoyproxy/go-control-plane/envoy v1.37.1-0.20260409083702-98966259b99a/go.mod h1:5yRfenlmRH8sxKrhXyiFtK8BDz3syDWcFm81rkCcATM= github.com/envoyproxy/go-control-plane/ratelimit v0.1.1-0.20260131204543-4ca8b9cded3e h1:EHL6eLDhQduyYGEKh+QSXE7s7Yhg/hpeeHFT0ET0gBw= github.com/envoyproxy/go-control-plane/ratelimit v0.1.1-0.20260131204543-4ca8b9cded3e/go.mod h1:buWyXJdrI6ayYbeGm3upu3Qf/qHHrdWfUHKnVrTD+vM= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= -github.com/envoyproxy/protoc-gen-validate v1.3.0 h1:TvGH1wof4H33rezVKWSpqKz5NXWg5VPuZ0uONDT6eb4= -github.com/envoyproxy/protoc-gen-validate v1.3.0/go.mod h1:HvYl7zwPa5mffgyeTUHA9zHIH36nmrm7oCbo4YKoSWA= +github.com/envoyproxy/protoc-gen-validate v1.3.3 h1:MVQghNeW+LZcmXe7SY1V36Z+WFMDjpqGAGacLe2T0ds= +github.com/envoyproxy/protoc-gen-validate v1.3.3/go.mod h1:TsndJ/ngyIdQRhMcVVGDDHINPLWB7C82oDArY51KfB0= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= diff --git a/src/limiter/base_limiter.go b/src/limiter/base_limiter.go index 3ee5ce8e5..304a60291 100644 --- a/src/limiter/base_limiter.go +++ b/src/limiter/base_limiter.go @@ -14,6 +14,15 @@ import ( "github.com/envoyproxy/ratelimit/src/utils" ) +// DecrementScript atomically decrements a rate limit counter, floored at 0. +const DecrementScript = ` +local current = tonumber(redis.call('GET', KEYS[1]) or '0') -- get current count, default 0 +local new_val = math.max(0, current - tonumber(ARGV[1])) -- subtract hits, floor at 0 +redis.call('SET', KEYS[1], tostring(math.floor(new_val))) -- persist new value +redis.call('EXPIRE', KEYS[1], tonumber(ARGV[2])) -- reset TTL +return new_val -- return count after decrement +` + type BaseRateLimiter struct { timeSource utils.TimeSource JitterRand *rand.Rand @@ -44,7 +53,7 @@ func NewRateLimitInfo(limit *config.RateLimit, limitBeforeIncrease uint64, limit // Generates cache keys for given rate limit request. Each cache key is represented by a concatenation of // domain, descriptor and current timestamp. func (this *BaseRateLimiter) GenerateCacheKeys(request *pb.RateLimitRequest, - limits []*config.RateLimit, hitsAddends []uint64, + limits []*config.RateLimit, hitsAddends []utils.HitsAddend, ) []CacheKey { assert.Assert(len(request.Descriptors) == len(limits)) cacheKeys := make([]CacheKey, len(request.Descriptors)) @@ -55,7 +64,11 @@ func (this *BaseRateLimiter) GenerateCacheKeys(request *pb.RateLimitRequest, cacheKeys[i] = this.cacheKeyGenerator.GenerateCacheKey(request.Domain, request.Descriptors[i], limits[i], now) // Increase statistics for limits hit by their respective requests. if limits[i] != nil { - limits[i].Stats.TotalHits.Add(hitsAddends[i]) + if hitsAddends[i].IsNegative { + limits[i].Stats.TotalNegativeHits.Add(hitsAddends[i].Value) + } else { + limits[i].Stats.TotalHits.Add(hitsAddends[i].Value) + } } } return cacheKeys diff --git a/src/memcached/cache_impl.go b/src/memcached/cache_impl.go index c8bcd0774..e66d1c221 100644 --- a/src/memcached/cache_impl.go +++ b/src/memcached/cache_impl.go @@ -85,6 +85,12 @@ func (this *rateLimitMemcacheImpl) DoLimit( continue } + // Negative hits skip the over-limit check — they always proceed. + if hitsAddends[i].IsNegative { + keysToGet = append(keysToGet, cacheKey.Key) + continue + } + // Check if key is over the limit in local cache. if this.baseRateLimiter.IsOverLimitWithLocalCache(cacheKey.Key) { isOverLimitWithLocalCache[i] = true @@ -129,15 +135,25 @@ func (this *rateLimitMemcacheImpl) DoLimit( } else { limitBeforeIncrease = uint64(decoded) } - } - limitAfterIncrease := limitBeforeIncrease + hitsAddends[i] + if hitsAddends[i].IsNegative { + // Predict the post-decrement value (guard against uint64 underflow). + var limitAfterDecrease uint64 + if limitBeforeIncrease > hitsAddends[i].Value { + limitAfterDecrease = limitBeforeIncrease - hitsAddends[i].Value + } + responseDescriptorStatuses[i] = this.baseRateLimiter.GetResponseDescriptorStatus(cacheKey.Key, + limiter.NewRateLimitInfo(limits[i], limitAfterDecrease, limitAfterDecrease, 0, 0), + false, 0) + } else { + limitAfterIncrease := limitBeforeIncrease + hitsAddends[i].Value - limitInfo := limiter.NewRateLimitInfo(limits[i], limitBeforeIncrease, limitAfterIncrease, 0, 0) + limitInfo := limiter.NewRateLimitInfo(limits[i], limitBeforeIncrease, limitAfterIncrease, 0, 0) - responseDescriptorStatuses[i] = this.baseRateLimiter.GetResponseDescriptorStatus(cacheKey.Key, - limitInfo, isOverLimitWithLocalCache[i], hitsAddends[i]) + responseDescriptorStatuses[i] = this.baseRateLimiter.GetResponseDescriptorStatus(cacheKey.Key, + limitInfo, isOverLimitWithLocalCache[i], hitsAddends[i].Value) + } } this.waitGroup.Add(1) @@ -150,7 +166,7 @@ func (this *rateLimitMemcacheImpl) DoLimit( } func (this *rateLimitMemcacheImpl) increaseAsync(cacheKeys []limiter.CacheKey, isOverLimitWithLocalCache []bool, - limits []*config.RateLimit, hitsAddends []uint64, + limits []*config.RateLimit, hitsAddends []utils.HitsAddend, ) { defer this.waitGroup.Done() for i, cacheKey := range cacheKeys { @@ -158,7 +174,16 @@ func (this *rateLimitMemcacheImpl) increaseAsync(cacheKeys []limiter.CacheKey, i continue } - _, err := this.client.Increment(cacheKey.Key, hitsAddends[i]) + if hitsAddends[i].IsNegative { + // Memcached Decrement natively floors at 0. + _, err := this.client.Decrement(cacheKey.Key, hitsAddends[i].Value) + if err != nil && err != memcache.ErrCacheMiss { + logger.Errorf("Failed to decrement key %s: %s", cacheKey.Key, err) + } + continue + } + + _, err := this.client.Increment(cacheKey.Key, hitsAddends[i].Value) if err == memcache.ErrCacheMiss { expirationSeconds := utils.UnitToDivider(limits[i].Limit.Unit) if this.expirationJitterMaxSeconds > 0 { @@ -168,13 +193,13 @@ func (this *rateLimitMemcacheImpl) increaseAsync(cacheKeys []limiter.CacheKey, i // Need to add instead of increment. err = this.client.Add(&memcache.Item{ Key: cacheKey.Key, - Value: []byte(strconv.FormatUint(hitsAddends[i], 10)), + Value: []byte(strconv.FormatUint(hitsAddends[i].Value, 10)), Expiration: int32(expirationSeconds), }) if err == memcache.ErrNotStored { // There was a race condition to do this add. We should be able to increment // now instead. - _, err := this.client.Increment(cacheKey.Key, hitsAddends[i]) + _, err := this.client.Increment(cacheKey.Key, hitsAddends[i].Value) if err != nil { logger.Errorf("Failed to increment key %s after failing to add: %s", cacheKey.Key, err) continue diff --git a/src/memcached/client.go b/src/memcached/client.go index e80902692..fb858cf48 100644 --- a/src/memcached/client.go +++ b/src/memcached/client.go @@ -17,5 +17,6 @@ var _ Client = (*memcache.Client)(nil) type Client interface { GetMulti(keys []string) (map[string]*memcache.Item, error) Increment(key string, delta uint64) (newValue uint64, err error) + Decrement(key string, delta uint64) (newValue uint64, err error) Add(item *memcache.Item) error } diff --git a/src/memcached/stats_collecting_client.go b/src/memcached/stats_collecting_client.go index 12b67bad5..afe71947d 100644 --- a/src/memcached/stats_collecting_client.go +++ b/src/memcached/stats_collecting_client.go @@ -13,6 +13,9 @@ type statsCollectingClient struct { incrementSuccess stats.Counter incrementMiss stats.Counter incrementError stats.Counter + decrementSuccess stats.Counter + decrementMiss stats.Counter + decrementError stats.Counter addSuccess stats.Counter addError stats.Counter addNotStored stats.Counter @@ -28,6 +31,9 @@ func CollectStats(c Client, scope stats.Scope) Client { incrementSuccess: scope.NewCounterWithTags("increment", map[string]string{"code": "success"}), incrementMiss: scope.NewCounterWithTags("increment", map[string]string{"code": "miss"}), incrementError: scope.NewCounterWithTags("increment", map[string]string{"code": "error"}), + decrementSuccess: scope.NewCounterWithTags("decrement", map[string]string{"code": "success"}), + decrementMiss: scope.NewCounterWithTags("decrement", map[string]string{"code": "miss"}), + decrementError: scope.NewCounterWithTags("decrement", map[string]string{"code": "error"}), addSuccess: scope.NewCounterWithTags("add", map[string]string{"code": "success"}), addError: scope.NewCounterWithTags("add", map[string]string{"code": "error"}), addNotStored: scope.NewCounterWithTags("add", map[string]string{"code": "not_stored"}), @@ -64,6 +70,19 @@ func (scc statsCollectingClient) Increment(key string, delta uint64) (newValue u return } +func (scc statsCollectingClient) Decrement(key string, delta uint64) (newValue uint64, err error) { + newValue, err = scc.c.Decrement(key, delta) + switch err { + case memcache.ErrCacheMiss: + scc.decrementMiss.Inc() + case nil: + scc.decrementSuccess.Inc() + default: + scc.decrementError.Inc() + } + return +} + func (scc statsCollectingClient) Add(item *memcache.Item) error { err := scc.c.Add(item) diff --git a/src/redis/fixed_cache_impl.go b/src/redis/fixed_cache_impl.go index 1034d3beb..df4185658 100644 --- a/src/redis/fixed_cache_impl.go +++ b/src/redis/fixed_cache_impl.go @@ -37,11 +37,28 @@ func pipelineAppend(client Client, pipeline *Pipeline, key string, hitsAddend ui *pipeline = client.PipeAppend(*pipeline, nil, "EXPIRE", key, expirationSeconds) } +func pipelineAppendDecrement(client Client, pipeline *Pipeline, key string, hitsAddend uint64, result *uint64, expirationSeconds int64) { + *pipeline = client.PipeAppend(*pipeline, result, "EVAL", limiter.DecrementScript, 1, key, hitsAddend, expirationSeconds) +} + +func (this *fixedRateLimitCacheImpl) selectPipeline(cacheKey limiter.CacheKey, pipeline *Pipeline, perSecondPipeline *Pipeline) (Client, *Pipeline) { + if this.perSecondClient != nil && cacheKey.PerSecond { + if *perSecondPipeline == nil { + *perSecondPipeline = Pipeline{} + } + return this.perSecondClient, perSecondPipeline + } + if *pipeline == nil { + *pipeline = Pipeline{} + } + return this.client, pipeline +} + func pipelineAppendtoGet(client Client, pipeline *Pipeline, key string, result *uint64) { *pipeline = client.PipeAppend(*pipeline, result, "GET", key) } -func (this *fixedRateLimitCacheImpl) getHitsAddend(hitsAddend uint64, isCacheKeyOverlimit, isCacheKeyNearlimit, +func (this *fixedRateLimitCacheImpl) getHitsAddendValue(hitsAddend uint64, isCacheKeyOverlimit, isCacheKeyNearlimit, isNearLimt bool, ) uint64 { // If stopCacheKeyIncrementWhenOverlimit is false, then we always increment the cache key. @@ -94,8 +111,9 @@ func (this *fixedRateLimitCacheImpl) DoLimit( isCacheKeyNearlimit := false // Check if any of the keys are already to the over limit in cache. + // Negative hits (decrements) skip this check — they always proceed. for i, cacheKey := range cacheKeys { - if cacheKey.Key == "" { + if cacheKey.Key == "" || hitsAddends[i].IsNegative { continue } @@ -116,7 +134,7 @@ func (this *fixedRateLimitCacheImpl) DoLimit( // then we check if any of the keys are near limit in redis cache. if this.stopCacheKeyIncrementWhenOverlimit && !isCacheKeyOverlimit { for i, cacheKey := range cacheKeys { - if cacheKey.Key == "" { + if cacheKey.Key == "" || hitsAddends[i].IsNegative { continue } @@ -141,12 +159,12 @@ func (this *fixedRateLimitCacheImpl) DoLimit( } for i, cacheKey := range cacheKeys { - if cacheKey.Key == "" { + if cacheKey.Key == "" || hitsAddends[i].IsNegative { continue } // Now fetch the pipeline. limitBeforeIncrease := currentCount[i] - limitAfterIncrease := limitBeforeIncrease + hitsAddends[i] + limitAfterIncrease := limitBeforeIncrease + hitsAddends[i].Value limitInfo := limiter.NewRateLimitInfo(limits[i], limitBeforeIncrease, limitAfterIncrease, 0, 0) @@ -157,7 +175,7 @@ func (this *fixedRateLimitCacheImpl) DoLimit( } } - // Now, actually setup the pipeline to increase the usage of cache key, skipping empty cache keys. + // Now, actually setup the pipeline to increase/decrease the usage of cache key, skipping empty cache keys. for i, cacheKey := range cacheKeys { if cacheKey.Key == "" || overlimitIndexes[i] { continue @@ -170,19 +188,12 @@ func (this *fixedRateLimitCacheImpl) DoLimit( expirationSeconds += this.baseRateLimiter.JitterRand.Int63n(this.baseRateLimiter.ExpirationJitterMaxSeconds) } - // Use the perSecondConn if it is not nil and the cacheKey represents a per second Limit. - if this.perSecondClient != nil && cacheKey.PerSecond { - if perSecondPipeline == nil { - perSecondPipeline = Pipeline{} - } - pipelineAppend(this.perSecondClient, &perSecondPipeline, cacheKey.Key, this.getHitsAddend(hitsAddends[i], - isCacheKeyOverlimit, isCacheKeyNearlimit, nearlimitIndexes[i]), &results[i], expirationSeconds) + client, p := this.selectPipeline(cacheKey, &pipeline, &perSecondPipeline) + if hitsAddends[i].IsNegative { + pipelineAppendDecrement(client, p, cacheKey.Key, hitsAddends[i].Value, &results[i], expirationSeconds) } else { - if pipeline == nil { - pipeline = Pipeline{} - } - pipelineAppend(this.client, &pipeline, cacheKey.Key, this.getHitsAddend(hitsAddends[i], isCacheKeyOverlimit, - isCacheKeyNearlimit, nearlimitIndexes[i]), &results[i], expirationSeconds) + pipelineAppend(client, p, cacheKey.Key, this.getHitsAddendValue(hitsAddends[i].Value, + isCacheKeyOverlimit, isCacheKeyNearlimit, nearlimitIndexes[i]), &results[i], expirationSeconds) } } @@ -206,14 +217,20 @@ func (this *fixedRateLimitCacheImpl) DoLimit( responseDescriptorStatuses := make([]*pb.RateLimitResponse_DescriptorStatus, len(request.Descriptors)) for i, cacheKey := range cacheKeys { + if hitsAddends[i].IsNegative { + // Negative hits always return OK with the remaining capacity. + responseDescriptorStatuses[i] = this.baseRateLimiter.GetResponseDescriptorStatus(cacheKey.Key, + limiter.NewRateLimitInfo(limits[i], results[i], results[i], 0, 0), + false, 0) + } else { + limitAfterIncrease := results[i] + limitBeforeIncrease := limitAfterIncrease - hitsAddends[i].Value - limitAfterIncrease := results[i] - limitBeforeIncrease := limitAfterIncrease - hitsAddends[i] - - limitInfo := limiter.NewRateLimitInfo(limits[i], limitBeforeIncrease, limitAfterIncrease, 0, 0) + limitInfo := limiter.NewRateLimitInfo(limits[i], limitBeforeIncrease, limitAfterIncrease, 0, 0) - responseDescriptorStatuses[i] = this.baseRateLimiter.GetResponseDescriptorStatus(cacheKey.Key, - limitInfo, isOverLimitWithLocalCache[i], hitsAddends[i]) + responseDescriptorStatuses[i] = this.baseRateLimiter.GetResponseDescriptorStatus(cacheKey.Key, + limitInfo, isOverLimitWithLocalCache[i], hitsAddends[i].Value) + } } return responseDescriptorStatuses diff --git a/src/stats/manager.go b/src/stats/manager.go index 43d72d59b..a63a7b5c9 100644 --- a/src/stats/manager.go +++ b/src/stats/manager.go @@ -54,6 +54,7 @@ type RateLimitStats struct { OverLimitWithLocalCache gostats.Counter WithinLimit gostats.Counter ShadowMode gostats.Counter + TotalNegativeHits gostats.Counter } // Stats for a domain entry diff --git a/src/stats/manager_impl.go b/src/stats/manager_impl.go index ee0cf0135..e629064c7 100644 --- a/src/stats/manager_impl.go +++ b/src/stats/manager_impl.go @@ -36,6 +36,7 @@ func (this *ManagerImpl) NewStats(key string) RateLimitStats { ret.OverLimitWithLocalCache = this.rlStatsScope.NewCounter(key + ".over_limit_with_local_cache") ret.WithinLimit = this.rlStatsScope.NewCounter(key + ".within_limit") ret.ShadowMode = this.rlStatsScope.NewCounter(key + ".shadow_mode") + ret.TotalNegativeHits = this.rlStatsScope.NewCounter(key + ".total_negative_hits") return ret } diff --git a/src/utils/utilities.go b/src/utils/utilities.go index fb398b764..b650e9e5c 100644 --- a/src/utils/utilities.go +++ b/src/utils/utilities.go @@ -72,19 +72,25 @@ func SanitizeStatName(s string) string { }) } -func GetHitsAddends(request *pb.RateLimitRequest) []uint64 { - hitsAddends := make([]uint64, len(request.Descriptors)) +type HitsAddend struct { + Value uint64 + IsNegative bool +} + +func GetHitsAddends(request *pb.RateLimitRequest) []HitsAddend { + hitsAddends := make([]HitsAddend, len(request.Descriptors)) for i, descriptor := range request.Descriptors { if descriptor.HitsAddend != nil { // If the per descriptor hits_addend is set, use that. It allows to be zero. The zero value is // means check only by no increment the hits. - hitsAddends[i] = descriptor.HitsAddend.Value + hitsAddends[i].Value = descriptor.HitsAddend.Value } else { // If the per descriptor hits_addend is not set, use the request's hits_addend. If the value is // zero (default value if not specified by the caller), use 1 for backward compatibility. - hitsAddends[i] = uint64(max(1, uint64(request.HitsAddend))) + hitsAddends[i].Value = uint64(max(1, uint64(request.HitsAddend))) } + hitsAddends[i].IsNegative = descriptor.GetIsNegativeHits() } return hitsAddends } diff --git a/test/common/common.go b/test/common/common.go index 6a414ca09..4f2187197 100644 --- a/test/common/common.go +++ b/test/common/common.go @@ -83,6 +83,17 @@ func NewRateLimitRequestWithPerDescriptorHitsAddend(domain string, descriptors [ return request } +func NewRateLimitRequestWithNegativeHits(domain string, descriptors [][][2]string, + hitsAddends []uint64, negativeFlags []bool, +) *pb.RateLimitRequest { + request := NewRateLimitRequest(domain, descriptors, 1) + for i, hitsAddend := range hitsAddends { + request.Descriptors[i].HitsAddend = &wrapperspb.UInt64Value{Value: hitsAddend} + request.Descriptors[i].IsNegativeHits = negativeFlags[i] + } + return request +} + func AssertProtoEqual(assert *assert.Assertions, expected proto.Message, actual proto.Message) { assert.True(proto.Equal(expected, actual), fmt.Sprintf("These two protobuf messages are not equal:\nexpected: %v\nactual: %v", expected, actual)) diff --git a/test/integration/integration_test.go b/test/integration/integration_test.go index df5e6fe6d..d7d2c512d 100644 --- a/test/integration/integration_test.go +++ b/test/integration/integration_test.go @@ -89,6 +89,52 @@ func makeSimpleRedisSettingsWithStopCacheKeyIncrementWhenOverlimit(redisPort int return s } +func TestNegativeHitsIntegration(t *testing.T) { + common.WithMultiRedis(t, []common.RedisConfig{ + {Port: 6383}, + }, func() { + t.Run("Redis", testNegativeHits(makeSimpleRedisSettings(6383, 6380, false, 0))) + }) +} + +func testNegativeHits(s settings.Settings) func(*testing.T) { + return func(t *testing.T) { + runner := startTestRunner(t, s) + defer runner.Stop() + + assert := assert.New(t) + conn, err := grpc.Dial(fmt.Sprintf("localhost:%v", s.GrpcPort), grpc.WithInsecure()) + assert.NoError(err) + defer conn.Close() + c := pb.NewRateLimitServiceClient(conn) + + // Consume 5 hits from a limit of 50/second. + response, err := c.ShouldRateLimit( + context.Background(), + common.NewRateLimitRequest("basic", [][][2]string{{{"key1", "foo"}}}, 5)) + assert.NoError(err) + assert.Equal(pb.RateLimitResponse_OK, response.OverallCode) + assert.Equal(uint32(45), response.Statuses[0].LimitRemaining) + + // Now send a negative hits request to refill 3 tokens. + response, err = c.ShouldRateLimit( + context.Background(), + common.NewRateLimitRequestWithNegativeHits("basic", [][][2]string{{{"key1", "foo"}}}, []uint64{3}, []bool{true})) + assert.NoError(err) + assert.Equal(pb.RateLimitResponse_OK, response.OverallCode) + // Counter was 5, decremented by 3 = 2. Remaining = 50 - 2 = 48. + assert.Equal(uint32(48), response.Statuses[0].LimitRemaining) + + // Verify by consuming 1 more hit - remaining should be 47. + response, err = c.ShouldRateLimit( + context.Background(), + common.NewRateLimitRequest("basic", [][][2]string{{{"key1", "foo"}}}, 1)) + assert.NoError(err) + assert.Equal(pb.RateLimitResponse_OK, response.OverallCode) + assert.Equal(uint32(47), response.Statuses[0].LimitRemaining) + } +} + func TestBasicConfig(t *testing.T) { common.WithMultiRedis(t, []common.RedisConfig{ {Port: 6383}, diff --git a/test/limiter/base_limiter_test.go b/test/limiter/base_limiter_test.go index 7bc404079..3c8c4e33b 100644 --- a/test/limiter/base_limiter_test.go +++ b/test/limiter/base_limiter_test.go @@ -14,6 +14,7 @@ import ( "github.com/envoyproxy/ratelimit/src/config" "github.com/envoyproxy/ratelimit/src/limiter" + "github.com/envoyproxy/ratelimit/src/utils" "github.com/envoyproxy/ratelimit/test/common" mock_utils "github.com/envoyproxy/ratelimit/test/mocks/utils" ) @@ -31,12 +32,36 @@ func TestGenerateCacheKeys(t *testing.T) { request := common.NewRateLimitRequest("domain", [][][2]string{{{"key", "value"}}}, 1) limits := []*config.RateLimit{config.NewRateLimit(10, pb.RateLimitResponse_RateLimit_SECOND, sm.NewStats("key_value"), false, false, false, "", nil, false)} assert.Equal(uint64(0), limits[0].Stats.TotalHits.Value()) - cacheKeys := baseRateLimit.GenerateCacheKeys(request, limits, []uint64{1}) + cacheKeys := baseRateLimit.GenerateCacheKeys(request, limits, []utils.HitsAddend{{Value: 1}}) assert.Equal(1, len(cacheKeys)) assert.Equal("domain_key_value_1234", cacheKeys[0].Key) assert.Equal(uint64(1), limits[0].Stats.TotalHits.Value()) } +func TestGenerateCacheKeysNegativeHits(t *testing.T) { + assert := assert.New(t) + controller := gomock.NewController(t) + defer controller.Finish() + timeSource := mock_utils.NewMockTimeSource(controller) + jitterSource := mock_utils.NewMockJitterRandSource(controller) + statsStore := stats.NewStore(stats.NewNullSink(), false) + sm := mockstats.NewMockStatManager(statsStore) + timeSource.EXPECT().UnixNow().Return(int64(1234)) + baseRateLimit := limiter.NewBaseRateLimit(timeSource, rand.New(jitterSource), 3600, nil, 0.8, "", sm) + request := common.NewRateLimitRequest("domain", [][][2]string{{{"key", "value"}}}, 1) + limits := []*config.RateLimit{config.NewRateLimit(10, pb.RateLimitResponse_RateLimit_SECOND, sm.NewStats("key_value"), false, false, false, "", nil, false)} + assert.Equal(uint64(0), limits[0].Stats.TotalHits.Value()) + assert.Equal(uint64(0), limits[0].Stats.TotalNegativeHits.Value()) + + cacheKeys := baseRateLimit.GenerateCacheKeys(request, limits, []utils.HitsAddend{{Value: 3, IsNegative: true}}) + assert.Equal(1, len(cacheKeys)) + assert.Equal("domain_key_value_1234", cacheKeys[0].Key) + // TotalHits should NOT be incremented for negative hits. + assert.Equal(uint64(0), limits[0].Stats.TotalHits.Value()) + // TotalNegativeHits should be incremented. + assert.Equal(uint64(3), limits[0].Stats.TotalNegativeHits.Value()) +} + func TestGenerateCacheKeysPrefix(t *testing.T) { assert := assert.New(t) controller := gomock.NewController(t) @@ -50,7 +75,7 @@ func TestGenerateCacheKeysPrefix(t *testing.T) { request := common.NewRateLimitRequest("domain", [][][2]string{{{"key", "value"}}}, 1) limits := []*config.RateLimit{config.NewRateLimit(10, pb.RateLimitResponse_RateLimit_SECOND, sm.NewStats("key_value"), false, false, false, "", nil, false)} assert.Equal(uint64(0), limits[0].Stats.TotalHits.Value()) - cacheKeys := baseRateLimit.GenerateCacheKeys(request, limits, []uint64{1}) + cacheKeys := baseRateLimit.GenerateCacheKeys(request, limits, []utils.HitsAddend{{Value: 1}}) assert.Equal(1, len(cacheKeys)) assert.Equal("prefix:domain_key_value_1234", cacheKeys[0].Key) assert.Equal(uint64(1), limits[0].Stats.TotalHits.Value()) @@ -73,13 +98,13 @@ func TestGenerateCacheKeysWithShareThreshold(t *testing.T) { request1 := common.NewRateLimitRequest("domain", [][][2]string{{{"files", "files/a.pdf"}}}, 1) limits1 := []*config.RateLimit{limit} - cacheKeys1 := baseRateLimit.GenerateCacheKeys(request1, limits1, []uint64{1}) + cacheKeys1 := baseRateLimit.GenerateCacheKeys(request1, limits1, []utils.HitsAddend{{Value: 1}}) assert.Equal(1, len(cacheKeys1)) assert.Equal("domain_files_files/*_1234", cacheKeys1[0].Key) request2 := common.NewRateLimitRequest("domain", [][][2]string{{{"files", "files/b.csv"}}}, 1) limits2 := []*config.RateLimit{limit} - cacheKeys2 := baseRateLimit.GenerateCacheKeys(request2, limits2, []uint64{1}) + cacheKeys2 := baseRateLimit.GenerateCacheKeys(request2, limits2, []utils.HitsAddend{{Value: 1}}) assert.Equal(1, len(cacheKeys2)) // Should generate the same cache key as the first request assert.Equal("domain_files_files/*_1234", cacheKeys2[0].Key) @@ -89,7 +114,7 @@ func TestGenerateCacheKeysWithShareThreshold(t *testing.T) { testValues := []string{"files/c.txt", "files/d.json", "files/e.xml", "files/subdir/f.txt"} for _, value := range testValues { request := common.NewRateLimitRequest("domain", [][][2]string{{{"files", value}}}, 1) - cacheKeys := baseRateLimit.GenerateCacheKeys(request, limits1, []uint64{1}) + cacheKeys := baseRateLimit.GenerateCacheKeys(request, limits1, []utils.HitsAddend{{Value: 1}}) assert.Equal(1, len(cacheKeys)) assert.Equal("domain_files_files/*_1234", cacheKeys[0].Key, "Value %s should generate same cache key", value) } @@ -102,14 +127,14 @@ func TestGenerateCacheKeysWithShareThreshold(t *testing.T) { {{"parent", "value1"}, {"files", "nested/file1.txt"}}, }, 1) limits3a := []*config.RateLimit{limitNested} - cacheKeys3a := baseRateLimit.GenerateCacheKeys(request3a, limits3a, []uint64{1}) + cacheKeys3a := baseRateLimit.GenerateCacheKeys(request3a, limits3a, []utils.HitsAddend{{Value: 1}}) assert.Equal(1, len(cacheKeys3a)) assert.Equal("domain_parent_value1_files_nested/*_1234", cacheKeys3a[0].Key) request3b := common.NewRateLimitRequest("domain", [][][2]string{ {{"parent", "value1"}, {"files", "nested/file2.csv"}}, }, 1) - cacheKeys3b := baseRateLimit.GenerateCacheKeys(request3b, limits3a, []uint64{1}) + cacheKeys3b := baseRateLimit.GenerateCacheKeys(request3b, limits3a, []utils.HitsAddend{{Value: 1}}) assert.Equal(1, len(cacheKeys3b)) // Should generate the same cache key despite different file values assert.Equal("domain_parent_value1_files_nested/*_1234", cacheKeys3b[0].Key) @@ -123,14 +148,14 @@ func TestGenerateCacheKeysWithShareThreshold(t *testing.T) { {{"files", "top/file1.txt"}, {"files", "nested/sub1.txt"}}, }, 1) limits4a := []*config.RateLimit{limitMulti} - cacheKeys4a := baseRateLimit.GenerateCacheKeys(request4a, limits4a, []uint64{1}) + cacheKeys4a := baseRateLimit.GenerateCacheKeys(request4a, limits4a, []utils.HitsAddend{{Value: 1}}) assert.Equal(1, len(cacheKeys4a)) assert.Equal("domain_files_top/*_files_nested/*_1234", cacheKeys4a[0].Key) request4b := common.NewRateLimitRequest("domain", [][][2]string{ {{"files", "top/file2.pdf"}, {"files", "nested/sub2.csv"}}, }, 1) - cacheKeys4b := baseRateLimit.GenerateCacheKeys(request4b, limits4a, []uint64{1}) + cacheKeys4b := baseRateLimit.GenerateCacheKeys(request4b, limits4a, []utils.HitsAddend{{Value: 1}}) assert.Equal(1, len(cacheKeys4b)) // Should generate the same cache key despite different values assert.Equal("domain_files_top/*_files_nested/*_1234", cacheKeys4b[0].Key) diff --git a/test/memcached/cache_impl_test.go b/test/memcached/cache_impl_test.go index 606a9d844..db26902d5 100644 --- a/test/memcached/cache_impl_test.go +++ b/test/memcached/cache_impl_test.go @@ -695,3 +695,101 @@ func getMultiResult(vals map[string]int) map[string]*memcache.Item { } return result } + +func TestMemcachedNegativeHits(t *testing.T) { + assert := assert.New(t) + controller := gomock.NewController(t) + defer controller.Finish() + + timeSource := mock_utils.NewMockTimeSource(controller) + client := mock_memcached.NewMockClient(controller) + statsStore := stats.NewStore(stats.NewNullSink(), false) + sm := mockstats.NewMockStatManager(statsStore) + cache := memcached.NewRateLimitCacheImpl(client, timeSource, nil, 0, nil, sm, 0.8, "") + + // Counter at 7, requesting -3. Memcached Decrement floors at 0 natively. + timeSource.EXPECT().UnixNow().Return(int64(1234)).MaxTimes(3) + client.EXPECT().GetMulti([]string{"domain_key_value_1234"}).Return( + getMultiResult(map[string]int{"domain_key_value_1234": 7}), nil, + ) + client.EXPECT().Decrement("domain_key_value_1234", uint64(3)).Return(uint64(4), nil) + + request := common.NewRateLimitRequestWithNegativeHits( + "domain", [][][2]string{{{"key", "value"}}}, []uint64{3}, []bool{true}) + limits := []*config.RateLimit{config.NewRateLimit(10, pb.RateLimitResponse_RateLimit_SECOND, sm.NewStats("key_value"), false, false, false, "", nil, false)} + + result := cache.DoLimit(context.Background(), request, limits) + cache.Flush() + + assert.Equal(pb.RateLimitResponse_OK, result[0].Code) + assert.Equal(uint64(6), uint64(result[0].LimitRemaining)) + assert.Equal(limits[0].Limit, result[0].CurrentLimit) + assert.Equal(uint64(0), limits[0].Stats.TotalHits.Value()) + assert.Equal(uint64(3), limits[0].Stats.TotalNegativeHits.Value()) + assert.Equal(uint64(0), limits[0].Stats.OverLimit.Value()) +} + +func TestMemcachedNegativeHitsFloorAtZero(t *testing.T) { + assert := assert.New(t) + controller := gomock.NewController(t) + defer controller.Finish() + + timeSource := mock_utils.NewMockTimeSource(controller) + client := mock_memcached.NewMockClient(controller) + statsStore := stats.NewStore(stats.NewNullSink(), false) + sm := mockstats.NewMockStatManager(statsStore) + cache := memcached.NewRateLimitCacheImpl(client, timeSource, nil, 0, nil, sm, 0.8, "") + + // Counter at 2, requesting -5. Floor at 0. + timeSource.EXPECT().UnixNow().Return(int64(1234)).MaxTimes(3) + client.EXPECT().GetMulti([]string{"domain_key_value_1234"}).Return( + getMultiResult(map[string]int{"domain_key_value_1234": 2}), nil, + ) + client.EXPECT().Decrement("domain_key_value_1234", uint64(5)).Return(uint64(0), nil) + + request := common.NewRateLimitRequestWithNegativeHits( + "domain", [][][2]string{{{"key", "value"}}}, []uint64{5}, []bool{true}) + limits := []*config.RateLimit{config.NewRateLimit(10, pb.RateLimitResponse_RateLimit_SECOND, sm.NewStats("key_value"), false, false, false, "", nil, false)} + + result := cache.DoLimit(context.Background(), request, limits) + cache.Flush() + + assert.Equal(pb.RateLimitResponse_OK, result[0].Code) + assert.Equal(uint64(10), uint64(result[0].LimitRemaining)) + assert.Equal(uint64(0), limits[0].Stats.TotalHits.Value()) + assert.Equal(uint64(5), limits[0].Stats.TotalNegativeHits.Value()) +} + +func TestMemcachedNegativeHitsSkipsOverLimitCheck(t *testing.T) { + assert := assert.New(t) + controller := gomock.NewController(t) + defer controller.Finish() + + timeSource := mock_utils.NewMockTimeSource(controller) + client := mock_memcached.NewMockClient(controller) + statsStore := stats.NewStore(stats.NewNullSink(), false) + sm := mockstats.NewMockStatManager(statsStore) + localCache := freecache.NewCache(100) + cache := memcached.NewRateLimitCacheImpl(client, timeSource, nil, 0, localCache, sm, 0.8, "") + + // Set the key as over-limit in local cache. + localCache.Set([]byte("domain_key_value_1234"), []byte{}, 60) + + // Negative hits should still proceed despite key being in over-limit local cache. + timeSource.EXPECT().UnixNow().Return(int64(1234)).MaxTimes(3) + client.EXPECT().GetMulti([]string{"domain_key_value_1234"}).Return( + getMultiResult(map[string]int{"domain_key_value_1234": 10}), nil, + ) + client.EXPECT().Decrement("domain_key_value_1234", uint64(2)).Return(uint64(8), nil) + + request := common.NewRateLimitRequestWithNegativeHits( + "domain", [][][2]string{{{"key", "value"}}}, []uint64{2}, []bool{true}) + limits := []*config.RateLimit{config.NewRateLimit(10, pb.RateLimitResponse_RateLimit_SECOND, sm.NewStats("key_value"), false, false, false, "", nil, false)} + + result := cache.DoLimit(context.Background(), request, limits) + cache.Flush() + + assert.Equal(pb.RateLimitResponse_OK, result[0].Code) + assert.Equal(uint64(0), limits[0].Stats.OverLimit.Value()) + assert.Equal(uint64(0), limits[0].Stats.OverLimitWithLocalCache.Value()) +} diff --git a/test/mocks/memcached/client.go b/test/mocks/memcached/client.go index b00abf1e2..2d2d40d13 100644 --- a/test/mocks/memcached/client.go +++ b/test/mocks/memcached/client.go @@ -77,3 +77,18 @@ func (mr *MockClientMockRecorder) Increment(arg0, arg1 interface{}) *gomock.Call mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Increment", reflect.TypeOf((*MockClient)(nil).Increment), arg0, arg1) } + +// Decrement mocks base method +func (m *MockClient) Decrement(arg0 string, arg1 uint64) (uint64, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Decrement", arg0, arg1) + ret0, _ := ret[0].(uint64) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Decrement indicates an expected call of Decrement +func (mr *MockClientMockRecorder) Decrement(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Decrement", reflect.TypeOf((*MockClient)(nil).Decrement), arg0, arg1) +} diff --git a/test/mocks/stats/manager.go b/test/mocks/stats/manager.go index 522f84868..253926410 100644 --- a/test/mocks/stats/manager.go +++ b/test/mocks/stats/manager.go @@ -44,6 +44,7 @@ func (m *MockStatManager) NewStats(key string) stats.RateLimitStats { ret.OverLimitWithLocalCache = m.store.NewCounter(key + ".over_limit_with_local_cache") ret.WithinLimit = m.store.NewCounter(key + ".within_limit") ret.ShadowMode = m.store.NewCounter(key + ".shadow_mode") + ret.TotalNegativeHits = m.store.NewCounter(key + ".total_negative_hits") return ret } diff --git a/test/redis/fixed_cache_impl_test.go b/test/redis/fixed_cache_impl_test.go index e52abb68d..c3a3ceeee 100644 --- a/test/redis/fixed_cache_impl_test.go +++ b/test/redis/fixed_cache_impl_test.go @@ -29,6 +29,93 @@ import ( var testSpanExporter = trace.GetTestSpanExporter() +func TestNegativeHits(t *testing.T) { + assert := assert.New(t) + controller := gomock.NewController(t) + defer controller.Finish() + statsStore := gostats.NewStore(gostats.NewNullSink(), false) + sm := stats.NewMockStatManager(statsStore) + + client := mock_redis.NewMockClient(controller) + timeSource := mock_utils.NewMockTimeSource(controller) + cache := redis.NewFixedRateLimitCacheImpl(client, nil, timeSource, rand.New(rand.NewSource(1)), 0, nil, 0.8, "", sm, false) + + // Decrement from a counter at 5, requesting -3. Lua returns 2 (5-3=2). + timeSource.EXPECT().UnixNow().Return(int64(1234)).MaxTimes(3) + client.EXPECT().PipeAppend(gomock.Any(), gomock.Any(), "EVAL", limiter.DecrementScript, 1, "domain_key_value_1234", uint64(3), int64(1)).SetArg(1, uint64(2)).DoAndReturn(pipeAppend) + client.EXPECT().PipeDo(gomock.Any()).Return(nil) + + request := common.NewRateLimitRequestWithNegativeHits( + "domain", [][][2]string{{{"key", "value"}}}, []uint64{3}, []bool{true}) + limits := []*config.RateLimit{config.NewRateLimit(10, pb.RateLimitResponse_RateLimit_SECOND, sm.NewStats("key_value"), false, false, false, "", nil, false)} + + result := cache.DoLimit(context.Background(), request, limits) + assert.Equal(pb.RateLimitResponse_OK, result[0].Code) + assert.Equal(uint64(8), uint64(result[0].LimitRemaining)) + assert.Equal(limits[0].Limit, result[0].CurrentLimit) + assert.Equal(uint64(0), limits[0].Stats.TotalHits.Value()) + assert.Equal(uint64(3), limits[0].Stats.TotalNegativeHits.Value()) + assert.Equal(uint64(0), limits[0].Stats.OverLimit.Value()) +} + +func TestNegativeHitsFloorAtZero(t *testing.T) { + assert := assert.New(t) + controller := gomock.NewController(t) + defer controller.Finish() + statsStore := gostats.NewStore(gostats.NewNullSink(), false) + sm := stats.NewMockStatManager(statsStore) + + client := mock_redis.NewMockClient(controller) + timeSource := mock_utils.NewMockTimeSource(controller) + cache := redis.NewFixedRateLimitCacheImpl(client, nil, timeSource, rand.New(rand.NewSource(1)), 0, nil, 0.8, "", sm, false) + + // Counter at 2, requesting -5. Lua floors at 0 and returns 0. + timeSource.EXPECT().UnixNow().Return(int64(1234)).MaxTimes(3) + client.EXPECT().PipeAppend(gomock.Any(), gomock.Any(), "EVAL", limiter.DecrementScript, 1, "domain_key_value_1234", uint64(5), int64(1)).SetArg(1, uint64(0)).DoAndReturn(pipeAppend) + client.EXPECT().PipeDo(gomock.Any()).Return(nil) + + request := common.NewRateLimitRequestWithNegativeHits( + "domain", [][][2]string{{{"key", "value"}}}, []uint64{5}, []bool{true}) + limits := []*config.RateLimit{config.NewRateLimit(10, pb.RateLimitResponse_RateLimit_SECOND, sm.NewStats("key_value"), false, false, false, "", nil, false)} + + result := cache.DoLimit(context.Background(), request, limits) + assert.Equal(pb.RateLimitResponse_OK, result[0].Code) + assert.Equal(uint64(10), uint64(result[0].LimitRemaining)) + assert.Equal(uint64(0), limits[0].Stats.TotalHits.Value()) + assert.Equal(uint64(5), limits[0].Stats.TotalNegativeHits.Value()) +} + +func TestNegativeHitsSkipsOverLimitCheck(t *testing.T) { + assert := assert.New(t) + controller := gomock.NewController(t) + defer controller.Finish() + statsStore := gostats.NewStore(gostats.NewNullSink(), false) + sm := stats.NewMockStatManager(statsStore) + + client := mock_redis.NewMockClient(controller) + timeSource := mock_utils.NewMockTimeSource(controller) + localCache := freecache.NewCache(100) + cache := redis.NewFixedRateLimitCacheImpl(client, nil, timeSource, rand.New(rand.NewSource(1)), 0, localCache, 0.8, "", sm, false) + + // Set the key as over-limit in local cache. + localCache.Set([]byte("domain_key_value_1234"), []byte{}, 60) + + // Even though the key is in the over-limit local cache, negative hits should still proceed. + timeSource.EXPECT().UnixNow().Return(int64(1234)).MaxTimes(3) + client.EXPECT().PipeAppend(gomock.Any(), gomock.Any(), "EVAL", limiter.DecrementScript, 1, "domain_key_value_1234", uint64(2), int64(1)).SetArg(1, uint64(8)).DoAndReturn(pipeAppend) + client.EXPECT().PipeDo(gomock.Any()).Return(nil) + + request := common.NewRateLimitRequestWithNegativeHits( + "domain", [][][2]string{{{"key", "value"}}}, []uint64{2}, []bool{true}) + limits := []*config.RateLimit{config.NewRateLimit(10, pb.RateLimitResponse_RateLimit_SECOND, sm.NewStats("key_value"), false, false, false, "", nil, false)} + + result := cache.DoLimit(context.Background(), request, limits) + assert.Equal(pb.RateLimitResponse_OK, result[0].Code) + assert.Equal(uint64(2), uint64(result[0].LimitRemaining)) + assert.Equal(uint64(0), limits[0].Stats.OverLimit.Value()) + assert.Equal(uint64(0), limits[0].Stats.OverLimitWithLocalCache.Value()) +} + func TestRedis(t *testing.T) { t.Run("WithoutPerSecondRedis", testRedis(false)) t.Run("WithPerSecondRedis", testRedis(true))