From 29cb4b6b47f6b8384b4f1d066b4c850629775548 Mon Sep 17 00:00:00 2001 From: Milton Mazzarri Date: Tue, 13 Jan 2026 13:19:14 -0600 Subject: [PATCH 01/16] update dependencies --- mix.lock | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/mix.lock b/mix.lock index 10b4326..4318c2d 100644 --- a/mix.lock +++ b/mix.lock @@ -1,22 +1,22 @@ %{ "connection": {:hex, :connection, "1.1.0", "ff2a49c4b75b6fb3e674bfc5536451607270aac754ffd1bdfe175abe4a6d7a68", [:mix], [], "hexpm", "722c1eb0a418fbe91ba7bd59a47e28008a189d47e37e0e7bb85585a016b2869c"}, - "earmark_parser": {:hex, :earmark_parser, "1.4.43", "34b2f401fe473080e39ff2b90feb8ddfeef7639f8ee0bbf71bb41911831d77c5", [:mix], [], "hexpm", "970a3cd19503f5e8e527a190662be2cee5d98eed1ff72ed9b3d1a3d466692de8"}, + "earmark_parser": {:hex, :earmark_parser, "1.4.44", "f20830dd6b5c77afe2b063777ddbbff09f9759396500cdbe7523efd58d7a339c", [:mix], [], "hexpm", "4778ac752b4701a5599215f7030989c989ffdc4f6df457c5f36938cc2d2a2750"}, "elasticachex": {:hex, :elasticachex, "1.1.3", "c5cc1255b3f25c53df16206959816824cc65e65be5be8462af069be59af63013", [:mix], [{:socket, "~> 0.3", [hex: :socket, repo: "hexpm", optional: false]}], "hexpm", "425814b1406729f2f037ff3b90755162b1d8b7fef23b3c23deac295e05cec2fc"}, "ex2ms": {:hex, :ex2ms, "1.7.0", "45b9f523d0b777667ded60070d82d871a37e294f0b6c5b8eca86771f00f82ee1", [:mix], [], "hexpm", "2589eee51f81f1b1caa6d08c990b1ad409215fe6f64c73f73c67d36ed10be827"}, - "ex_doc": {:hex, :ex_doc, "0.37.2", "2a3aa7014094f0e4e286a82aa5194a34dd17057160988b8509b15aa6c292720c", [:mix], [{:earmark_parser, "~> 1.4.42", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_c, ">= 0.1.0", [hex: :makeup_c, repo: "hexpm", optional: true]}, {:makeup_elixir, "~> 0.14 or ~> 1.0", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1 or ~> 1.0", [hex: :makeup_erlang, repo: "hexpm", optional: false]}, {:makeup_html, ">= 0.1.0", [hex: :makeup_html, repo: "hexpm", optional: true]}], "hexpm", "4dfa56075ce4887e4e8b1dcc121cd5fcb0f02b00391fd367ff5336d98fa49049"}, + "ex_doc": {:hex, :ex_doc, "0.39.3", "519c6bc7e84a2918b737aec7ef48b96aa4698342927d080437f61395d361dcee", [:mix], [{:earmark_parser, "~> 1.4.44", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_c, ">= 0.1.0", [hex: :makeup_c, repo: "hexpm", optional: true]}, {:makeup_elixir, "~> 0.14 or ~> 1.0", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1 or ~> 1.0", [hex: :makeup_erlang, repo: "hexpm", optional: false]}, {:makeup_html, ">= 0.1.0", [hex: :makeup_html, repo: "hexpm", optional: true]}], "hexpm", "0590955cf7ad3b625780ee1c1ea627c28a78948c6c0a9b0322bd976a079996e1"}, "herd": {:hex, :herd, "0.4.3", "97469cf289c1e89a4f2b356da486ae5a354751f91c10cd3749af6aedebd9a775", [:mix], [{:libring, "~> 1.1", [hex: :libring, repo: "hexpm", optional: false]}, {:poolboy, "~> 1.5", [hex: :poolboy, repo: "hexpm", optional: false]}], "hexpm", "44bfd2c42a206431495d5103a77f52a992f4f1391a13459a9e8fd7b143cd99c9"}, "libring": {:hex, :libring, "1.7.0", "4f245d2f1476cd7ed8f03740f6431acba815401e40299208c7f5c640e1883bda", [:mix], [], "hexpm", "070e3593cb572e04f2c8470dd0c119bc1817a7a0a7f88229f43cf0345268ec42"}, "makeup": {:hex, :makeup, "1.2.1", "e90ac1c65589ef354378def3ba19d401e739ee7ee06fb47f94c687016e3713d1", [:mix], [{:nimble_parsec, "~> 1.4", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "d36484867b0bae0fea568d10131197a4c2e47056a6fbe84922bf6ba71c8d17ce"}, "makeup_elixir": {:hex, :makeup_elixir, "1.0.1", "e928a4f984e795e41e3abd27bfc09f51db16ab8ba1aebdba2b3a575437efafc2", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "7284900d412a3e5cfd97fdaed4f5ed389b8f2b4cb49efc0eb3bd10e2febf9507"}, - "makeup_erlang": {:hex, :makeup_erlang, "1.0.2", "03e1804074b3aa64d5fad7aa64601ed0fb395337b982d9bcf04029d68d51b6a7", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "af33ff7ef368d5893e4a267933e7744e46ce3cf1f61e2dccf53a111ed3aa3727"}, + "makeup_erlang": {:hex, :makeup_erlang, "1.0.3", "4252d5d4098da7415c390e847c814bad3764c94a814a0b4245176215615e1035", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "953297c02582a33411ac6208f2c6e55f0e870df7f80da724ed613f10e6706afd"}, "memcachex": {:hex, :memcachex, "0.5.7", "00dc47d926eba11dfc1f2db606c37caff34d9fafd57f0adc10b49418931e77af", [:mix], [{:connection, "~> 1.0", [hex: :connection, repo: "hexpm", optional: false]}, {:poison, "~> 2.1 or ~> 3.0 or ~> 4.0 or ~> 5.0 or ~> 6.0", [hex: :poison, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.0 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "a75e5b712c17bc25326c03c4c1b45fae39816b6fdacce3242724c7131f210ec1"}, "memcachir": {:hex, :memcachir, "3.3.1", "2099fd7d518b58172b417f932f9590e3fd3e11deb811c2a3a252605751af6aa7", [:mix], [{:elasticachex, "~> 1.1", [hex: :elasticachex, repo: "hexpm", optional: false]}, {:herd, "~> 0.4.3", [hex: :herd, repo: "hexpm", optional: false]}, {:memcachex, "~> 0.5", [hex: :memcachex, repo: "hexpm", optional: false]}], "hexpm", "526536e9585820894381a643aa3042f7f84e25e986e3c90f413fe0896f8bb513"}, - "mime": {:hex, :mime, "2.0.6", "8f18486773d9b15f95f4f4f1e39b710045fa1de891fada4516559967276e4dc2", [:mix], [], "hexpm", "c9945363a6b26d747389aac3643f8e0e09d30499a138ad64fe8fd1d13d9b153e"}, + "mime": {:hex, :mime, "2.0.7", "b8d739037be7cd402aee1ba0306edfdef982687ee7e9859bee6198c1e7e2f128", [:mix], [], "hexpm", "6171188e399ee16023ffc5b76ce445eb6d9672e2e241d2df6050f3c771e80ccd"}, "nimble_parsec": {:hex, :nimble_parsec, "1.4.2", "8efba0122db06df95bfaa78f791344a89352ba04baedd3849593bfce4d0dc1c6", [:mix], [], "hexpm", "4b21398942dda052b403bbe1da991ccd03a053668d147d53fb8c4e0efe09c973"}, - "plug": {:hex, :plug, "1.16.1", "40c74619c12f82736d2214557dedec2e9762029b2438d6d175c5074c933edc9d", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "a13ff6b9006b03d7e33874945b2755253841b238c34071ed85b0e86057f8cddc"}, - "plug_crypto": {:hex, :plug_crypto, "2.1.0", "f44309c2b06d249c27c8d3f65cfe08158ade08418cf540fd4f72d4d6863abb7b", [:mix], [], "hexpm", "131216a4b030b8f8ce0f26038bc4421ae60e4bb95c5cf5395e1421437824c4fa"}, + "plug": {:hex, :plug, "1.19.1", "09bac17ae7a001a68ae393658aa23c7e38782be5c5c00c80be82901262c394c0", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "560a0017a8f6d5d30146916862aaf9300b7280063651dd7e532b8be168511e62"}, + "plug_crypto": {:hex, :plug_crypto, "2.1.1", "19bda8184399cb24afa10be734f84a16ea0a2bc65054e23a62bb10f06bc89491", [:mix], [], "hexpm", "6470bce6ffe41c8bd497612ffde1a7e4af67f36a15eea5f921af71cf3e11247c"}, "poolboy": {:hex, :poolboy, "1.5.2", "392b007a1693a64540cead79830443abf5762f5d30cf50bc95cb2c1aaafa006b", [:rebar3], [], "hexpm", "dad79704ce5440f3d5a3681c8590b9dc25d1a561e8f5a9c995281012860901e3"}, "socket": {:hex, :socket, "0.3.13", "98a2ab20ce17f95fb512c5cadddba32b57273e0d2dba2d2e5f976c5969d0c632", [:mix], [], "hexpm", "f82ea9833ef49dde272e6568ab8aac657a636acb4cf44a7de8a935acb8957c2e"}, - "styler": {:hex, :styler, "1.4.0", "5944723d08afe4d38210b674d7e97dd1137a75968a85a633983cc308e86dc5f2", [:mix], [], "hexpm", "07de0e89c27490c8e469bb814d77ddaaa3283d7d8038501021d80a7705cf13e9"}, + "styler": {:hex, :styler, "1.10.1", "9229050c978bfaaab1d94e8673843576d0127d48fe64824a30babde3d6342475", [:mix], [], "hexpm", "d86cbcc70e8ab424393af313d1d885931ba9dc7c383d7dd30f4ab255a8d39f73"}, "telemetry": {:hex, :telemetry, "1.3.0", "fedebbae410d715cf8e7062c96a1ef32ec22e764197f70cda73d82778d61e7a2", [:rebar3], [], "hexpm", "7015fc8919dbe63764f4b4b87a95b7c0996bd539e0d499be6ec9d7f3875b79e6"}, } From 9755c109ee4617c78e1dc7f759672418073d42ea Mon Sep 17 00:00:00 2001 From: Milton Mazzarri Date: Tue, 13 Jan 2026 13:19:41 -0600 Subject: [PATCH 02/16] remove compilation warnings --- lib/ex_limiter.ex | 2 +- lib/ex_limiter/plug.ex | 9 +++++---- lib/ex_limiter/storage/pg2_shard/pruner.ex | 11 ++++++----- lib/ex_limiter/storage/pg2_shard/supervisor.ex | 2 +- test/ex_limiter/plug_test.exs | 3 ++- 5 files changed, 15 insertions(+), 12 deletions(-) diff --git a/lib/ex_limiter.ex b/lib/ex_limiter.ex index c7bf372..80d0994 100644 --- a/lib/ex_limiter.ex +++ b/lib/ex_limiter.ex @@ -25,5 +25,5 @@ defmodule ExLimiter do end ``` """ - use ExLimiter.Base, storage: Application.get_env(:ex_limiter, :storage) + use ExLimiter.Base, storage: Application.compile_env(:ex_limiter, :storage) end diff --git a/lib/ex_limiter/plug.ex b/lib/ex_limiter/plug.ex index 1ff1676..a3a3168 100644 --- a/lib/ex_limiter/plug.ex +++ b/lib/ex_limiter/plug.ex @@ -39,13 +39,14 @@ defmodule ExLimiter.Plug do """ import Plug.Conn - @limiter Application.get_env(:ex_limiter, __MODULE__)[:limiter] + @limiter Application.compile_env(:ex_limiter, __MODULE__)[:limiter] defmodule Config do @moduledoc false - @limit Application.get_env(:ex_limiter, ExLimiter.Plug)[:limit] - @scale Application.get_env(:ex_limiter, ExLimiter.Plug)[:scale] - @fallback Application.get_env(:ex_limiter, ExLimiter.Plug)[:fallback] + @opts Application.compile_env(:ex_limiter, ExLimiter.Plug) + @limit @opts[:limit] + @scale @opts[:scale] + @fallback @opts[:fallback] defstruct scale: @scale, limit: @limit, diff --git a/lib/ex_limiter/storage/pg2_shard/pruner.ex b/lib/ex_limiter/storage/pg2_shard/pruner.ex index 0bbe8e1..a993f32 100644 --- a/lib/ex_limiter/storage/pg2_shard/pruner.ex +++ b/lib/ex_limiter/storage/pg2_shard/pruner.ex @@ -11,11 +11,12 @@ defmodule ExLimiter.Storage.PG2Shard.Pruner do alias ExLimiter.Utils @table_name :exlimiter_buckets - @expiry Application.get_env(:ex_limiter, PG2Shard)[:expiry] || 10 * 60_000 - @eviction_count Application.get_env(:ex_limiter, PG2Shard)[:eviction_count] || 1000 - @max_size Application.get_env(:ex_limiter, PG2Shard)[:max_size] || 50_000 - @prune_interval Application.get_env(:ex_limiter, PG2Shard)[:prune_interval] || 5_000 - @eviction_interval Application.get_env(:ex_limiter, PG2Shard)[:eviction_interval] || 30_000 + @opts Application.compile_env(:ex_limiter, PG2Shard) + @expiry @opts[:expiry] || 10 * 60_000 + @eviction_count @opts[:eviction_count] || 1000 + @max_size @opts[:max_size] || 50_000 + @prune_interval @opts[:prune_interval] || 5_000 + @eviction_interval @opts[:eviction_interval] || 30_000 def start_link(_args \\ :ok) do GenServer.start_link(__MODULE__, [], name: __MODULE__) diff --git a/lib/ex_limiter/storage/pg2_shard/supervisor.ex b/lib/ex_limiter/storage/pg2_shard/supervisor.ex index 2b5285b..931e17b 100644 --- a/lib/ex_limiter/storage/pg2_shard/supervisor.ex +++ b/lib/ex_limiter/storage/pg2_shard/supervisor.ex @@ -14,7 +14,7 @@ defmodule ExLimiter.Storage.PG2Shard.Supervisor do alias ExLimiter.Storage.PG2Shard.Shutdown alias ExLimiter.Storage.PG2Shard.Worker - @telemetry Application.get_env(:ex_limiter, PG2Shard)[:telemetry] || Worker + @telemetry Application.compile_env(:ex_limiter, PG2Shard)[:telemetry] || Worker def start_link(_args \\ :ok) do Supervisor.start_link(__MODULE__, [], name: __MODULE__) diff --git a/test/ex_limiter/plug_test.exs b/test/ex_limiter/plug_test.exs index 2562ac8..aea108c 100644 --- a/test/ex_limiter/plug_test.exs +++ b/test/ex_limiter/plug_test.exs @@ -1,6 +1,7 @@ defmodule ExLimiter.PlugTest do use ExUnit.Case - use Plug.Test + import Plug.Test + import Plug.Conn alias ExLimiter.TestUtils From 5b7e385c3a0e5995d53373d945ef4abc8397777d Mon Sep 17 00:00:00 2001 From: Milton Mazzarri Date: Tue, 13 Jan 2026 14:24:48 -0600 Subject: [PATCH 03/16] review docs --- README.md | 13 ++++++----- lib/ex_limiter.ex | 27 ++++++++++------------ lib/ex_limiter/base.ex | 14 ++++++----- lib/ex_limiter/plug.ex | 20 ++++++---------- lib/ex_limiter/storage.ex | 10 ++++---- lib/ex_limiter/storage/pg2_shard.ex | 11 +++------ lib/ex_limiter/storage/pg2_shard/worker.ex | 25 ++++++++------------ 7 files changed, 53 insertions(+), 67 deletions(-) diff --git a/README.md b/README.md index 294bccd..117725c 100644 --- a/README.md +++ b/README.md @@ -1,18 +1,19 @@ # ex_limiter -Rate Limiter written in elixir with configurable backends -Implements leaky bucket rate limiting ([wiki](https://en.wikipedia.org/wiki/Leaky_bucket)), which is superior to most naive approaches by handling bursts even around time windows. You can define your own storage backend by implementing the `ExLimiter.Storage` behaviour, and configuring it with +Rate Limiter written in elixir with configurable backends. + +Implements leaky bucket rate limiting ([wiki](https://en.wikipedia.org/wiki/Leaky_bucket)), which is superior to most naive approaches by handling bursts even around time windows. You can define your own storage backend by implementing the `ExLimiter.Storage` behaviour, and configuring it with ```elixir config :ex_limiter, :storage, MyStorage ``` -usage once configured is: +Usage once configured is: ```elixir case ExLimiter.consume(bucket, 1, scale: 1000, limit: 5) do - {:ok, bucket} -> #do some work - {:error, :rate_limited} -> #fail + {:ok, bucket} -> # do some work + {:error, :rate_limited} -> # fail end ``` @@ -32,4 +33,4 @@ ExLimiter also ships with a simple plug implementation. Usage is plug ExLimiter.Plug, scale: 5000, limit: 20 ``` -You can also configure how the bucket is inferred from the given conn, how many tokens to consume and what limiter to use. \ No newline at end of file +You can also configure how the bucket is inferred from the given `conn`, how many tokens to consume and what limiter to use. diff --git a/lib/ex_limiter.ex b/lib/ex_limiter.ex index 80d0994..98ad3aa 100644 --- a/lib/ex_limiter.ex +++ b/lib/ex_limiter.ex @@ -1,29 +1,26 @@ defmodule ExLimiter do @moduledoc """ - Configurable, leaky bucket rate limiting. You can define your own storage backend by + Configurable, leaky bucket rate limiting. + + You can define your own storage backend by implementing the `ExLimiter.Storage` behaviour, and configuring it with - ``` - config :ex_limiter, :storage, MyStorage - ``` + config :ex_limiter, :storage, MyStorage + usage once configured is: - ``` - case ExLimiter.consume(bucket, 1, scale: 1000, limit: 5) do - {:ok, bucket} -> #do some work - {:error, :rate_limited} -> #fail - end - ``` + case ExLimiter.consume(bucket, 1, scale: 1000, limit: 5) do + {:ok, bucket} -> #do some work + {:error, :rate_limited} -> #fail + end Additionally, if you want to have multiple rate limiters with diverse backend implementations, you can use the `ExLimiter.Base` macro, like so: - ``` - defmodule MyLimiter do - use ExLimiter.Base, storage: MyStorage - end - ``` + defmodule MyLimiter do + use ExLimiter.Base, storage: MyStorage + end """ use ExLimiter.Base, storage: Application.compile_env(:ex_limiter, :storage) end diff --git a/lib/ex_limiter/base.ex b/lib/ex_limiter/base.ex index 482a90f..c4649a6 100644 --- a/lib/ex_limiter/base.ex +++ b/lib/ex_limiter/base.ex @@ -1,12 +1,12 @@ defmodule ExLimiter.Base do @moduledoc """ - Base module for arbitrary rate limiter implementations. Usage is: + Base module for arbitrary rate limiter implementations. - ``` - defmodule MyLimiter do - use ExLimiterBase, storage: MyCustomStorage - end - ``` + Usage is: + + defmodule MyLimiter do + use ExLimiterBase, storage: MyCustomStorage + end """ alias ExLimiter.Bucket alias ExLimiter.Utils @@ -28,12 +28,14 @@ defmodule ExLimiter.Base do Consumes `amount` from the rate limiter aliased by bucket. `opts` params are: + * `:limit` - the maximum amount for the rate limiter (default 10) * `:scale` - the duration under which `:limit` applies in milliseconds """ @spec consume(bucket :: binary, amount :: integer, opts :: keyword) :: {:ok, Bucket.t()} | {:error, :rate_limited} def consume(bucket, amount \\ 1, opts \\ []), do: consume(@storage, bucket, amount, opts) + @doc "Deletes the bucket from the storage" def delete(bucket), do: @storage.delete(%Bucket{key: bucket}) end end diff --git a/lib/ex_limiter/plug.ex b/lib/ex_limiter/plug.ex index a3a3168..c878c04 100644 --- a/lib/ex_limiter/plug.ex +++ b/lib/ex_limiter/plug.ex @@ -1,10 +1,10 @@ defmodule ExLimiter.Plug do @moduledoc """ - Plug for enforcing rate limits. The usage should be something like + Plug for enforcing rate limits. - ``` - plug ExLimiter.Plug, scale: 1000, limit: 5 - ``` + The usage should be something like + + plug ExLimiter.Plug, scale: 1000, limit: 5 Additionally, you can pass the following options: @@ -25,15 +25,11 @@ defmodule ExLimiter.Plug do Additionally, you can configure a custom limiter with - ``` - config :ex_limiter, ExLimiter.Plug, limiter: MyLimiter - ``` + config :ex_limiter, ExLimiter.Plug, limiter: MyLimiter and you can also configure the rate limited response with - ``` - config :ex_limiter, ExLimiter.Plug, fallback: MyFallback - ``` + config :ex_limiter, ExLimiter.Plug, fallback: MyFallback `MyFallback` needs to implement a function `render_error(conn, :rate_limited)` """ @@ -91,9 +87,7 @@ defmodule ExLimiter.Plug do }) do bucket_name = bucket_fun.(conn) - bucket_name - |> @limiter.consume(consume_fun.(conn), scale: scale, limit: limit) - |> case do + case @limiter.consume(bucket_name, consume_fun.(conn), scale: scale, limit: limit) do {:ok, bucket} = response -> remaining = @limiter.remaining(bucket, scale: scale, limit: limit) diff --git a/lib/ex_limiter/storage.ex b/lib/ex_limiter/storage.ex index 99439e6..534155b 100644 --- a/lib/ex_limiter/storage.ex +++ b/lib/ex_limiter/storage.ex @@ -26,15 +26,17 @@ defmodule ExLimiter.Storage do @callback fetch(bucket :: Bucket.t()) :: Bucket.t() @doc """ - Set the current state of the given bucket. Specify hard if you want to - force a write + Set the current state of the given bucket. + + Specify hard if you want to force a write """ @callback refresh(bucket :: Bucket.t()) :: response @callback refresh(bucket :: Bucket.t(), type :: :hard | :soft) :: response @doc """ - Atomically update the bucket denoted by `key` with `fun`. Leverage whatever - concurrency controls are available in the given storage mechanism (eg cas for memcached) + Atomically update the bucket denoted by `key` with `fun`. + + Leverage whatever concurrency controls are available in the given storage mechanism (eg cas for memcached) """ @callback update(key :: binary, fun :: (Bucket.t() -> Bucket.t())) :: Bucket.t() diff --git a/lib/ex_limiter/storage/pg2_shard.ex b/lib/ex_limiter/storage/pg2_shard.ex index 1e074e6..88a849c 100644 --- a/lib/ex_limiter/storage/pg2_shard.ex +++ b/lib/ex_limiter/storage/pg2_shard.ex @@ -5,18 +5,13 @@ defmodule ExLimiter.Storage.PG2Shard do To configure the pool size, do: - ``` - config :ex_limit, ExLimiter.Storage.PG2Shard, - shard_count: 20 - ``` + config :ex_limit, ExLimiter.Storage.PG2Shard, + shard_count: 20 You must also include the shard supervisor in your app supervision tree, with something like: - ``` - ... - supervise(ExLimiter.Storage.PG2Shard.Supervisor, []) - ``` + supervise(ExLimiter.Storage.PG2Shard.Supervisor, []) """ use ExLimiter.Storage diff --git a/lib/ex_limiter/storage/pg2_shard/worker.ex b/lib/ex_limiter/storage/pg2_shard/worker.ex index 726e934..ac88553 100644 --- a/lib/ex_limiter/storage/pg2_shard/worker.ex +++ b/lib/ex_limiter/storage/pg2_shard/worker.ex @@ -4,28 +4,23 @@ defmodule ExLimiter.Storage.PG2Shard.Worker do of buckets. Buckets are pruned after 10 minutes of inactivity, and buckets will be evicted - if a maximum threshold is reached. To tune these values, use: + if a maximum threshold is reached. To tune these values, use: - ``` - config :ex_limiter, ExLimiter.Storage.PG2Shard, - max_size: 50_000, - eviction_count: 1000 - ``` + config :ex_limiter, ExLimiter.Storage.PG2Shard, + max_size: 50_000, + eviction_count: 1000 It will also publish these metrics via telemetry: - ``` - [:ex_limiter, :shards, :map_size], - [:ex_limiter, :shards, :evictions], - [:ex_limiter, :shards, :expirations] - ``` + + [:ex_limiter, :shards, :map_size], + [:ex_limiter, :shards, :evictions], + [:ex_limiter, :shards, :expirations] You can auto-configure a telemetry handler via: - ``` - config :ex_limiter, ExLimiter.Storage.PG2Shard, - telemetry: MyTelemetryHandler - ``` + config :ex_limiter, ExLimiter.Storage.PG2Shard, + telemetry: MyTelemetryHandler """ use GenServer From da302e30ac0ea8cb02e32fcae239236a3f4387c0 Mon Sep 17 00:00:00 2001 From: Milton Mazzarri Date: Tue, 13 Jan 2026 14:25:38 -0600 Subject: [PATCH 04/16] pass functions in the style &Mod.fun/arity --- lib/ex_limiter/base.ex | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/lib/ex_limiter/base.ex b/lib/ex_limiter/base.ex index c4649a6..664b491 100644 --- a/lib/ex_limiter/base.ex +++ b/lib/ex_limiter/base.ex @@ -53,17 +53,21 @@ defmodule ExLimiter.Base do storage.leak_and_consume( bucket, - fn %Bucket{value: value, last: time} = b -> - now = Utils.now() - amount = max(value - (now - time), 0) - - %{b | last: now, value: amount} - end, - fn - %Bucket{value: v} = b when v + incr <= scale -> b - _ -> {:error, :rate_limited} - end, + &__MODULE__.update/1, + &__MODULE__.boundary(&1, incr, scale), incr ) end + + @doc false + def update(%Bucket{value: value, last: time} = b) do + now = Utils.now() + amount = max(value - (now - time), 0) + + %{b | last: now, value: amount} + end + + @doc false + def boundary(%Bucket{value: v} = b, incr, scale) when v + incr <= scale, do: b + def boundary(_, _, _), do: {:error, :rate_limited} end From 007dcfae1eac3e044c449e857ef3ebc8eae41d72 Mon Sep 17 00:00:00 2001 From: Milton Mazzarri Date: Tue, 13 Jan 2026 14:44:07 -0600 Subject: [PATCH 05/16] add original ci from fabian --- .github/workflows/ci.yml | 56 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 56 insertions(+) create mode 100644 .github/workflows/ci.yml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..fb4b85c --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,56 @@ +name: Elixir CI + +on: + push: + branches: ["master"] + pull_request: + branches: ["master"] + +permissions: + contents: read + +jobs: + build: + name: Build and test + runs-on: ubuntu-latest + strategy: + matrix: + include: + - elixir: "1.15" + otp: "25" + - elixir: "1.18" + otp: "27" + lint: true + steps: + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Elixir + uses: erlef/setup-beam@v1 + with: + otp-version: ${{matrix.otp}} + elixir-version: ${{matrix.elixir}} + + - name: Restore dependencies cache + uses: actions/cache@v3 + with: + path: deps + key: ${{ runner.os }}-mix-${{ hashFiles('**/mix.lock') }} + restore-keys: ${{ runner.os }}-mix- + + - name: Install dependencies + run: mix deps.get + + - name: Compile + run: mix compile --warnings-as-errors + + - name: Run tests + run: mix test + + - name: checks that the mix.lock file has no unused deps + run: mix deps.unlock --check-unused + if: ${{ matrix.lint }} + + - name: check if files are already formatted + run: mix format --check-formatted + if: ${{ matrix.lint }} From c68c540cd42bdbcba81819b4be324858c40a5e49 Mon Sep 17 00:00:00 2001 From: Milton Mazzarri Date: Tue, 13 Jan 2026 14:44:25 -0600 Subject: [PATCH 06/16] add memcached service --- .github/workflows/ci.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fb4b85c..aeb11f9 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -13,6 +13,11 @@ jobs: build: name: Build and test runs-on: ubuntu-latest + services: + memcached: + image: memcached:alpine + ports: + - 11211:11211 strategy: matrix: include: From e68ce32ecc1cd7e1263df9e026f19724d9feca75 Mon Sep 17 00:00:00 2001 From: Milton Mazzarri Date: Tue, 13 Jan 2026 15:05:55 -0600 Subject: [PATCH 07/16] remove config directory --- config/config.exs | 10 ---------- lib/ex_limiter.ex | 2 +- lib/ex_limiter/plug.ex | 11 ++++++----- lib/ex_limiter/storage/pg2_shard/supervisor.ex | 4 ++-- mix.exs | 4 ++-- test/test_helper.exs | 1 + 6 files changed, 12 insertions(+), 20 deletions(-) delete mode 100644 config/config.exs diff --git a/config/config.exs b/config/config.exs deleted file mode 100644 index a80f183..0000000 --- a/config/config.exs +++ /dev/null @@ -1,10 +0,0 @@ -import Config - -config :ex_limiter, ExLimiter.Plug, - limiter: ExLimiter, - fallback: ExLimiter.Plug, - limit: 10, - scale: 1000 - -config :ex_limiter, ExLimiter.Storage.PG2Shard, shard_count: 20 -config :ex_limiter, :storage, ExLimiter.Storage.Memcache diff --git a/lib/ex_limiter.ex b/lib/ex_limiter.ex index 98ad3aa..63ba5b1 100644 --- a/lib/ex_limiter.ex +++ b/lib/ex_limiter.ex @@ -22,5 +22,5 @@ defmodule ExLimiter do use ExLimiter.Base, storage: MyStorage end """ - use ExLimiter.Base, storage: Application.compile_env(:ex_limiter, :storage) + use ExLimiter.Base, storage: Application.compile_env(:ex_limiter, :storage, ExLimiter.Storage.Memcache) end diff --git a/lib/ex_limiter/plug.ex b/lib/ex_limiter/plug.ex index c878c04..bded1b5 100644 --- a/lib/ex_limiter/plug.ex +++ b/lib/ex_limiter/plug.ex @@ -35,14 +35,15 @@ defmodule ExLimiter.Plug do """ import Plug.Conn - @limiter Application.compile_env(:ex_limiter, __MODULE__)[:limiter] + @compile_opts Application.compile_env(:ex_limiter, __MODULE__, []) + @limiter @compile_opts[:limiter] || ExLimiter defmodule Config do @moduledoc false - @opts Application.compile_env(:ex_limiter, ExLimiter.Plug) - @limit @opts[:limit] - @scale @opts[:scale] - @fallback @opts[:fallback] + @compile_opts Application.compile_env(:ex_limiter, ExLimiter.Plug, []) + @limit @compile_opts[:limit] || 10 + @scale @compile_opts[:scale] || 1000 + @fallback @compile_opts[:fallback] || ExLimiter.Plug defstruct scale: @scale, limit: @limit, diff --git a/lib/ex_limiter/storage/pg2_shard/supervisor.ex b/lib/ex_limiter/storage/pg2_shard/supervisor.ex index 931e17b..85472a3 100644 --- a/lib/ex_limiter/storage/pg2_shard/supervisor.ex +++ b/lib/ex_limiter/storage/pg2_shard/supervisor.ex @@ -14,7 +14,7 @@ defmodule ExLimiter.Storage.PG2Shard.Supervisor do alias ExLimiter.Storage.PG2Shard.Shutdown alias ExLimiter.Storage.PG2Shard.Worker - @telemetry Application.compile_env(:ex_limiter, PG2Shard)[:telemetry] || Worker + @telemetry Application.compile_env(:ex_limiter, PG2Shard, [])[:telemetry] || Worker def start_link(_args \\ :ok) do Supervisor.start_link(__MODULE__, [], name: __MODULE__) @@ -41,7 +41,7 @@ defmodule ExLimiter.Storage.PG2Shard.Supervisor do defp shard_count do :ex_limiter - |> Application.get_env(PG2Shard) + |> Application.get_env(PG2Shard, []) |> Keyword.get(:shard_count, 0) end end diff --git a/mix.exs b/mix.exs index fbe2966..62acab2 100644 --- a/mix.exs +++ b/mix.exs @@ -1,7 +1,7 @@ defmodule ExLimiter.Mixfile do use Mix.Project - @version "1.5.0" + @version "1.5.1" def project do [ @@ -31,7 +31,7 @@ defmodule ExLimiter.Mixfile do {:memcachir, "~> 3.3.1", [optional: true] ++ run_in_test()}, {:plug, "~> 1.4"}, {:libring, "~> 1.0"}, - {:telemetry, "~> 0.4 or ~> 1.0"}, + {:telemetry, "~> 1.0"}, {:ex2ms, "~> 1.5"}, {:ex_doc, "~> 0.19", only: :dev}, {:styler, ">= 0.0.0", only: [:dev, :test], runtime: false} diff --git a/test/test_helper.exs b/test/test_helper.exs index 898d610..ab0813f 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -1,2 +1,3 @@ +Application.put_env(:ex_limiter, ExLimiter.Storage.PG2Shard, shard_count: 20) ExUnit.start() {:ok, _pid} = ExLimiter.Storage.PG2Shard.Supervisor.start_link() From f4b05df7b810298fde997b5fac6a4c2c004df415 Mon Sep 17 00:00:00 2001 From: Milton Mazzarri Date: Tue, 13 Jan 2026 15:10:05 -0600 Subject: [PATCH 08/16] set a default value for compile opts --- lib/ex_limiter/storage/pg2_shard/pruner.ex | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/lib/ex_limiter/storage/pg2_shard/pruner.ex b/lib/ex_limiter/storage/pg2_shard/pruner.ex index a993f32..763ea2d 100644 --- a/lib/ex_limiter/storage/pg2_shard/pruner.ex +++ b/lib/ex_limiter/storage/pg2_shard/pruner.ex @@ -11,12 +11,12 @@ defmodule ExLimiter.Storage.PG2Shard.Pruner do alias ExLimiter.Utils @table_name :exlimiter_buckets - @opts Application.compile_env(:ex_limiter, PG2Shard) - @expiry @opts[:expiry] || 10 * 60_000 - @eviction_count @opts[:eviction_count] || 1000 - @max_size @opts[:max_size] || 50_000 - @prune_interval @opts[:prune_interval] || 5_000 - @eviction_interval @opts[:eviction_interval] || 30_000 + @compile_opts Application.compile_env(:ex_limiter, PG2Shard, []) + @expiry @compile_opts[:expiry] || 10 * 60_000 + @eviction_count @compile_opts[:eviction_count] || 1000 + @max_size @compile_opts[:max_size] || 50_000 + @prune_interval @compile_opts[:prune_interval] || 5_000 + @eviction_interval @compile_opts[:eviction_interval] || 30_000 def start_link(_args \\ :ok) do GenServer.start_link(__MODULE__, [], name: __MODULE__) From 20156be189e30484bec3898bb6b3e29efd41ce86 Mon Sep 17 00:00:00 2001 From: Milton Mazzarri Date: Tue, 13 Jan 2026 15:21:08 -0600 Subject: [PATCH 09/16] remove ex2ms dependency --- lib/ex_limiter/storage/pg2_shard/pruner.ex | 11 ++++------- mix.exs | 1 - mix.lock | 1 - 3 files changed, 4 insertions(+), 9 deletions(-) diff --git a/lib/ex_limiter/storage/pg2_shard/pruner.ex b/lib/ex_limiter/storage/pg2_shard/pruner.ex index 763ea2d..16a754b 100644 --- a/lib/ex_limiter/storage/pg2_shard/pruner.ex +++ b/lib/ex_limiter/storage/pg2_shard/pruner.ex @@ -5,15 +5,12 @@ defmodule ExLimiter.Storage.PG2Shard.Pruner do """ use GenServer - import Ex2ms - alias ExLimiter.Storage.PG2Shard alias ExLimiter.Utils @table_name :exlimiter_buckets @compile_opts Application.compile_env(:ex_limiter, PG2Shard, []) - @expiry @compile_opts[:expiry] || 10 * 60_000 - @eviction_count @compile_opts[:eviction_count] || 1000 + @expiry @compile_opts[:expiry] || @eviction_count(@compile_opts[:eviction_count] || 1000) @max_size @compile_opts[:max_size] || 50_000 @prune_interval @compile_opts[:prune_interval] || 5_000 @eviction_interval @compile_opts[:eviction_interval] || 30_000 @@ -41,9 +38,9 @@ defmodule ExLimiter.Storage.PG2Shard.Pruner do count = :ets.select_delete( table, - fun do - {_, updated_at, _} when updated_at < ^now - ^@expiry -> true - end + [ + {{:_, :"$1", :_}, [{:<, :"$1", {:-, {:const, now}, {:const, @expiry}}}], [true]} + ] ) :telemetry.execute([:ex_limiter, :shards, :expirations], %{value: count}) diff --git a/mix.exs b/mix.exs index 62acab2..b5cdf7d 100644 --- a/mix.exs +++ b/mix.exs @@ -32,7 +32,6 @@ defmodule ExLimiter.Mixfile do {:plug, "~> 1.4"}, {:libring, "~> 1.0"}, {:telemetry, "~> 1.0"}, - {:ex2ms, "~> 1.5"}, {:ex_doc, "~> 0.19", only: :dev}, {:styler, ">= 0.0.0", only: [:dev, :test], runtime: false} ] diff --git a/mix.lock b/mix.lock index 4318c2d..9df4c5d 100644 --- a/mix.lock +++ b/mix.lock @@ -2,7 +2,6 @@ "connection": {:hex, :connection, "1.1.0", "ff2a49c4b75b6fb3e674bfc5536451607270aac754ffd1bdfe175abe4a6d7a68", [:mix], [], "hexpm", "722c1eb0a418fbe91ba7bd59a47e28008a189d47e37e0e7bb85585a016b2869c"}, "earmark_parser": {:hex, :earmark_parser, "1.4.44", "f20830dd6b5c77afe2b063777ddbbff09f9759396500cdbe7523efd58d7a339c", [:mix], [], "hexpm", "4778ac752b4701a5599215f7030989c989ffdc4f6df457c5f36938cc2d2a2750"}, "elasticachex": {:hex, :elasticachex, "1.1.3", "c5cc1255b3f25c53df16206959816824cc65e65be5be8462af069be59af63013", [:mix], [{:socket, "~> 0.3", [hex: :socket, repo: "hexpm", optional: false]}], "hexpm", "425814b1406729f2f037ff3b90755162b1d8b7fef23b3c23deac295e05cec2fc"}, - "ex2ms": {:hex, :ex2ms, "1.7.0", "45b9f523d0b777667ded60070d82d871a37e294f0b6c5b8eca86771f00f82ee1", [:mix], [], "hexpm", "2589eee51f81f1b1caa6d08c990b1ad409215fe6f64c73f73c67d36ed10be827"}, "ex_doc": {:hex, :ex_doc, "0.39.3", "519c6bc7e84a2918b737aec7ef48b96aa4698342927d080437f61395d361dcee", [:mix], [{:earmark_parser, "~> 1.4.44", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_c, ">= 0.1.0", [hex: :makeup_c, repo: "hexpm", optional: true]}, {:makeup_elixir, "~> 0.14 or ~> 1.0", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1 or ~> 1.0", [hex: :makeup_erlang, repo: "hexpm", optional: false]}, {:makeup_html, ">= 0.1.0", [hex: :makeup_html, repo: "hexpm", optional: true]}], "hexpm", "0590955cf7ad3b625780ee1c1ea627c28a78948c6c0a9b0322bd976a079996e1"}, "herd": {:hex, :herd, "0.4.3", "97469cf289c1e89a4f2b356da486ae5a354751f91c10cd3749af6aedebd9a775", [:mix], [{:libring, "~> 1.1", [hex: :libring, repo: "hexpm", optional: false]}, {:poolboy, "~> 1.5", [hex: :poolboy, repo: "hexpm", optional: false]}], "hexpm", "44bfd2c42a206431495d5103a77f52a992f4f1391a13459a9e8fd7b143cd99c9"}, "libring": {:hex, :libring, "1.7.0", "4f245d2f1476cd7ed8f03740f6431acba815401e40299208c7f5c640e1883bda", [:mix], [], "hexpm", "070e3593cb572e04f2c8470dd0c119bc1817a7a0a7f88229f43cf0345268ec42"}, From 02be4579a7b5f87c50d2e316f157ab7796503e10 Mon Sep 17 00:00:00 2001 From: Milton Mazzarri Date: Tue, 13 Jan 2026 15:34:36 -0600 Subject: [PATCH 10/16] bump elixir/erlang versions in ci --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index aeb11f9..2fa494e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,8 +23,8 @@ jobs: include: - elixir: "1.15" otp: "25" - - elixir: "1.18" - otp: "27" + - elixir: "1.19" + otp: "28" lint: true steps: - name: Checkout code From 943040585e3b15084f697b4723ec66fe3436d089 Mon Sep 17 00:00:00 2001 From: Milton Mazzarri Date: Wed, 14 Jan 2026 10:16:13 -0600 Subject: [PATCH 11/16] add ExLimiter.DataCase --- test/ex_limiter/plug_test.exs | 6 ++---- test/ex_limiter/storage/pg2_shard_test.exs | 22 +++++++------------ test/ex_limiter/storage/pruner_test.exs | 2 +- test/ex_limiter_test.exs | 25 ++++++++-------------- test/support/data_case.ex | 12 +++++++++++ test/support/test_utils.ex | 6 ------ 6 files changed, 32 insertions(+), 41 deletions(-) create mode 100644 test/support/data_case.ex delete mode 100644 test/support/test_utils.ex diff --git a/test/ex_limiter/plug_test.exs b/test/ex_limiter/plug_test.exs index aea108c..300fbaa 100644 --- a/test/ex_limiter/plug_test.exs +++ b/test/ex_limiter/plug_test.exs @@ -1,10 +1,8 @@ defmodule ExLimiter.PlugTest do - use ExUnit.Case + use ExLimiter.DataCase import Plug.Test import Plug.Conn - alias ExLimiter.TestUtils - describe "#call/2" do setup [:setup_limiter, :setup_conn] @@ -73,7 +71,7 @@ defmodule ExLimiter.PlugTest do end defp setup_conn(_) do - random = TestUtils.rand_string() + random = Base.encode64(:crypto.strong_rand_bytes(8)) conn = :get diff --git a/test/ex_limiter/storage/pg2_shard_test.exs b/test/ex_limiter/storage/pg2_shard_test.exs index 0e6ef0f..de19364 100644 --- a/test/ex_limiter/storage/pg2_shard_test.exs +++ b/test/ex_limiter/storage/pg2_shard_test.exs @@ -1,11 +1,13 @@ defmodule ExLimiter.Storage.PG2ShardTest do - use ExUnit.Case, async: false + use ExLimiter.DataCase, async: false alias ExLimiter.PG2Limiter - alias ExLimiter.TestUtils + + setup do + {:ok, bucket_name: bucket_name()} + end describe "#consume" do - test "it will rate limit" do - bucket_name = bucket() + test "it will rate limit", %{bucket_name: bucket_name} do {:ok, bucket} = PG2Limiter.consume(bucket_name, 1) assert bucket.key == bucket_name @@ -20,9 +22,7 @@ defmodule ExLimiter.Storage.PG2ShardTest do end describe "#delete" do - test "It will wipe a bucket" do - bucket_name = bucket() - + test "It will wipe a bucket", %{bucket_name: bucket_name} do {:ok, bucket} = PG2Limiter.consume(bucket_name, 5) assert bucket.value >= 500 @@ -36,16 +36,10 @@ defmodule ExLimiter.Storage.PG2ShardTest do end describe "#remaining" do - test "It will properly deconvert the remaining capacity in a bucket" do - bucket_name = bucket() - + test "It will properly deconvert the remaining capacity in a bucket", %{bucket_name: bucket_name} do {:ok, bucket} = PG2Limiter.consume(bucket_name, 5) assert PG2Limiter.remaining(bucket) == 5 end end - - defp bucket() do - "test_bucket_#{TestUtils.rand_string()}" - end end diff --git a/test/ex_limiter/storage/pruner_test.exs b/test/ex_limiter/storage/pruner_test.exs index 2d54b10..bb6c30a 100644 --- a/test/ex_limiter/storage/pruner_test.exs +++ b/test/ex_limiter/storage/pruner_test.exs @@ -1,5 +1,5 @@ defmodule ExLimiter.Storage.PG2Shard.PrunerTest do - use ExUnit.Case, async: false + use ExLimiter.DataCase, async: false alias ExLimiter.Storage.PG2Shard.Pruner @table_name :pruner_test diff --git a/test/ex_limiter_test.exs b/test/ex_limiter_test.exs index d9c0618..f841634 100644 --- a/test/ex_limiter_test.exs +++ b/test/ex_limiter_test.exs @@ -1,11 +1,13 @@ defmodule ExLimiterTest do - use ExUnit.Case - alias ExLimiter.TestUtils + use ExLimiter.DataCase, async: true doctest ExLimiter + setup do + {:ok, bucket_name: bucket_name()} + end + describe "#consume" do - test "it will rate limit" do - bucket_name = bucket() + test "it will rate limit", %{bucket_name: bucket_name} do {:ok, bucket} = ExLimiter.consume(bucket_name, 1) assert bucket.key == bucket_name @@ -18,8 +20,7 @@ defmodule ExLimiterTest do {:error, :rate_limited} = ExLimiter.consume(bucket_name, 6) end - test "it will rate limit for custom scale/limits" do - bucket_name = bucket() + test "it will rate limit for custom scale/limits", %{bucket_name: bucket_name} do args = [scale: 60_000, limit: 50] {:ok, bucket} = ExLimiter.consume(bucket_name, 1, args) @@ -35,9 +36,7 @@ defmodule ExLimiterTest do end describe "#delete" do - test "It will wipe a bucket" do - bucket_name = bucket() - + test "It will wipe a bucket", %{bucket_name: bucket_name} do {:ok, bucket} = ExLimiter.consume(bucket_name, 5) assert bucket.value >= 500 @@ -51,16 +50,10 @@ defmodule ExLimiterTest do end describe "#remaining" do - test "It will properly deconvert the remaining capacity in a bucket" do - bucket_name = bucket() - + test "It will properly deconvert the remaining capacity in a bucket", %{bucket_name: bucket_name} do {:ok, bucket} = ExLimiter.consume(bucket_name, 5) assert ExLimiter.remaining(bucket) == 5 end end - - defp bucket() do - "test_bucket_#{TestUtils.rand_string()}" - end end diff --git a/test/support/data_case.ex b/test/support/data_case.ex new file mode 100644 index 0000000..0f9c103 --- /dev/null +++ b/test/support/data_case.ex @@ -0,0 +1,12 @@ +defmodule ExLimiter.DataCase do + @moduledoc false + use ExUnit.CaseTemplate + + using do + quote do + import ExLimiter.DataCase + end + end + + def bucket_name, do: Base.encode64(:crypto.strong_rand_bytes(8)) +end diff --git a/test/support/test_utils.ex b/test/support/test_utils.ex deleted file mode 100644 index f18083e..0000000 --- a/test/support/test_utils.ex +++ /dev/null @@ -1,6 +0,0 @@ -defmodule ExLimiter.TestUtils do - def rand_string() do - :crypto.strong_rand_bytes(8) - |> Base.encode64() - end -end \ No newline at end of file From 6b9c12303488e698eb084134efd15eb3d0b19c0c Mon Sep 17 00:00:00 2001 From: Milton Mazzarri Date: Wed, 14 Jan 2026 10:27:54 -0600 Subject: [PATCH 12/16] remove ExLimiter.Utils module --- lib/ex_limiter/base.ex | 3 +- lib/ex_limiter/bucket.ex | 4 +-- lib/ex_limiter/storage/memcache.ex | 11 ++++--- lib/ex_limiter/storage/pg2_shard/pruner.ex | 24 ++++++++++++-- lib/ex_limiter/utils.ex | 38 ---------------------- 5 files changed, 30 insertions(+), 50 deletions(-) delete mode 100644 lib/ex_limiter/utils.ex diff --git a/lib/ex_limiter/base.ex b/lib/ex_limiter/base.ex index 664b491..80d783d 100644 --- a/lib/ex_limiter/base.ex +++ b/lib/ex_limiter/base.ex @@ -9,7 +9,6 @@ defmodule ExLimiter.Base do end """ alias ExLimiter.Bucket - alias ExLimiter.Utils defmacro __using__(storage: storage) do quote do @@ -61,7 +60,7 @@ defmodule ExLimiter.Base do @doc false def update(%Bucket{value: value, last: time} = b) do - now = Utils.now() + now = System.system_time(:millisecond) amount = max(value - (now - time), 0) %{b | last: now, value: amount} diff --git a/lib/ex_limiter/bucket.ex b/lib/ex_limiter/bucket.ex index 1e62220..506c403 100644 --- a/lib/ex_limiter/bucket.ex +++ b/lib/ex_limiter/bucket.ex @@ -1,7 +1,5 @@ defmodule ExLimiter.Bucket do @moduledoc false - alias ExLimiter.Utils - @type t :: %__MODULE__{} defstruct key: nil, @@ -9,7 +7,7 @@ defmodule ExLimiter.Bucket do last: nil, version: %{} - def new(key), do: %__MODULE__{key: key, last: Utils.now()} + def new(key), do: %__MODULE__{key: key, last: System.system_time(:millisecond)} def new(contents, key) when is_map(contents) do struct(__MODULE__, Map.put(contents, :key, key)) diff --git a/lib/ex_limiter/storage/memcache.ex b/lib/ex_limiter/storage/memcache.ex index 253fd2b..ff2d982 100644 --- a/lib/ex_limiter/storage/memcache.ex +++ b/lib/ex_limiter/storage/memcache.ex @@ -5,8 +5,6 @@ defmodule ExLimiter.Storage.Memcache do """ use ExLimiter.Storage - alias ExLimiter.Utils - def fetch(%Bucket{key: key}) do key_map = keys(key) @@ -88,12 +86,17 @@ defmodule ExLimiter.Storage.Memcache do defp add_result(%{version: versions} = acc, bucket_key, {val, cas}) do acc - |> Map.put(bucket_key, Utils.parse_integer(val)) + |> Map.put(bucket_key, parse_integer(val)) |> Map.put(:version, Map.put(versions, bucket_key, cas)) end defp add_result(acc, bucket_key, _), do: add_result(acc, bucket_key, default(bucket_key)) defp default(:value), do: {0, 0} - defp default(:last), do: {Utils.now(), 0} + defp default(:last), do: {System.system_time(:millisecond), 0} + + def parse_integer(val) when is_binary(val), do: val |> Integer.parse() |> parse_integer() + def parse_integer(val) when is_integer(val), do: val + def parse_integer(:error), do: :error + def parse_integer({val, _}), do: val end diff --git a/lib/ex_limiter/storage/pg2_shard/pruner.ex b/lib/ex_limiter/storage/pg2_shard/pruner.ex index 16a754b..8d5919a 100644 --- a/lib/ex_limiter/storage/pg2_shard/pruner.ex +++ b/lib/ex_limiter/storage/pg2_shard/pruner.ex @@ -6,7 +6,6 @@ defmodule ExLimiter.Storage.PG2Shard.Pruner do use GenServer alias ExLimiter.Storage.PG2Shard - alias ExLimiter.Utils @table_name :exlimiter_buckets @compile_opts Application.compile_env(:ex_limiter, PG2Shard, []) @@ -33,7 +32,7 @@ defmodule ExLimiter.Storage.PG2Shard.Pruner do def handle_info(:expire, table) do expire() - now = Utils.now() + now = System.system_time(:millisecond) count = :ets.select_delete( @@ -61,7 +60,7 @@ defmodule ExLimiter.Storage.PG2Shard.Pruner do end def remove(table, count) do - Utils.batched_ets(table, {:"$1", :_, :_}, 1000, count, fn keys -> + batched_ets(table, {:"$1", :_, :_}, 1000, count, fn keys -> for [key] <- keys, do: :ets.delete(table, key) end) @@ -70,4 +69,23 @@ defmodule ExLimiter.Storage.PG2Shard.Pruner do defp prune, do: Process.send_after(self(), :prune, @prune_interval) defp expire, do: Process.send_after(self(), :expire, @eviction_interval) + + defp batched_ets(table, match_spec, batch_size, total, fnc) do + table + |> :ets.match(match_spec, batch_size) + |> process_batch(0, total, fnc) + end + + defp process_batch(_, count, total, _) when count >= total, do: count + + defp process_batch({elem, cnt}, count, total, fnc) do + fnc.(elem) + + cnt + |> :ets.match() + |> process_batch(length(elem) + count, total, fnc) + end + + defp process_batch(:"$end_of_table", count, _, _), do: count + end diff --git a/lib/ex_limiter/utils.ex b/lib/ex_limiter/utils.ex deleted file mode 100644 index aa33719..0000000 --- a/lib/ex_limiter/utils.ex +++ /dev/null @@ -1,38 +0,0 @@ -defmodule ExLimiter.Utils do - @moduledoc false - def now, do: :os.system_time(:millisecond) - - def batched_ets(table, match_spec \\ {:"$1", :_, :_}, batch_size \\ 1000, total \\ 100_000, fnc) do - table - |> :ets.match(match_spec, batch_size) - |> process_batch(0, total, fnc) - end - - defp process_batch(_, count, total, _) when count >= total, do: count - - defp process_batch({elem, cnt}, count, total, fnc) do - fnc.(elem) - - cnt - |> :ets.match() - |> process_batch(length(elem) + count, total, fnc) - end - - defp process_batch(:"$end_of_table", count, _, _), do: count - - def ets_stream(table) do - Stream.resource( - fn -> :ets.first(table) end, - fn - :"$end_of_table" -> {:halt, nil} - previous_key -> {[previous_key], :ets.next(table, previous_key)} - end, - fn _ -> :ok end - ) - end - - def parse_integer(val) when is_binary(val), do: val |> Integer.parse() |> parse_integer() - def parse_integer(val) when is_integer(val), do: val - def parse_integer(:error), do: :error - def parse_integer({val, _}), do: val -end From ca4ec2e9af7173d2f39f9231a8139e6f279accc7 Mon Sep 17 00:00:00 2001 From: Milton Mazzarri Date: Wed, 14 Jan 2026 11:12:48 -0600 Subject: [PATCH 13/16] fix compile options for pruner --- lib/ex_limiter/storage/pg2_shard/pruner.ex | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/ex_limiter/storage/pg2_shard/pruner.ex b/lib/ex_limiter/storage/pg2_shard/pruner.ex index 8d5919a..2c48045 100644 --- a/lib/ex_limiter/storage/pg2_shard/pruner.ex +++ b/lib/ex_limiter/storage/pg2_shard/pruner.ex @@ -9,7 +9,8 @@ defmodule ExLimiter.Storage.PG2Shard.Pruner do @table_name :exlimiter_buckets @compile_opts Application.compile_env(:ex_limiter, PG2Shard, []) - @expiry @compile_opts[:expiry] || @eviction_count(@compile_opts[:eviction_count] || 1000) + @expiry @compile_opts[:expiry_interval] || to_timeout(minute: 1) + @eviction_count @compile_opts[:eviction_count] || 1000 @max_size @compile_opts[:max_size] || 50_000 @prune_interval @compile_opts[:prune_interval] || 5_000 @eviction_interval @compile_opts[:eviction_interval] || 30_000 @@ -87,5 +88,4 @@ defmodule ExLimiter.Storage.PG2Shard.Pruner do end defp process_batch(:"$end_of_table", count, _, _), do: count - end From 123e43114aacc72d17711b19250bf0598fee2cfd Mon Sep 17 00:00:00 2001 From: Milton Mazzarri Date: Wed, 14 Jan 2026 11:13:13 -0600 Subject: [PATCH 14/16] apply formatter in unit test suite --- .formatter.exs | 2 +- test/ex_limiter/plug_test.exs | 3 ++- test/ex_limiter/storage/pg2_shard_test.exs | 1 + test/ex_limiter_test.exs | 1 + test/support/pg2_limiter.ex | 1 + 5 files changed, 6 insertions(+), 2 deletions(-) diff --git a/.formatter.exs b/.formatter.exs index a378d5e..7e1db10 100644 --- a/.formatter.exs +++ b/.formatter.exs @@ -3,7 +3,7 @@ "mix.exs", ".formatter.exs", "config/*.exs", - "lib/**/*.ex" + "{lib,test}/**/*.{ex,exs}" ], line_length: 120, plugins: [Styler], diff --git a/test/ex_limiter/plug_test.exs b/test/ex_limiter/plug_test.exs index 300fbaa..bd30459 100644 --- a/test/ex_limiter/plug_test.exs +++ b/test/ex_limiter/plug_test.exs @@ -1,7 +1,8 @@ defmodule ExLimiter.PlugTest do use ExLimiter.DataCase - import Plug.Test + import Plug.Conn + import Plug.Test describe "#call/2" do setup [:setup_limiter, :setup_conn] diff --git a/test/ex_limiter/storage/pg2_shard_test.exs b/test/ex_limiter/storage/pg2_shard_test.exs index de19364..06bf235 100644 --- a/test/ex_limiter/storage/pg2_shard_test.exs +++ b/test/ex_limiter/storage/pg2_shard_test.exs @@ -1,5 +1,6 @@ defmodule ExLimiter.Storage.PG2ShardTest do use ExLimiter.DataCase, async: false + alias ExLimiter.PG2Limiter setup do diff --git a/test/ex_limiter_test.exs b/test/ex_limiter_test.exs index f841634..e713dc0 100644 --- a/test/ex_limiter_test.exs +++ b/test/ex_limiter_test.exs @@ -1,5 +1,6 @@ defmodule ExLimiterTest do use ExLimiter.DataCase, async: true + doctest ExLimiter setup do diff --git a/test/support/pg2_limiter.ex b/test/support/pg2_limiter.ex index c994a1f..917bcfd 100644 --- a/test/support/pg2_limiter.ex +++ b/test/support/pg2_limiter.ex @@ -1,3 +1,4 @@ defmodule ExLimiter.PG2Limiter do + @moduledoc false use ExLimiter.Base, storage: ExLimiter.Storage.PG2Shard end From 2ccd44ac8d317301fe945a67c5b5f321d757cfcc Mon Sep 17 00:00:00 2001 From: Milton Mazzarri Date: Wed, 14 Jan 2026 11:14:13 -0600 Subject: [PATCH 15/16] add unit test to verify deletion of expired records --- test/ex_limiter/storage/pruner_test.exs | 35 ++++++++++++++++++++++--- 1 file changed, 32 insertions(+), 3 deletions(-) diff --git a/test/ex_limiter/storage/pruner_test.exs b/test/ex_limiter/storage/pruner_test.exs index bb6c30a..6525764 100644 --- a/test/ex_limiter/storage/pruner_test.exs +++ b/test/ex_limiter/storage/pruner_test.exs @@ -1,15 +1,44 @@ defmodule ExLimiter.Storage.PG2Shard.PrunerTest do use ExLimiter.DataCase, async: false + alias ExLimiter.Storage.PG2Shard.Pruner - @table_name :pruner_test describe "#remove" do test "it will remove stale bucket entries" do - table = :ets.new(@table_name, [:set, :public, read_concurrency: true, write_concurrency: true]) + table = :ets.new(:pruner_test, [:set, :public, read_concurrency: true, write_concurrency: true]) - :ets.insert(table, {"bucket", :os.system_time(:millisecond), %ExLimiter.Bucket{}}) + :ets.insert(table, {"bucket", System.system_time(:millisecond), %ExLimiter.Bucket{}}) assert Pruner.remove(table, 1) == 1 assert :ets.lookup(table, "bucket") == [] end end + + describe "#expire" do + def expiration_handler([:ex_limiter, :shards, :expirations], %{value: count}, _, %{test_pid: test_pid}) do + send(test_pid, {:expired_records, count}) + end + + test "it will remove expired bucket entries" do + table_name = :expired_test + table = :ets.new(table_name, [:set, :public, read_concurrency: true, write_concurrency: true]) + + # Listen for the expirations events so we can assert on it + test_pid = self() + + :telemetry.attach("expiration-test", [:ex_limiter, :shards, :expirations], &__MODULE__.expiration_handler/4, %{ + test_pid: test_pid + }) + + on_exit(fn -> :telemetry.detach("expiration-test") end) + + time = System.system_time(:millisecond) - to_timeout(second: 100) + + :ets.insert(table, {"bucket", time, %ExLimiter.Bucket{}}) + + assert Pruner.handle_info(:expire, table) == {:noreply, table} + + assert_receive {:expired_records, 1}, to_timeout(second: 5) + assert :ets.lookup(table, "bucket") == [] + end + end end From 331561a43ec899df061ae96cc0e5c25b0e6de6ff Mon Sep 17 00:00:00 2001 From: Milton Mazzarri Date: Wed, 14 Jan 2026 11:19:09 -0600 Subject: [PATCH 16/16] bump minimum elixir version in ci --- .github/workflows/ci.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2fa494e..957e642 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -21,8 +21,8 @@ jobs: strategy: matrix: include: - - elixir: "1.15" - otp: "25" + - elixir: "1.17" + otp: "27" - elixir: "1.19" otp: "28" lint: true