From 48b493026fbcbddff29422901b82e9e098872df2 Mon Sep 17 00:00:00 2001 From: nico piderman Date: Tue, 13 Jan 2026 15:27:23 +0100 Subject: [PATCH 1/5] Optimize S3 fetcher with conditional GETs and new download_if_changed hook MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - add optional download_if_changed/2 to Fetcher behaviour and wire update loop to use it when available - implement conditional GETs in the S3 fetcher via If-None-Match and 304 handling - switch S3 version checks to ETag-only and document versioned/non‑versioned behavior - add S3 tests covering conditional GET and header usage --- AGENTS.md | 35 ++++ lib/remote_persistent_term.ex | 63 +++++-- lib/remote_persistent_term/fetcher.ex | 10 ++ lib/remote_persistent_term/fetcher/s3.ex | 154 ++++++++++++++---- .../fetcher/s3_test.exs | 36 +++- 5 files changed, 243 insertions(+), 55 deletions(-) create mode 100644 AGENTS.md diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..4830628 --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,35 @@ +# Repository Guidelines + +## Project Structure & Module Organization +- `lib/remote_persistent_term.ex` defines the core behaviour and `use RemotePersistentTerm` macro. +- `lib/remote_persistent_term/fetcher/` holds fetcher implementations (HTTP, S3, Static) and helpers like HTTP cache logic. +- Tests live in `test/` with paths mirroring modules (e.g., `test/remote_persistent_term/fetcher/http_test.exs`). +- Generated artifacts (`_build/`, `deps/`, `doc/`, `cover/`) are outputs of Mix tasks and should not be edited by hand. + +## Build, Test, and Development Commands +- `mix deps.get` installs dependencies. +- `mix compile` builds the library. +- `mix test` runs the full ExUnit suite. +- `mix test test/remote_persistent_term/fetcher/http_test.exs:30` runs a focused test by file and line. +- `mix format` applies the project formatter (see `.formatter.exs`). +- `mix docs` generates ExDoc output into `doc/`. + +## Coding Style & Naming Conventions +- Follow `mix format` output (Elixir defaults to 2-space indentation). +- Modules use `CamelCase`; files and functions use `snake_case` (predicates end in `?`). +- Keep option keys consistent with `RemotePersistentTerm` options and fetcher configuration keys. + +## Testing Guidelines +- Tests use ExUnit; Mox mocks the ExAws client and Bypass is used for HTTP fetcher tests. +- Prefer deterministic tests and keep external network access mocked or bypassed. +- Name tests with clear behaviour statements; add coverage for new fetcher logic and option validation. + +## Commit & Pull Request Guidelines +- Commit messages are short, imperative, and capitalized (e.g., “Improve logs”, “Fix tests”, “Increment version”). +- Keep commits scoped to one change; avoid unrelated refactors in the same commit. +- PRs should include a concise summary, motivation, and the tests you ran (or note if none). +- If a change affects public behaviour or configuration, update documentation and mention it in the PR. + +## Notes for Contributors +- CI runs `mix test` on recent OTP/Elixir versions; ensure your changes pass locally before pushing. +- When touching S3 or HTTP fetchers, prefer tests that use Mox/Bypass rather than real services. diff --git a/lib/remote_persistent_term.ex b/lib/remote_persistent_term.ex index 55e4534..e40bff4 100644 --- a/lib/remote_persistent_term.ex +++ b/lib/remote_persistent_term.ex @@ -268,21 +268,43 @@ defmodule RemotePersistentTerm do start_meta, fn -> {status, version} = - with {:ok, current_version} <- state.fetcher_mod.current_version(state.fetcher_state), - true <- state.current_version != current_version, - :ok <- download_and_store_term(state, deserialize_fun, put_fun) do - {:updated, current_version} + if function_exported?(state.fetcher_mod, :download_if_changed, 2) do + case state.fetcher_mod.download_if_changed( + state.fetcher_state, + state.current_version + ) do + {:ok, term, new_version} -> + case store_term(state, deserialize_fun, put_fun, term) do + :ok -> + {:updated, new_version} + + {:error, reason} -> + log_update_error(state.name, reason) + {:not_updated, state.current_version} + end + + {:not_modified, version} -> + Logger.info("#{state.name} - up to date") + {:not_updated, version || state.current_version} + + {:error, reason} -> + log_update_error(state.name, reason) + {:not_updated, state.current_version} + end else - false -> - Logger.info("#{state.name} - up to date") - {:not_updated, state.current_version} - - {:error, reason} -> - Logger.error( - "#{state.name} - failed to update remote term, reason: #{inspect(reason)}" - ) - - {:not_updated, state.current_version} + with {:ok, current_version} <- state.fetcher_mod.current_version(state.fetcher_state), + true <- state.current_version != current_version, + :ok <- download_and_store_term(state, deserialize_fun, put_fun) do + {:updated, current_version} + else + false -> + Logger.info("#{state.name} - up to date") + {:not_updated, state.current_version} + + {:error, reason} -> + log_update_error(state.name, reason) + {:not_updated, state.current_version} + end end {version, Map.put(start_meta, :status, status)} @@ -305,9 +327,18 @@ defmodule RemotePersistentTerm do @doc false def validate_options(opts), do: NimbleOptions.validate(opts, @opts_schema) + defp log_update_error(name, reason) do + Logger.error("#{name} - failed to update remote term, reason: #{inspect(reason)}") + end + defp download_and_store_term(state, deserialize_fun, put_fun) do - with {:ok, term} <- state.fetcher_mod.download(state.fetcher_state), - {:ok, decompressed} <- maybe_decompress(state, term), + with {:ok, term} <- state.fetcher_mod.download(state.fetcher_state) do + store_term(state, deserialize_fun, put_fun, term) + end + end + + defp store_term(state, deserialize_fun, put_fun, term) do + with {:ok, decompressed} <- maybe_decompress(state, term), {:ok, deserialized} <- deserialize_fun.(decompressed) do put_fun.(deserialized) end diff --git a/lib/remote_persistent_term/fetcher.ex b/lib/remote_persistent_term/fetcher.ex index 30933a5..888a78e 100644 --- a/lib/remote_persistent_term/fetcher.ex +++ b/lib/remote_persistent_term/fetcher.ex @@ -10,6 +10,8 @@ defmodule RemotePersistentTerm.Fetcher do @type state :: term() @type opts :: Keyword.t() @type version :: String.t() + @type download_if_changed_result :: + {:ok, term(), version()} | {:not_modified, version() | nil} | {:error, term()} @doc """ Initialize the implementation specific state of the Fetcher. @@ -26,4 +28,12 @@ defmodule RemotePersistentTerm.Fetcher do Download the term from the remote source. """ @callback download(state()) :: {:ok, term()} | {:error, term()} + + @doc """ + Optionally download the term only if it has changed. When implemented, it should + return `{:not_modified, current_version}` for an unchanged term or `{:ok, term, new_version}`. + """ + @callback download_if_changed(state(), version() | nil) :: download_if_changed_result + + @optional_callbacks download_if_changed: 2 end diff --git a/lib/remote_persistent_term/fetcher/s3.ex b/lib/remote_persistent_term/fetcher/s3.ex index f2d16f2..4479355 100644 --- a/lib/remote_persistent_term/fetcher/s3.ex +++ b/lib/remote_persistent_term/fetcher/s3.ex @@ -1,6 +1,26 @@ defmodule RemotePersistentTerm.Fetcher.S3 do @moduledoc """ A Fetcher implementation for AWS S3. + + ## Versioned vs. non-versioned buckets + + This fetcher works with both versioned and non-versioned buckets. It uses the object's + `ETag` as a change token and performs conditional GETs with `If-None-Match` to avoid + re-downloading unchanged data. + + - **Versioned buckets**: `HEAD`/`GET` responses include `ETag`; the fetcher uses it for + change detection. The latest object is always whatever S3 returns for the key (no explicit + version ID required). + - **Non-versioned buckets**: only `ETag` is available, which is sufficient to detect + content changes. Overwriting an object with identical bytes may keep the same `ETag`, + which is fine because the content is unchanged. + + ## S3-compatible services + + S3-compatible providers (e.g., DigitalOcean Spaces, Linode Object Storage) should work + as long as they support standard S3 headers: `ETag`, `If-None-Match`, and `304 Not Modified`. + If a provider ignores conditional requests, the fetcher will still function but will + download on every refresh. """ require Logger @@ -82,8 +102,8 @@ defmodule RemotePersistentTerm.Fetcher.S3 do @impl true def current_version(state) do - with {:ok, versions} <- list_object_versions(state), - {:ok, %{etag: etag, version_id: version}} <- find_latest(versions) do + with {:ok, %{headers: headers}} <- head_object(state), + {:ok, version} <- extract_version(headers) do Logger.info( bucket: state.bucket, key: state.key, @@ -91,8 +111,11 @@ defmodule RemotePersistentTerm.Fetcher.S3 do message: "Found latest version of object" ) - {:ok, etag} + {:ok, version} else + {:error, {:http_error, 404, _}} -> + {:error, "could not find s3://#{state.bucket}/#{state.key}"} + {:error, {:unexpected_response, %{body: reason}}} -> {:error, reason} @@ -133,60 +156,127 @@ defmodule RemotePersistentTerm.Fetcher.S3 do end end - defp list_object_versions(state) do + @impl true + def download_if_changed(state, current_version) do res = - aws_client_request( - &ExAws.S3.get_bucket_object_versions/2, + get_object_request( state, - prefix: state.key + if_none_match_opts(current_version), + &failover_on_error?/1 ) - with {:ok, %{body: %{versions: versions}}} <- res do - {:ok, versions} + case res do + {:ok, %{status_code: 304}} -> + {:not_modified, current_version} + + {:error, {:http_error, 304, _}} -> + {:not_modified, current_version} + + {:ok, %{body: body, headers: headers}} -> + with {:ok, version} <- extract_version(headers) do + {:ok, body, version} + end + + {:error, reason} -> + {:error, inspect(reason)} end end defp get_object(state) do - aws_client_request(&ExAws.S3.get_object/2, state, state.key) + get_object_request(state, []) + end + + defp get_object_request(state, opts, failover_on_error? \\ fn _ -> true end) do + aws_client_request( + fn bucket, request_opts -> ExAws.S3.get_object(bucket, state.key, request_opts) end, + state, + opts, + failover_on_error? + ) + end + + defp head_object(state) do + aws_client_request(&ExAws.S3.head_object/2, state, state.key) end - defp find_latest([_ | _] = contents) do - Enum.find(contents, fn - %{is_latest: "true"} -> - true + defp extract_version(headers) do + case header_value(headers, "etag") do + nil -> {:error, :not_found} + value -> {:ok, normalize_etag(value)} + end + end + + defp header_value(headers, name) do + downcased = String.downcase(name) + + Enum.find_value(headers, fn + {key, value} when is_binary(key) and is_binary(value) -> + if String.downcase(key) == downcased, do: value, else: nil + + {key, value} when is_atom(key) and is_binary(value) -> + if String.downcase(Atom.to_string(key)) == downcased, do: value, else: nil _ -> - false + nil end) - |> case do - res when is_map(res) -> {:ok, res} - _ -> {:error, :not_found} + end + + defp normalize_etag(value) when is_binary(value) do + value + |> String.trim() + |> String.trim("\"") + end + + defp if_none_match_opts(nil), do: [] + defp if_none_match_opts(etag), do: [if_none_match: quote_etag(etag)] + + defp quote_etag(etag) do + etag = String.trim(etag) + + if String.starts_with?(etag, "\"") and String.ends_with?(etag, "\"") do + etag + else + "\"#{etag}\"" end end - defp find_latest(_), do: {:error, :not_found} + defp failover_on_error?({:http_error, 304, _}), do: false + defp failover_on_error?(_reason), do: true + + defp aws_client_request(op, state, opts) do + aws_client_request(op, state, opts, fn _ -> true end) + end - defp aws_client_request(op, %{failover_buckets: nil} = state, opts) do + defp aws_client_request(op, %{failover_buckets: nil} = state, opts, _failover_on_error?) do perform_request(op, state.bucket, state.region, opts) end defp aws_client_request( op, %{ - failover_buckets: [_|_] = failover_buckets + failover_buckets: [_ | _] = failover_buckets } = state, - opts + opts, + failover_on_error? ) do - with {:error, reason} <- perform_request(op, state.bucket, state.region, opts) do - Logger.error(%{ - bucket: state.bucket, - key: state.key, - region: state.region, - reason: inspect(reason), - message: "Failed to fetch from primary bucket, attempting failover buckets" - }) - - try_failover_buckets(op, failover_buckets, opts, state) + case perform_request(op, state.bucket, state.region, opts) do + {:error, reason} = error -> + if failover_on_error?.(reason) do + Logger.error(%{ + bucket: state.bucket, + key: state.key, + region: state.region, + reason: inspect(reason), + message: "Failed to fetch from primary bucket, attempting failover buckets" + }) + + try_failover_buckets(op, failover_buckets, opts, state) + else + error + end + + result -> + result end end diff --git a/test/remote_persistent_term/fetcher/s3_test.exs b/test/remote_persistent_term/fetcher/s3_test.exs index 9e7d34d..69864c8 100644 --- a/test/remote_persistent_term/fetcher/s3_test.exs +++ b/test/remote_persistent_term/fetcher/s3_test.exs @@ -12,8 +12,6 @@ defmodule RemotePersistentTerm.Fetcher.S3Test do [bucket: "failover-bucket-1", region: "failover-region-1"], [bucket: "failover-bucket-2", region: "failover-region-2"] ] - @version "F76V.weh4uOlU15f7a2OLHPgCLXkDpm4" - test "Unknown error returns an error for current_version/1" do expect(AwsClientMock, :request, fn _op, _opts -> {:error, :unknown_error} @@ -84,11 +82,9 @@ defmodule RemotePersistentTerm.Fetcher.S3Test do op_bucket == "failover-bucket-1" && region == "failover-region-1" -> {:ok, %{ - body: %{ - versions: [ - %{version_id: @version, etag: "current-etag", is_latest: "true"} - ] - } + headers: [ + {"etag", "\"current-etag\""} + ] }} true -> @@ -253,4 +249,30 @@ defmodule RemotePersistentTerm.Fetcher.S3Test do assert log =~ "Downloaded object from S3" end end + + describe "download_if_changed/2" do + test "returns not_modified on 304 and sends if-none-match" do + state = %S3{bucket: @bucket, key: @key, region: @region} + + expect(AwsClientMock, :request, fn operation, _opts -> + assert operation.headers["if-none-match"] == "\"current-etag\"" + {:error, {:http_error, 304, %{}}} + end) + + assert {:not_modified, "current-etag"} = + S3.download_if_changed(state, "current-etag") + end + + test "returns body and version on 200" do + state = %S3{bucket: @bucket, key: @key, region: @region} + + expect(AwsClientMock, :request, fn operation, _opts -> + assert operation.headers["if-none-match"] == "\"old-etag\"" + {:ok, %{body: "new-content", headers: [{"etag", "\"new-etag\""}]}} + end) + + assert {:ok, "new-content", "new-etag"} = + S3.download_if_changed(state, "old-etag") + end + end end From f681a3d8e16a2a15885999aacdcd1e80aed2cb8f Mon Sep 17 00:00:00 2001 From: nico piderman Date: Tue, 13 Jan 2026 15:32:25 +0100 Subject: [PATCH 2/5] update deps --- mix.exs | 2 +- mix.lock | 30 +++++++++++++++--------------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/mix.exs b/mix.exs index e6cbd69..bbab609 100644 --- a/mix.exs +++ b/mix.exs @@ -2,7 +2,7 @@ defmodule RemotePersistentTerm.MixProject do use Mix.Project @name "RemotePersistentTerm" - @version "0.12.0" + @version "0.13.0" @repo_url "https://github.com/AppMonet/remote_persistent_term" def project do diff --git a/mix.lock b/mix.lock index 0a9b629..6dd48b3 100644 --- a/mix.lock +++ b/mix.lock @@ -1,40 +1,40 @@ %{ "bypass": {:hex, :bypass, "2.1.0", "909782781bf8e20ee86a9cabde36b259d44af8b9f38756173e8f5e2e1fabb9b1", [:mix], [{:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.0", [hex: :plug_cowboy, repo: "hexpm", optional: false]}, {:ranch, "~> 1.3", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "d9b5df8fa5b7a6efa08384e9bbecfe4ce61c77d28a4282f79e02f1ef78d96b80"}, "castore": {:hex, :castore, "1.0.5", "9eeebb394cc9a0f3ae56b813459f990abb0a3dedee1be6b27fdb50301930502f", [:mix], [], "hexpm", "8d7c597c3e4a64c395980882d4bca3cebb8d74197c590dc272cfd3b6a6310578"}, - "certifi": {:hex, :certifi, "2.14.0", "ed3bef654e69cde5e6c022df8070a579a79e8ba2368a00acf3d75b82d9aceeed", [:rebar3], [], "hexpm", "ea59d87ef89da429b8e905264fdec3419f84f2215bb3d81e07a18aac919026c3"}, + "certifi": {:hex, :certifi, "2.15.0", "0e6e882fcdaaa0a5a9f2b3db55b1394dba07e8d6d9bcad08318fb604c6839712", [:rebar3], [], "hexpm", "b147ed22ce71d72eafdad94f055165c1c182f61a2ff49df28bcc71d1d5b94a60"}, "configparser_ex": {:hex, :configparser_ex, "4.0.0", "17e2b831cfa33a08c56effc610339b2986f0d82a9caa0ed18880a07658292ab6", [:mix], [], "hexpm", "02e6d1a559361a063cba7b75bc3eb2d6ad7e62730c551cc4703541fd11e65e5b"}, - "cowboy": {:hex, :cowboy, "2.13.0", "09d770dd5f6a22cc60c071f432cd7cb87776164527f205c5a6b0f24ff6b38990", [:make, :rebar3], [{:cowlib, ">= 2.14.0 and < 3.0.0", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, ">= 1.8.0 and < 3.0.0", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "e724d3a70995025d654c1992c7b11dbfea95205c047d86ff9bf1cda92ddc5614"}, + "cowboy": {:hex, :cowboy, "2.14.2", "4008be1df6ade45e4f2a4e9e2d22b36d0b5aba4e20b0a0d7049e28d124e34847", [:make, :rebar3], [{:cowlib, ">= 2.16.0 and < 3.0.0", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, ">= 1.8.0 and < 3.0.0", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "569081da046e7b41b5df36aa359be71a0c8874e5b9cff6f747073fc57baf1ab9"}, "cowboy_telemetry": {:hex, :cowboy_telemetry, "0.4.0", "f239f68b588efa7707abce16a84d0d2acf3a0f50571f8bb7f56a15865aae820c", [:rebar3], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "7d98bac1ee4565d31b62d59f8823dfd8356a169e7fcbb83831b8a5397404c9de"}, - "cowlib": {:hex, :cowlib, "2.14.0", "623791c56c1cc9df54a71a9c55147a401549917f00a2e48a6ae12b812c586ced", [:make, :rebar3], [], "hexpm", "0af652d1550c8411c3b58eed7a035a7fb088c0b86aff6bc504b0bc3b7f791aa2"}, + "cowlib": {:hex, :cowlib, "2.16.0", "54592074ebbbb92ee4746c8a8846e5605052f29309d3a873468d76cdf932076f", [:make, :rebar3], [], "hexpm", "7f478d80d66b747344f0ea7708c187645cfcc08b11aa424632f78e25bf05db51"}, "earmark_parser": {:hex, :earmark_parser, "1.4.44", "f20830dd6b5c77afe2b063777ddbbff09f9759396500cdbe7523efd58d7a339c", [:mix], [], "hexpm", "4778ac752b4701a5599215f7030989c989ffdc4f6df457c5f36938cc2d2a2750"}, "ex_aws": {:hex, :ex_aws, "2.5.9", "8e2455172f0e5cbe2f56dd68de514f0dae6bb26d6b6e2f435a06434cf9dbb412", [:mix], [{:configparser_ex, "~> 4.0", [hex: :configparser_ex, repo: "hexpm", optional: true]}, {:hackney, "~> 1.16", [hex: :hackney, repo: "hexpm", optional: true]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: true]}, {:jsx, "~> 2.8 or ~> 3.0", [hex: :jsx, repo: "hexpm", optional: true]}, {:mime, "~> 1.2 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:req, "~> 0.5.10 or ~> 0.6 or ~> 1.0", [hex: :req, repo: "hexpm", optional: true]}, {:sweet_xml, "~> 0.7", [hex: :sweet_xml, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "cbdb6ffb0e6c6368de05ed8641fe1376298ba23354674428e5b153a541f23359"}, - "ex_aws_s3": {:hex, :ex_aws_s3, "2.5.7", "e571424d2f345299753382f3a01b005c422b1a460a8bc3ed47659b3d3ef91e9e", [:mix], [{:ex_aws, "~> 2.0", [hex: :ex_aws, repo: "hexpm", optional: false]}, {:sweet_xml, ">= 0.0.0", [hex: :sweet_xml, repo: "hexpm", optional: true]}], "hexpm", "858e51241e50181e29aa2bc128fef548873a3a9cd580471f57eda5b64dec937f"}, - "ex_doc": {:hex, :ex_doc, "0.37.3", "f7816881a443cd77872b7d6118e8a55f547f49903aef8747dbcb345a75b462f9", [:mix], [{:earmark_parser, "~> 1.4.42", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_c, ">= 0.1.0", [hex: :makeup_c, repo: "hexpm", optional: true]}, {:makeup_elixir, "~> 0.14 or ~> 1.0", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1 or ~> 1.0", [hex: :makeup_erlang, repo: "hexpm", optional: false]}, {:makeup_html, ">= 0.1.0", [hex: :makeup_html, repo: "hexpm", optional: true]}], "hexpm", "e6aebca7156e7c29b5da4daa17f6361205b2ae5f26e5c7d8ca0d3f7e18972233"}, - "finch": {:hex, :finch, "0.19.0", "c644641491ea854fc5c1bbaef36bfc764e3f08e7185e1f084e35e0672241b76d", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.6.2 or ~> 1.7", [hex: :mint, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.4 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:nimble_pool, "~> 1.1", [hex: :nimble_pool, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "fc5324ce209125d1e2fa0fcd2634601c52a787aff1cd33ee833664a5af4ea2b6"}, - "hackney": {:hex, :hackney, "1.23.0", "55cc09077112bcb4a69e54be46ed9bc55537763a96cd4a80a221663a7eafd767", [:rebar3], [{:certifi, "~> 2.14.0", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "~> 6.1.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "~> 1.0.0", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~> 1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "3.4.1", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~> 1.1.0", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}, {:unicode_util_compat, "~> 0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "6cd1c04cd15c81e5a493f167b226a15f0938a84fc8f0736ebe4ddcab65c0b44e"}, + "ex_aws_s3": {:hex, :ex_aws_s3, "2.5.9", "862b7792f2e60d7010e2920d79964e3fab289bc0fd951b0ba8457a3f7f9d1199", [:mix], [{:ex_aws, "~> 2.0", [hex: :ex_aws, repo: "hexpm", optional: false]}, {:sweet_xml, ">= 0.0.0", [hex: :sweet_xml, repo: "hexpm", optional: true]}], "hexpm", "a480d2bb2da64610014021629800e1e9457ca5e4a62f6775bffd963360c2bf90"}, + "ex_doc": {:hex, :ex_doc, "0.39.3", "519c6bc7e84a2918b737aec7ef48b96aa4698342927d080437f61395d361dcee", [:mix], [{:earmark_parser, "~> 1.4.44", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_c, ">= 0.1.0", [hex: :makeup_c, repo: "hexpm", optional: true]}, {:makeup_elixir, "~> 0.14 or ~> 1.0", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1 or ~> 1.0", [hex: :makeup_erlang, repo: "hexpm", optional: false]}, {:makeup_html, ">= 0.1.0", [hex: :makeup_html, repo: "hexpm", optional: true]}], "hexpm", "0590955cf7ad3b625780ee1c1ea627c28a78948c6c0a9b0322bd976a079996e1"}, + "finch": {:hex, :finch, "0.20.0", "5330aefb6b010f424dcbbc4615d914e9e3deae40095e73ab0c1bb0968933cadf", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.6.2 or ~> 1.7", [hex: :mint, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.4 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:nimble_pool, "~> 1.1", [hex: :nimble_pool, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "2658131a74d051aabfcba936093c903b8e89da9a1b63e430bee62045fa9b2ee2"}, + "hackney": {:hex, :hackney, "1.25.0", "390e9b83f31e5b325b9f43b76e1a785cbdb69b5b6cd4e079aa67835ded046867", [:rebar3], [{:certifi, "~> 2.15.0", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "~> 6.1.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "~> 1.0.0", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~> 1.4", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "3.4.1", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~> 1.1.0", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}, {:unicode_util_compat, "~> 0.7.1", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "7209bfd75fd1f42467211ff8f59ea74d6f2a9e81cbcee95a56711ee79fd6b1d4"}, "hpax": {:hex, :hpax, "1.0.3", "ed67ef51ad4df91e75cc6a1494f851850c0bd98ebc0be6e81b026e765ee535aa", [:mix], [], "hexpm", "8eab6e1cfa8d5918c2ce4ba43588e894af35dbd8e91e6e55c817bca5847df34a"}, "idna": {:hex, :idna, "6.1.1", "8a63070e9f7d0c62eb9d9fcb360a7de382448200fbbd1b106cc96d3d8099df8d", [:rebar3], [{:unicode_util_compat, "~> 0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "92376eb7894412ed19ac475e4a86f7b413c1b9fbb5bd16dccd57934157944cea"}, "jason": {:hex, :jason, "1.4.4", "b9226785a9aa77b6857ca22832cffa5d5011a667207eb2a0ad56adb5db443b8a", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "c5eb0cab91f094599f94d55bc63409236a8ec69a21a67814529e8d5f6cc90b3b"}, "makeup": {:hex, :makeup, "1.2.1", "e90ac1c65589ef354378def3ba19d401e739ee7ee06fb47f94c687016e3713d1", [:mix], [{:nimble_parsec, "~> 1.4", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "d36484867b0bae0fea568d10131197a4c2e47056a6fbe84922bf6ba71c8d17ce"}, "makeup_elixir": {:hex, :makeup_elixir, "1.0.1", "e928a4f984e795e41e3abd27bfc09f51db16ab8ba1aebdba2b3a575437efafc2", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "7284900d412a3e5cfd97fdaed4f5ed389b8f2b4cb49efc0eb3bd10e2febf9507"}, - "makeup_erlang": {:hex, :makeup_erlang, "1.0.2", "03e1804074b3aa64d5fad7aa64601ed0fb395337b982d9bcf04029d68d51b6a7", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "af33ff7ef368d5893e4a267933e7744e46ce3cf1f61e2dccf53a111ed3aa3727"}, + "makeup_erlang": {:hex, :makeup_erlang, "1.0.3", "4252d5d4098da7415c390e847c814bad3764c94a814a0b4245176215615e1035", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "953297c02582a33411ac6208f2c6e55f0e870df7f80da724ed613f10e6706afd"}, "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm", "69b09adddc4f74a40716ae54d140f93beb0fb8978d8636eaded0c31b6f099f16"}, - "mime": {:hex, :mime, "2.0.6", "8f18486773d9b15f95f4f4f1e39b710045fa1de891fada4516559967276e4dc2", [:mix], [], "hexpm", "c9945363a6b26d747389aac3643f8e0e09d30499a138ad64fe8fd1d13d9b153e"}, - "mimerl": {:hex, :mimerl, "1.3.0", "d0cd9fc04b9061f82490f6581e0128379830e78535e017f7780f37fea7545726", [:rebar3], [], "hexpm", "a1e15a50d1887217de95f0b9b0793e32853f7c258a5cd227650889b38839fe9d"}, + "mime": {:hex, :mime, "2.0.7", "b8d739037be7cd402aee1ba0306edfdef982687ee7e9859bee6198c1e7e2f128", [:mix], [], "hexpm", "6171188e399ee16023ffc5b76ce445eb6d9672e2e241d2df6050f3c771e80ccd"}, + "mimerl": {:hex, :mimerl, "1.4.0", "3882a5ca67fbbe7117ba8947f27643557adec38fa2307490c4c4207624cb213b", [:rebar3], [], "hexpm", "13af15f9f68c65884ecca3a3891d50a7b57d82152792f3e19d88650aa126b144"}, "mint": {:hex, :mint, "1.7.1", "113fdb2b2f3b59e47c7955971854641c61f378549d73e829e1768de90fc1abf1", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:hpax, "~> 0.1.1 or ~> 0.2.0 or ~> 1.0", [hex: :hpax, repo: "hexpm", optional: false]}], "hexpm", "fceba0a4d0f24301ddee3024ae116df1c3f4bb7a563a731f45fdfeb9d39a231b"}, "mox": {:hex, :mox, "1.2.0", "a2cd96b4b80a3883e3100a221e8adc1b98e4c3a332a8fc434c39526babafd5b3", [:mix], [{:nimble_ownership, "~> 1.0", [hex: :nimble_ownership, repo: "hexpm", optional: false]}], "hexpm", "c7b92b3cc69ee24a7eeeaf944cd7be22013c52fcb580c1f33f50845ec821089a"}, "nimble_options": {:hex, :nimble_options, "1.1.1", "e3a492d54d85fc3fd7c5baf411d9d2852922f66e69476317787a7b2bb000a61b", [:mix], [], "hexpm", "821b2470ca9442c4b6984882fe9bb0389371b8ddec4d45a9504f00a66f650b44"}, - "nimble_ownership": {:hex, :nimble_ownership, "1.0.1", "f69fae0cdd451b1614364013544e66e4f5d25f36a2056a9698b793305c5aa3a6", [:mix], [], "hexpm", "3825e461025464f519f3f3e4a1f9b68c47dc151369611629ad08b636b73bb22d"}, + "nimble_ownership": {:hex, :nimble_ownership, "1.0.2", "fa8a6f2d8c592ad4d79b2ca617473c6aefd5869abfa02563a77682038bf916cf", [:mix], [], "hexpm", "098af64e1f6f8609c6672127cfe9e9590a5d3fcdd82bc17a377b8692fd81a879"}, "nimble_parsec": {:hex, :nimble_parsec, "1.4.2", "8efba0122db06df95bfaa78f791344a89352ba04baedd3849593bfce4d0dc1c6", [:mix], [], "hexpm", "4b21398942dda052b403bbe1da991ccd03a053668d147d53fb8c4e0efe09c973"}, "nimble_pool": {:hex, :nimble_pool, "1.1.0", "bf9c29fbdcba3564a8b800d1eeb5a3c58f36e1e11d7b7fb2e084a643f645f06b", [:mix], [], "hexpm", "af2e4e6b34197db81f7aad230c1118eac993acc0dae6bc83bac0126d4ae0813a"}, "parse_trans": {:hex, :parse_trans, "3.4.1", "6e6aa8167cb44cc8f39441d05193be6e6f4e7c2946cb2759f015f8c56b76e5ff", [:rebar3], [], "hexpm", "620a406ce75dada827b82e453c19cf06776be266f5a67cff34e1ef2cbb60e49a"}, - "plug": {:hex, :plug, "1.17.0", "a0832e7af4ae0f4819e0c08dd2e7482364937aea6a8a997a679f2cbb7e026b2e", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "f6692046652a69a00a5a21d0b7e11fcf401064839d59d6b8787f23af55b1e6bc"}, - "plug_cowboy": {:hex, :plug_cowboy, "2.7.3", "1304d36752e8bdde213cea59ef424ca932910a91a07ef9f3874be709c4ddb94b", [:mix], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:cowboy_telemetry, "~> 0.3", [hex: :cowboy_telemetry, repo: "hexpm", optional: false]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "77c95524b2aa5364b247fa17089029e73b951ebc1adeef429361eab0bb55819d"}, + "plug": {:hex, :plug, "1.19.1", "09bac17ae7a001a68ae393658aa23c7e38782be5c5c00c80be82901262c394c0", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "560a0017a8f6d5d30146916862aaf9300b7280063651dd7e532b8be168511e62"}, + "plug_cowboy": {:hex, :plug_cowboy, "2.7.5", "261f21b67aea8162239b2d6d3b4c31efde4daa22a20d80b19c2c0f21b34b270e", [:mix], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:cowboy_telemetry, "~> 0.3", [hex: :cowboy_telemetry, repo: "hexpm", optional: false]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "20884bf58a90ff5a5663420f5d2c368e9e15ed1ad5e911daf0916ea3c57f77ac"}, "plug_crypto": {:hex, :plug_crypto, "2.1.1", "19bda8184399cb24afa10be734f84a16ea0a2bc65054e23a62bb10f06bc89491", [:mix], [], "hexpm", "6470bce6ffe41c8bd497612ffde1a7e4af67f36a15eea5f921af71cf3e11247c"}, "ranch": {:hex, :ranch, "1.8.1", "208169e65292ac5d333d6cdbad49388c1ae198136e4697ae2f474697140f201c", [:make, :rebar3], [], "hexpm", "aed58910f4e21deea992a67bf51632b6d60114895eb03bb392bb733064594dd0"}, - "req": {:hex, :req, "0.5.10", "a3a063eab8b7510785a467f03d30a8d95f66f5c3d9495be3474b61459c54376c", [:mix], [{:brotli, "~> 0.3.1", [hex: :brotli, repo: "hexpm", optional: true]}, {:ezstd, "~> 1.0", [hex: :ezstd, repo: "hexpm", optional: true]}, {:finch, "~> 0.17", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mime, "~> 2.0.6 or ~> 2.1", [hex: :mime, repo: "hexpm", optional: false]}, {:nimble_csv, "~> 1.0", [hex: :nimble_csv, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "8a604815743f8a2d3b5de0659fa3137fa4b1cffd636ecb69b30b2b9b2c2559be"}, + "req": {:hex, :req, "0.5.17", "0096ddd5b0ed6f576a03dde4b158a0c727215b15d2795e59e0916c6971066ede", [:mix], [{:brotli, "~> 0.3.1", [hex: :brotli, repo: "hexpm", optional: true]}, {:ezstd, "~> 1.0", [hex: :ezstd, repo: "hexpm", optional: true]}, {:finch, "~> 0.17", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mime, "~> 2.0.6 or ~> 2.1", [hex: :mime, repo: "hexpm", optional: false]}, {:nimble_csv, "~> 1.0", [hex: :nimble_csv, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "0b8bc6ffdfebbc07968e59d3ff96d52f2202d0536f10fef4dc11dc02a2a43e39"}, "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.7", "354c321cf377240c7b8716899e182ce4890c5938111a1296add3ec74cf1715df", [:make, :mix, :rebar3], [], "hexpm", "fe4c190e8f37401d30167c8c405eda19469f34577987c76dde613e838bbc67f8"}, "sweet_xml": {:hex, :sweet_xml, "0.7.5", "803a563113981aaac202a1dbd39771562d0ad31004ddbfc9b5090bdcd5605277", [:mix], [], "hexpm", "193b28a9b12891cae351d81a0cead165ffe67df1b73fe5866d10629f4faefb12"}, "telemetry": {:hex, :telemetry, "1.3.0", "fedebbae410d715cf8e7062c96a1ef32ec22e764197f70cda73d82778d61e7a2", [:rebar3], [], "hexpm", "7015fc8919dbe63764f4b4b87a95b7c0996bd539e0d499be6ec9d7f3875b79e6"}, - "unicode_util_compat": {:hex, :unicode_util_compat, "0.7.0", "bc84380c9ab48177092f43ac89e4dfa2c6d62b40b8bd132b1059ecc7232f9a78", [:rebar3], [], "hexpm", "25eee6d67df61960cf6a794239566599b09e17e668d3700247bc498638152521"}, + "unicode_util_compat": {:hex, :unicode_util_compat, "0.7.1", "a48703a25c170eedadca83b11e88985af08d35f37c6f664d6dcfb106a97782fc", [:rebar3], [], "hexpm", "b3a917854ce3ae233619744ad1e0102e05673136776fb2fa76234f3e03b23642"}, } From dba0bae63b44e782e0fc7e844c07e7dba4fb6a06 Mon Sep 17 00:00:00 2001 From: nico piderman Date: Tue, 13 Jan 2026 16:09:58 +0100 Subject: [PATCH 3/5] clean up finding headers --- lib/remote_persistent_term/fetcher/s3.ex | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/lib/remote_persistent_term/fetcher/s3.ex b/lib/remote_persistent_term/fetcher/s3.ex index 4479355..1cd7927 100644 --- a/lib/remote_persistent_term/fetcher/s3.ex +++ b/lib/remote_persistent_term/fetcher/s3.ex @@ -207,14 +207,9 @@ defmodule RemotePersistentTerm.Fetcher.S3 do end defp header_value(headers, name) do - downcased = String.downcase(name) - Enum.find_value(headers, fn {key, value} when is_binary(key) and is_binary(value) -> - if String.downcase(key) == downcased, do: value, else: nil - - {key, value} when is_atom(key) and is_binary(value) -> - if String.downcase(Atom.to_string(key)) == downcased, do: value, else: nil + if key == name, do: value, else: nil _ -> nil From 405cc877fe605fd2dc7ed21c606619997f2b50a5 Mon Sep 17 00:00:00 2001 From: nico piderman Date: Wed, 14 Jan 2026 07:41:20 +0100 Subject: [PATCH 4/5] remove hackney --- lib/remote_persistent_term/fetcher/s3.ex | 12 +++++++++ .../fetcher/s3/http_client.ex | 26 +++++++++++++++++++ mix.exs | 1 - mix.lock | 9 ------- 4 files changed, 38 insertions(+), 10 deletions(-) create mode 100644 lib/remote_persistent_term/fetcher/s3/http_client.ex diff --git a/lib/remote_persistent_term/fetcher/s3.ex b/lib/remote_persistent_term/fetcher/s3.ex index 1cd7927..130d8a5 100644 --- a/lib/remote_persistent_term/fetcher/s3.ex +++ b/lib/remote_persistent_term/fetcher/s3.ex @@ -89,6 +89,8 @@ defmodule RemotePersistentTerm.Fetcher.S3 do """ @impl true def init(opts) do + ensure_http_client() + with {:ok, valid_opts} <- NimbleOptions.validate(opts, @opts_schema) do {:ok, %__MODULE__{ @@ -238,6 +240,16 @@ defmodule RemotePersistentTerm.Fetcher.S3 do defp failover_on_error?({:http_error, 304, _}), do: false defp failover_on_error?(_reason), do: true + defp ensure_http_client do + case Application.get_env(:ex_aws, :http_client) do + nil -> + Application.put_env(:ex_aws, :http_client, RemotePersistentTerm.Fetcher.S3.HttpClient) + + _ -> + :ok + end + end + defp aws_client_request(op, state, opts) do aws_client_request(op, state, opts, fn _ -> true end) end diff --git a/lib/remote_persistent_term/fetcher/s3/http_client.ex b/lib/remote_persistent_term/fetcher/s3/http_client.ex new file mode 100644 index 0000000..73f4afe --- /dev/null +++ b/lib/remote_persistent_term/fetcher/s3/http_client.ex @@ -0,0 +1,26 @@ +defmodule RemotePersistentTerm.Fetcher.S3.HttpClient do + @moduledoc """ + ExAws HTTP client implementation for Req. + """ + + @behaviour ExAws.Request.HttpClient + + @impl ExAws.Request.HttpClient + def request(method, url, body, headers, _http_opts) do + request = Req.new(decode_body: false, retry: false) + + case Req.request(request, method: method, url: url, body: body, headers: headers) do + {:ok, response} -> + response = %{ + status_code: response.status, + headers: Req.get_headers_list(response), + body: response.body + } + + {:ok, response} + + {:error, reason} -> + {:error, %{reason: reason}} + end + end +end diff --git a/mix.exs b/mix.exs index bbab609..ac5a66d 100644 --- a/mix.exs +++ b/mix.exs @@ -34,7 +34,6 @@ defmodule RemotePersistentTerm.MixProject do {:ex_aws, "~> 2.1"}, {:ex_aws_s3, "~> 2.5.7"}, {:configparser_ex, "~> 4.0", optional: true}, - {:hackney, "~> 1.9"}, {:sweet_xml, "~> 0.6"}, {:mox, "~> 1.0", only: :test}, {:req, "~> 0.4"}, diff --git a/mix.lock b/mix.lock index 6dd48b3..cbfa540 100644 --- a/mix.lock +++ b/mix.lock @@ -1,7 +1,5 @@ %{ "bypass": {:hex, :bypass, "2.1.0", "909782781bf8e20ee86a9cabde36b259d44af8b9f38756173e8f5e2e1fabb9b1", [:mix], [{:plug, "~> 1.7", [hex: :plug, repo: "hexpm", optional: false]}, {:plug_cowboy, "~> 2.0", [hex: :plug_cowboy, repo: "hexpm", optional: false]}, {:ranch, "~> 1.3", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "d9b5df8fa5b7a6efa08384e9bbecfe4ce61c77d28a4282f79e02f1ef78d96b80"}, - "castore": {:hex, :castore, "1.0.5", "9eeebb394cc9a0f3ae56b813459f990abb0a3dedee1be6b27fdb50301930502f", [:mix], [], "hexpm", "8d7c597c3e4a64c395980882d4bca3cebb8d74197c590dc272cfd3b6a6310578"}, - "certifi": {:hex, :certifi, "2.15.0", "0e6e882fcdaaa0a5a9f2b3db55b1394dba07e8d6d9bcad08318fb604c6839712", [:rebar3], [], "hexpm", "b147ed22ce71d72eafdad94f055165c1c182f61a2ff49df28bcc71d1d5b94a60"}, "configparser_ex": {:hex, :configparser_ex, "4.0.0", "17e2b831cfa33a08c56effc610339b2986f0d82a9caa0ed18880a07658292ab6", [:mix], [], "hexpm", "02e6d1a559361a063cba7b75bc3eb2d6ad7e62730c551cc4703541fd11e65e5b"}, "cowboy": {:hex, :cowboy, "2.14.2", "4008be1df6ade45e4f2a4e9e2d22b36d0b5aba4e20b0a0d7049e28d124e34847", [:make, :rebar3], [{:cowlib, ">= 2.16.0 and < 3.0.0", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, ">= 1.8.0 and < 3.0.0", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "569081da046e7b41b5df36aa359be71a0c8874e5b9cff6f747073fc57baf1ab9"}, "cowboy_telemetry": {:hex, :cowboy_telemetry, "0.4.0", "f239f68b588efa7707abce16a84d0d2acf3a0f50571f8bb7f56a15865aae820c", [:rebar3], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "7d98bac1ee4565d31b62d59f8823dfd8356a169e7fcbb83831b8a5397404c9de"}, @@ -11,30 +9,23 @@ "ex_aws_s3": {:hex, :ex_aws_s3, "2.5.9", "862b7792f2e60d7010e2920d79964e3fab289bc0fd951b0ba8457a3f7f9d1199", [:mix], [{:ex_aws, "~> 2.0", [hex: :ex_aws, repo: "hexpm", optional: false]}, {:sweet_xml, ">= 0.0.0", [hex: :sweet_xml, repo: "hexpm", optional: true]}], "hexpm", "a480d2bb2da64610014021629800e1e9457ca5e4a62f6775bffd963360c2bf90"}, "ex_doc": {:hex, :ex_doc, "0.39.3", "519c6bc7e84a2918b737aec7ef48b96aa4698342927d080437f61395d361dcee", [:mix], [{:earmark_parser, "~> 1.4.44", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_c, ">= 0.1.0", [hex: :makeup_c, repo: "hexpm", optional: true]}, {:makeup_elixir, "~> 0.14 or ~> 1.0", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1 or ~> 1.0", [hex: :makeup_erlang, repo: "hexpm", optional: false]}, {:makeup_html, ">= 0.1.0", [hex: :makeup_html, repo: "hexpm", optional: true]}], "hexpm", "0590955cf7ad3b625780ee1c1ea627c28a78948c6c0a9b0322bd976a079996e1"}, "finch": {:hex, :finch, "0.20.0", "5330aefb6b010f424dcbbc4615d914e9e3deae40095e73ab0c1bb0968933cadf", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.6.2 or ~> 1.7", [hex: :mint, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.4 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:nimble_pool, "~> 1.1", [hex: :nimble_pool, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "2658131a74d051aabfcba936093c903b8e89da9a1b63e430bee62045fa9b2ee2"}, - "hackney": {:hex, :hackney, "1.25.0", "390e9b83f31e5b325b9f43b76e1a785cbdb69b5b6cd4e079aa67835ded046867", [:rebar3], [{:certifi, "~> 2.15.0", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "~> 6.1.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "~> 1.0.0", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~> 1.4", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "3.4.1", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~> 1.1.0", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}, {:unicode_util_compat, "~> 0.7.1", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "7209bfd75fd1f42467211ff8f59ea74d6f2a9e81cbcee95a56711ee79fd6b1d4"}, "hpax": {:hex, :hpax, "1.0.3", "ed67ef51ad4df91e75cc6a1494f851850c0bd98ebc0be6e81b026e765ee535aa", [:mix], [], "hexpm", "8eab6e1cfa8d5918c2ce4ba43588e894af35dbd8e91e6e55c817bca5847df34a"}, - "idna": {:hex, :idna, "6.1.1", "8a63070e9f7d0c62eb9d9fcb360a7de382448200fbbd1b106cc96d3d8099df8d", [:rebar3], [{:unicode_util_compat, "~> 0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "92376eb7894412ed19ac475e4a86f7b413c1b9fbb5bd16dccd57934157944cea"}, "jason": {:hex, :jason, "1.4.4", "b9226785a9aa77b6857ca22832cffa5d5011a667207eb2a0ad56adb5db443b8a", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "c5eb0cab91f094599f94d55bc63409236a8ec69a21a67814529e8d5f6cc90b3b"}, "makeup": {:hex, :makeup, "1.2.1", "e90ac1c65589ef354378def3ba19d401e739ee7ee06fb47f94c687016e3713d1", [:mix], [{:nimble_parsec, "~> 1.4", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "d36484867b0bae0fea568d10131197a4c2e47056a6fbe84922bf6ba71c8d17ce"}, "makeup_elixir": {:hex, :makeup_elixir, "1.0.1", "e928a4f984e795e41e3abd27bfc09f51db16ab8ba1aebdba2b3a575437efafc2", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.2.3 or ~> 1.3", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "7284900d412a3e5cfd97fdaed4f5ed389b8f2b4cb49efc0eb3bd10e2febf9507"}, "makeup_erlang": {:hex, :makeup_erlang, "1.0.3", "4252d5d4098da7415c390e847c814bad3764c94a814a0b4245176215615e1035", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "953297c02582a33411ac6208f2c6e55f0e870df7f80da724ed613f10e6706afd"}, - "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm", "69b09adddc4f74a40716ae54d140f93beb0fb8978d8636eaded0c31b6f099f16"}, "mime": {:hex, :mime, "2.0.7", "b8d739037be7cd402aee1ba0306edfdef982687ee7e9859bee6198c1e7e2f128", [:mix], [], "hexpm", "6171188e399ee16023ffc5b76ce445eb6d9672e2e241d2df6050f3c771e80ccd"}, - "mimerl": {:hex, :mimerl, "1.4.0", "3882a5ca67fbbe7117ba8947f27643557adec38fa2307490c4c4207624cb213b", [:rebar3], [], "hexpm", "13af15f9f68c65884ecca3a3891d50a7b57d82152792f3e19d88650aa126b144"}, "mint": {:hex, :mint, "1.7.1", "113fdb2b2f3b59e47c7955971854641c61f378549d73e829e1768de90fc1abf1", [:mix], [{:castore, "~> 0.1.0 or ~> 1.0", [hex: :castore, repo: "hexpm", optional: true]}, {:hpax, "~> 0.1.1 or ~> 0.2.0 or ~> 1.0", [hex: :hpax, repo: "hexpm", optional: false]}], "hexpm", "fceba0a4d0f24301ddee3024ae116df1c3f4bb7a563a731f45fdfeb9d39a231b"}, "mox": {:hex, :mox, "1.2.0", "a2cd96b4b80a3883e3100a221e8adc1b98e4c3a332a8fc434c39526babafd5b3", [:mix], [{:nimble_ownership, "~> 1.0", [hex: :nimble_ownership, repo: "hexpm", optional: false]}], "hexpm", "c7b92b3cc69ee24a7eeeaf944cd7be22013c52fcb580c1f33f50845ec821089a"}, "nimble_options": {:hex, :nimble_options, "1.1.1", "e3a492d54d85fc3fd7c5baf411d9d2852922f66e69476317787a7b2bb000a61b", [:mix], [], "hexpm", "821b2470ca9442c4b6984882fe9bb0389371b8ddec4d45a9504f00a66f650b44"}, "nimble_ownership": {:hex, :nimble_ownership, "1.0.2", "fa8a6f2d8c592ad4d79b2ca617473c6aefd5869abfa02563a77682038bf916cf", [:mix], [], "hexpm", "098af64e1f6f8609c6672127cfe9e9590a5d3fcdd82bc17a377b8692fd81a879"}, "nimble_parsec": {:hex, :nimble_parsec, "1.4.2", "8efba0122db06df95bfaa78f791344a89352ba04baedd3849593bfce4d0dc1c6", [:mix], [], "hexpm", "4b21398942dda052b403bbe1da991ccd03a053668d147d53fb8c4e0efe09c973"}, "nimble_pool": {:hex, :nimble_pool, "1.1.0", "bf9c29fbdcba3564a8b800d1eeb5a3c58f36e1e11d7b7fb2e084a643f645f06b", [:mix], [], "hexpm", "af2e4e6b34197db81f7aad230c1118eac993acc0dae6bc83bac0126d4ae0813a"}, - "parse_trans": {:hex, :parse_trans, "3.4.1", "6e6aa8167cb44cc8f39441d05193be6e6f4e7c2946cb2759f015f8c56b76e5ff", [:rebar3], [], "hexpm", "620a406ce75dada827b82e453c19cf06776be266f5a67cff34e1ef2cbb60e49a"}, "plug": {:hex, :plug, "1.19.1", "09bac17ae7a001a68ae393658aa23c7e38782be5c5c00c80be82901262c394c0", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "560a0017a8f6d5d30146916862aaf9300b7280063651dd7e532b8be168511e62"}, "plug_cowboy": {:hex, :plug_cowboy, "2.7.5", "261f21b67aea8162239b2d6d3b4c31efde4daa22a20d80b19c2c0f21b34b270e", [:mix], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:cowboy_telemetry, "~> 0.3", [hex: :cowboy_telemetry, repo: "hexpm", optional: false]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "20884bf58a90ff5a5663420f5d2c368e9e15ed1ad5e911daf0916ea3c57f77ac"}, "plug_crypto": {:hex, :plug_crypto, "2.1.1", "19bda8184399cb24afa10be734f84a16ea0a2bc65054e23a62bb10f06bc89491", [:mix], [], "hexpm", "6470bce6ffe41c8bd497612ffde1a7e4af67f36a15eea5f921af71cf3e11247c"}, "ranch": {:hex, :ranch, "1.8.1", "208169e65292ac5d333d6cdbad49388c1ae198136e4697ae2f474697140f201c", [:make, :rebar3], [], "hexpm", "aed58910f4e21deea992a67bf51632b6d60114895eb03bb392bb733064594dd0"}, "req": {:hex, :req, "0.5.17", "0096ddd5b0ed6f576a03dde4b158a0c727215b15d2795e59e0916c6971066ede", [:mix], [{:brotli, "~> 0.3.1", [hex: :brotli, repo: "hexpm", optional: true]}, {:ezstd, "~> 1.0", [hex: :ezstd, repo: "hexpm", optional: true]}, {:finch, "~> 0.17", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mime, "~> 2.0.6 or ~> 2.1", [hex: :mime, repo: "hexpm", optional: false]}, {:nimble_csv, "~> 1.0", [hex: :nimble_csv, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "0b8bc6ffdfebbc07968e59d3ff96d52f2202d0536f10fef4dc11dc02a2a43e39"}, - "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.7", "354c321cf377240c7b8716899e182ce4890c5938111a1296add3ec74cf1715df", [:make, :mix, :rebar3], [], "hexpm", "fe4c190e8f37401d30167c8c405eda19469f34577987c76dde613e838bbc67f8"}, "sweet_xml": {:hex, :sweet_xml, "0.7.5", "803a563113981aaac202a1dbd39771562d0ad31004ddbfc9b5090bdcd5605277", [:mix], [], "hexpm", "193b28a9b12891cae351d81a0cead165ffe67df1b73fe5866d10629f4faefb12"}, "telemetry": {:hex, :telemetry, "1.3.0", "fedebbae410d715cf8e7062c96a1ef32ec22e764197f70cda73d82778d61e7a2", [:rebar3], [], "hexpm", "7015fc8919dbe63764f4b4b87a95b7c0996bd539e0d499be6ec9d7f3875b79e6"}, - "unicode_util_compat": {:hex, :unicode_util_compat, "0.7.1", "a48703a25c170eedadca83b11e88985af08d35f37c6f664d6dcfb106a97782fc", [:rebar3], [], "hexpm", "b3a917854ce3ae233619744ad1e0102e05673136776fb2fa76234f3e03b23642"}, } From 3df35177c0c02bf463644aff239f4053395b1746 Mon Sep 17 00:00:00 2001 From: nico piderman Date: Wed, 14 Jan 2026 07:44:58 +0100 Subject: [PATCH 5/5] make http header lookup case insensitive, just incase consumers use http client that doesnt downcase headers --- lib/remote_persistent_term/fetcher/s3.ex | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/lib/remote_persistent_term/fetcher/s3.ex b/lib/remote_persistent_term/fetcher/s3.ex index 130d8a5..a929365 100644 --- a/lib/remote_persistent_term/fetcher/s3.ex +++ b/lib/remote_persistent_term/fetcher/s3.ex @@ -209,9 +209,11 @@ defmodule RemotePersistentTerm.Fetcher.S3 do end defp header_value(headers, name) do + downcased = String.downcase(name) + Enum.find_value(headers, fn {key, value} when is_binary(key) and is_binary(value) -> - if key == name, do: value, else: nil + if String.downcase(key) == downcased, do: value, else: nil _ -> nil