From bc7d32ffb2b1b38e3376709325141ca303a7feb1 Mon Sep 17 00:00:00 2001 From: Louis Date: Thu, 10 Apr 2025 15:15:35 +0200 Subject: [PATCH 1/6] Add fallback region for rpt s3 --- lib/remote_persistent_term/fetcher/s3.ex | 30 +++-- .../fetcher/s3_test.exs | 103 ++++++++++++++++++ 2 files changed, 126 insertions(+), 7 deletions(-) diff --git a/lib/remote_persistent_term/fetcher/s3.ex b/lib/remote_persistent_term/fetcher/s3.ex index 2f96c96..5a69736 100644 --- a/lib/remote_persistent_term/fetcher/s3.ex +++ b/lib/remote_persistent_term/fetcher/s3.ex @@ -9,9 +9,10 @@ defmodule RemotePersistentTerm.Fetcher.S3 do @type t :: %__MODULE__{ bucket: String.t(), key: String.t(), - region: String.t() + region: String.t(), + failover_region: String.t() | nil } - defstruct [:bucket, :key, :region] + defstruct [:bucket, :key, :region, :failover_region] @opts_schema [ bucket: [ @@ -28,6 +29,11 @@ defmodule RemotePersistentTerm.Fetcher.S3 do type: :string, required: true, doc: "The AWS region of the s3 bucket." + ], + failover_region: [ + type: :string, + required: false, + doc: "The AWS region to use if calls to the default region fail." ] ] @@ -50,7 +56,8 @@ defmodule RemotePersistentTerm.Fetcher.S3 do %__MODULE__{ bucket: valid_opts[:bucket], key: valid_opts[:key], - region: valid_opts[:region] + region: valid_opts[:region], + failover_region: valid_opts[:failover_region] }} end end @@ -94,7 +101,7 @@ defmodule RemotePersistentTerm.Fetcher.S3 do res = state.bucket |> ExAws.S3.get_bucket_object_versions(prefix: state.key) - |> aws_client_request(state.region) + |> aws_client_request(state) with {:ok, %{body: %{versions: versions}}} <- res do {:ok, versions} @@ -104,7 +111,7 @@ defmodule RemotePersistentTerm.Fetcher.S3 do defp get_object(state) do state.bucket |> ExAws.S3.get_object(state.key) - |> aws_client_request(state.region) + |> aws_client_request(state) end defp find_latest([_ | _] = contents) do @@ -123,8 +130,17 @@ defmodule RemotePersistentTerm.Fetcher.S3 do defp find_latest(_), do: {:error, :not_found} - defp aws_client_request(op, region) do - client().request(op, region: region) + defp aws_client_request(op, %{region: region, failover_region: nil}), + do: client().request(op, region: region) + + defp aws_client_request(op, %{region: region, failover_region: failover_region}) do + with {:error, reason} <- client().request(op, region: region) do + Logger.error( + "Failed to fetch from primary region #{region}: #{inspect(reason)}, will try failover region #{failover_region}" + ) + + client().request(op, region: failover_region) + end end defp client, do: Application.get_env(:remote_persistent_term, :aws_client, ExAws) diff --git a/test/remote_persistent_term/fetcher/s3_test.exs b/test/remote_persistent_term/fetcher/s3_test.exs index 6226d5d..f3607bd 100644 --- a/test/remote_persistent_term/fetcher/s3_test.exs +++ b/test/remote_persistent_term/fetcher/s3_test.exs @@ -5,6 +5,12 @@ defmodule RemotePersistentTerm.Fetcher.S3Test do setup :verify_on_exit! import ExUnit.CaptureLog + @bucket "test-bucket" + @key "test-key" + @region "test-region" + @failover_region "failover-region" + @version "F76V.weh4uOlU15f7a2OLHPgCLXkDpm4" + test "Unknown error returns an error for current_version/1" do expect(AwsClientMock, :request, fn _op, _opts -> {:error, :unknown_error} @@ -26,4 +32,101 @@ defmodule RemotePersistentTerm.Fetcher.S3Test do S3.init(bucket: bucket, key: key, region: region) end end + + describe "failover_region" do + test "current_identifiers/1 tries failover region when primary region fails" do + # Setup state with failover region + state = %S3{ + bucket: @bucket, + key: @key, + region: @region, + failover_region: @failover_region + } + + # Mock the AWS client to fail for primary region but succeed for failover region + expect(AwsClientMock, :request, 2, fn _op, opts -> + case opts do + [region: @region] -> + {:error, "Primary region connection error"} + + [region: @failover_region] -> + {:ok, + %{ + body: %{ + versions: [ + %{version_id: @version, etag: "current-etag", is_latest: "true"} + ] + } + }} + end + end) + + log = + capture_log(fn -> + result = S3.current_version(state) + assert {:ok, "current-etag"} = result + end) + + assert log =~ "Failed to fetch from primary region #{@region}" + assert log =~ "will try failover region #{@failover_region}" + end + + test "download/1 tries failover region when primary region fails" do + state = %S3{ + bucket: @bucket, + key: @key, + region: @region, + failover_region: @failover_region + } + + # Mock the AWS client to fail for primary region but succeed for failover region + expect(AwsClientMock, :request, 2, fn _op, opts -> + case opts do + [region: @region] -> + {:error, "Primary region connection error"} + + [region: @failover_region] -> + {:ok, %{body: "content from failover region"}} + end + end) + + log = + capture_log(fn -> + result = S3.download(state) + assert {:ok, "content from failover region"} = result + end) + + assert log =~ "Failed to fetch from primary region #{@region}" + assert log =~ "will try failover region #{@failover_region}" + end + + test "returns error when both primary and failover regions fail" do + state = %S3{ + bucket: @bucket, + key: @key, + region: @region, + failover_region: @failover_region + } + + # Mock the AWS client to fail for both regions + expect(AwsClientMock, :request, 2, fn _op, opts -> + case opts do + [region: @region] -> + {:error, "Primary region connection error"} + + [region: @failover_region] -> + {:error, "Failover region connection error"} + end + end) + + log = + capture_log(fn -> + result = S3.download(state) + assert {:error, _} = result + end) + + assert log =~ "Failed to fetch from primary region #{@region}" + assert log =~ "will try failover region #{@failover_region}" + end + end end From e70b3148cf0d589512312550d39d0b36b71868fa Mon Sep 17 00:00:00 2001 From: Louis Date: Thu, 10 Apr 2025 15:40:03 +0200 Subject: [PATCH 2/6] Make failover_regions a list of regions --- lib/remote_persistent_term/fetcher/s3.ex | 37 +++++--- .../fetcher/s3_test.exs | 86 ++++++++++++++----- 2 files changed, 92 insertions(+), 31 deletions(-) diff --git a/lib/remote_persistent_term/fetcher/s3.ex b/lib/remote_persistent_term/fetcher/s3.ex index 5a69736..7453c90 100644 --- a/lib/remote_persistent_term/fetcher/s3.ex +++ b/lib/remote_persistent_term/fetcher/s3.ex @@ -10,9 +10,9 @@ defmodule RemotePersistentTerm.Fetcher.S3 do bucket: String.t(), key: String.t(), region: String.t(), - failover_region: String.t() | nil + failover_regions: [String.t()] | nil } - defstruct [:bucket, :key, :region, :failover_region] + defstruct [:bucket, :key, :region, :failover_regions] @opts_schema [ bucket: [ @@ -30,10 +30,11 @@ defmodule RemotePersistentTerm.Fetcher.S3 do required: true, doc: "The AWS region of the s3 bucket." ], - failover_region: [ - type: :string, + failover_regions: [ + type: {:list, :string}, required: false, - doc: "The AWS region to use if calls to the default region fail." + doc: + "A list of AWS regions to use if calls to the default region fail. They will be tried in order." ] ] @@ -57,7 +58,7 @@ defmodule RemotePersistentTerm.Fetcher.S3 do bucket: valid_opts[:bucket], key: valid_opts[:key], region: valid_opts[:region], - failover_region: valid_opts[:failover_region] + failover_regions: valid_opts[:failover_regions] }} end end @@ -130,16 +131,32 @@ defmodule RemotePersistentTerm.Fetcher.S3 do defp find_latest(_), do: {:error, :not_found} - defp aws_client_request(op, %{region: region, failover_region: nil}), + defp aws_client_request(op, %{region: region, failover_regions: nil}), do: client().request(op, region: region) - defp aws_client_request(op, %{region: region, failover_region: failover_region}) do + defp aws_client_request(op, %{region: region, failover_regions: failover_regions}) + when is_list(failover_regions) do with {:error, reason} <- client().request(op, region: region) do Logger.error( - "Failed to fetch from primary region #{region}: #{inspect(reason)}, will try failover region #{failover_region}" + "Failed to fetch from primary region #{region}: #{inspect(reason)}, will try failover regions" ) - client().request(op, region: failover_region) + try_failover_regions(op, failover_regions) + end + end + + defp try_failover_regions(_op, []), do: {:error, "All regions failed"} + + defp try_failover_regions(op, [region | remaining_regions]) do + Logger.info("Trying failover region: #{region}") + + case client().request(op, region: region) do + {:ok, result} -> + {:ok, result} + + {:error, reason} -> + Logger.error("Failed to fetch from failover region #{region}: #{inspect(reason)}") + try_failover_regions(op, remaining_regions) end end diff --git a/test/remote_persistent_term/fetcher/s3_test.exs b/test/remote_persistent_term/fetcher/s3_test.exs index f3607bd..4b4c70f 100644 --- a/test/remote_persistent_term/fetcher/s3_test.exs +++ b/test/remote_persistent_term/fetcher/s3_test.exs @@ -8,7 +8,7 @@ defmodule RemotePersistentTerm.Fetcher.S3Test do @bucket "test-bucket" @key "test-key" @region "test-region" - @failover_region "failover-region" + @failover_regions ["failover-region-1", "failover-region-2"] @version "F76V.weh4uOlU15f7a2OLHPgCLXkDpm4" test "Unknown error returns an error for current_version/1" do @@ -33,23 +33,23 @@ defmodule RemotePersistentTerm.Fetcher.S3Test do end end - describe "failover_region" do - test "current_identifiers/1 tries failover region when primary region fails" do - # Setup state with failover region + describe "failover_regions" do + test "current_identifiers/1 tries first failover region when primary region fails" do + # Setup state with failover regions state = %S3{ bucket: @bucket, key: @key, region: @region, - failover_region: @failover_region + failover_regions: @failover_regions } - # Mock the AWS client to fail for primary region but succeed for failover region + # Mock the AWS client to fail for primary region but succeed for first failover region expect(AwsClientMock, :request, 2, fn _op, opts -> case opts do [region: @region] -> {:error, "Primary region connection error"} - [region: @failover_region] -> + [region: "failover-region-1"] -> {:ok, %{ body: %{ @@ -68,24 +68,25 @@ defmodule RemotePersistentTerm.Fetcher.S3Test do end) assert log =~ "Failed to fetch from primary region #{@region}" - assert log =~ "will try failover region #{@failover_region}" + assert log =~ "will try failover regions" + assert log =~ "Trying failover region: failover-region-1" end - test "download/1 tries failover region when primary region fails" do + test "download/1 tries first failover region when primary region fails" do state = %S3{ bucket: @bucket, key: @key, region: @region, - failover_region: @failover_region + failover_regions: @failover_regions } - # Mock the AWS client to fail for primary region but succeed for failover region + # Mock the AWS client to fail for primary region but succeed for first failover region expect(AwsClientMock, :request, 2, fn _op, opts -> case opts do [region: @region] -> {:error, "Primary region connection error"} - [region: @failover_region] -> + [region: "failover-region-1"] -> {:ok, %{body: "content from failover region"}} end end) @@ -97,36 +98,79 @@ defmodule RemotePersistentTerm.Fetcher.S3Test do end) assert log =~ "Failed to fetch from primary region #{@region}" - assert log =~ "will try failover region #{@failover_region}" + assert log =~ "will try failover regions" + assert log =~ "Trying failover region: failover-region-1" end - test "returns error when both primary and failover regions fail" do + test "returns error when primary and all failover regions fail" do state = %S3{ bucket: @bucket, key: @key, region: @region, - failover_region: @failover_region + failover_regions: @failover_regions } - # Mock the AWS client to fail for both regions - expect(AwsClientMock, :request, 2, fn _op, opts -> + # Mock the AWS client to fail for all regions + expect(AwsClientMock, :request, 3, fn _op, opts -> + case opts do + [region: @region] -> + {:error, "Primary region connection error"} + + [region: "failover-region-1"] -> + {:error, "First failover region connection error"} + + [region: "failover-region-2"] -> + {:error, "Second failover region connection error"} + end + end) + + log = + capture_log(fn -> + result = S3.download(state) + assert {:error, message} = result + assert message =~ "All regions failed" + end) + + assert log =~ "Failed to fetch from primary region #{@region}" + assert log =~ "will try failover regions" + assert log =~ "Trying failover region: failover-region-1" + assert log =~ "Failed to fetch from failover region failover-region-1" + assert log =~ "Trying failover region: failover-region-2" + assert log =~ "Failed to fetch from failover region failover-region-2" + end + + test "tries second failover region when first failover region fails" do + state = %S3{ + bucket: @bucket, + key: @key, + region: @region, + failover_regions: @failover_regions + } + + # Mock the AWS client to fail for primary and first failover region but succeed for second failover region + expect(AwsClientMock, :request, 3, fn _op, opts -> case opts do [region: @region] -> {:error, "Primary region connection error"} - [region: @failover_region] -> - {:error, "Failover region connection error"} + [region: "failover-region-1"] -> + {:error, "First failover region connection error"} + + [region: "failover-region-2"] -> + {:ok, %{body: "content from second failover region"}} end end) log = capture_log(fn -> result = S3.download(state) - assert {:error, _} = result + assert {:ok, "content from second failover region"} = result end) assert log =~ "Failed to fetch from primary region #{@region}" - assert log =~ "will try failover region #{@failover_region}" + assert log =~ "Trying failover region: failover-region-1" + assert log =~ "Failed to fetch from failover region failover-region-1" + assert log =~ "Trying failover region: failover-region-2" end end end From 7d09146d0996b31b4163b39d725c0a6f5887f0d9 Mon Sep 17 00:00:00 2001 From: Louis Date: Wed, 23 Apr 2025 15:42:33 +0200 Subject: [PATCH 3/6] Improve logging --- lib/remote_persistent_term/fetcher/s3.ex | 18 +++++------ .../fetcher/s3_test.exs | 30 +++++++++---------- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/lib/remote_persistent_term/fetcher/s3.ex b/lib/remote_persistent_term/fetcher/s3.ex index 7453c90..87bb0b6 100644 --- a/lib/remote_persistent_term/fetcher/s3.ex +++ b/lib/remote_persistent_term/fetcher/s3.ex @@ -80,7 +80,7 @@ defmodule RemotePersistentTerm.Fetcher.S3 do {:error, "could not find s3://#{state.bucket}/#{state.key}"} {:error, reason} -> - Logger.error("#{__MODULE__} - unknown error: #{inspect(reason)}") + Logger.error("#{__MODULE__} - s3://#{state.bucket}/#{state.key} - unknown error: #{inspect(reason)}") {:error, "Unknown error"} end end @@ -134,29 +134,29 @@ defmodule RemotePersistentTerm.Fetcher.S3 do defp aws_client_request(op, %{region: region, failover_regions: nil}), do: client().request(op, region: region) - defp aws_client_request(op, %{region: region, failover_regions: failover_regions}) + defp aws_client_request(op, %{region: region, bucket: bucket, key: key, failover_regions: failover_regions}) when is_list(failover_regions) do with {:error, reason} <- client().request(op, region: region) do Logger.error( - "Failed to fetch from primary region #{region}: #{inspect(reason)}, will try failover regions" + "s3://#{bucket}/#{key} - Failed to fetch from primary region #{region}: #{inspect(reason)}, will try failover regions" ) - try_failover_regions(op, failover_regions) + try_failover_regions(op, failover_regions, bucket, key) end end - defp try_failover_regions(_op, []), do: {:error, "All regions failed"} + defp try_failover_regions(_op, [], _bucket, _key), do: {:error, "All regions failed"} - defp try_failover_regions(op, [region | remaining_regions]) do - Logger.info("Trying failover region: #{region}") + defp try_failover_regions(op, [region | remaining_regions], bucket, key) do + Logger.info("s3://#{bucket}/#{key} - Trying failover region: #{region}") case client().request(op, region: region) do {:ok, result} -> {:ok, result} {:error, reason} -> - Logger.error("Failed to fetch from failover region #{region}: #{inspect(reason)}") - try_failover_regions(op, remaining_regions) + Logger.error("s3://#{bucket}/#{key} - Failed to fetch from failover region #{region}: #{inspect(reason)}") + try_failover_regions(op, remaining_regions, bucket, key) end end diff --git a/test/remote_persistent_term/fetcher/s3_test.exs b/test/remote_persistent_term/fetcher/s3_test.exs index 4b4c70f..1891545 100644 --- a/test/remote_persistent_term/fetcher/s3_test.exs +++ b/test/remote_persistent_term/fetcher/s3_test.exs @@ -17,9 +17,9 @@ defmodule RemotePersistentTerm.Fetcher.S3Test do end) assert capture_log(fn -> - assert {:error, "Unknown error"} = S3.current_version(%S3{bucket: "bucket"}) + assert {:error, "Unknown error"} = S3.current_version(%S3{bucket: "bucket", key: "key"}) end) =~ - "Elixir.RemotePersistentTerm.Fetcher.S3 - unknown error: :unknown_error" + "Elixir.RemotePersistentTerm.Fetcher.S3 - s3://bucket/key - unknown error: :unknown_error" end describe "init/1" do @@ -67,9 +67,9 @@ defmodule RemotePersistentTerm.Fetcher.S3Test do assert {:ok, "current-etag"} = result end) - assert log =~ "Failed to fetch from primary region #{@region}" + assert log =~ "s3://#{@bucket}/#{@key} - Failed to fetch from primary region #{@region}" assert log =~ "will try failover regions" - assert log =~ "Trying failover region: failover-region-1" + assert log =~ "s3://#{@bucket}/#{@key} - Trying failover region: failover-region-1" end test "download/1 tries first failover region when primary region fails" do @@ -97,9 +97,9 @@ defmodule RemotePersistentTerm.Fetcher.S3Test do assert {:ok, "content from failover region"} = result end) - assert log =~ "Failed to fetch from primary region #{@region}" + assert log =~ "s3://#{@bucket}/#{@key} - Failed to fetch from primary region #{@region}" assert log =~ "will try failover regions" - assert log =~ "Trying failover region: failover-region-1" + assert log =~ "s3://#{@bucket}/#{@key} - Trying failover region: failover-region-1" end test "returns error when primary and all failover regions fail" do @@ -131,12 +131,12 @@ defmodule RemotePersistentTerm.Fetcher.S3Test do assert message =~ "All regions failed" end) - assert log =~ "Failed to fetch from primary region #{@region}" + assert log =~ "s3://#{@bucket}/#{@key} - Failed to fetch from primary region #{@region}" assert log =~ "will try failover regions" - assert log =~ "Trying failover region: failover-region-1" - assert log =~ "Failed to fetch from failover region failover-region-1" - assert log =~ "Trying failover region: failover-region-2" - assert log =~ "Failed to fetch from failover region failover-region-2" + assert log =~ "s3://#{@bucket}/#{@key} - Trying failover region: failover-region-1" + assert log =~ "s3://#{@bucket}/#{@key} - Failed to fetch from failover region failover-region-1" + assert log =~ "s3://#{@bucket}/#{@key} - Trying failover region: failover-region-2" + assert log =~ "s3://#{@bucket}/#{@key} - Failed to fetch from failover region failover-region-2" end test "tries second failover region when first failover region fails" do @@ -167,10 +167,10 @@ defmodule RemotePersistentTerm.Fetcher.S3Test do assert {:ok, "content from second failover region"} = result end) - assert log =~ "Failed to fetch from primary region #{@region}" - assert log =~ "Trying failover region: failover-region-1" - assert log =~ "Failed to fetch from failover region failover-region-1" - assert log =~ "Trying failover region: failover-region-2" + assert log =~ "s3://#{@bucket}/#{@key} - Failed to fetch from primary region #{@region}" + assert log =~ "s3://#{@bucket}/#{@key} - Trying failover region: failover-region-1" + assert log =~ "s3://#{@bucket}/#{@key} - Failed to fetch from failover region failover-region-1" + assert log =~ "s3://#{@bucket}/#{@key} - Trying failover region: failover-region-2" end end end From 6559ac0f001a17ffc546392d8610d7452418b0ea Mon Sep 17 00:00:00 2001 From: Louis Date: Wed, 23 Apr 2025 17:12:25 +0200 Subject: [PATCH 4/6] Improve logs --- lib/remote_persistent_term/fetcher/s3.ex | 72 +++++++++++++++---- .../fetcher/s3_test.exs | 67 +++++++++++------ 2 files changed, 104 insertions(+), 35 deletions(-) diff --git a/lib/remote_persistent_term/fetcher/s3.ex b/lib/remote_persistent_term/fetcher/s3.ex index 87bb0b6..a463874 100644 --- a/lib/remote_persistent_term/fetcher/s3.ex +++ b/lib/remote_persistent_term/fetcher/s3.ex @@ -68,7 +68,10 @@ defmodule RemotePersistentTerm.Fetcher.S3 do with {:ok, versions} <- list_object_versions(state), {:ok, %{etag: etag, version_id: version}} <- find_latest(versions) do Logger.info( - "found latest version of s3://#{state.bucket}/#{state.key}: #{etag} with version: #{version}" + bucket: state.bucket, + key: state.key, + version: version, + message: "Found latest version of object" ) {:ok, etag} @@ -80,17 +83,32 @@ defmodule RemotePersistentTerm.Fetcher.S3 do {:error, "could not find s3://#{state.bucket}/#{state.key}"} {:error, reason} -> - Logger.error("#{__MODULE__} - s3://#{state.bucket}/#{state.key} - unknown error: #{inspect(reason)}") + Logger.error(%{ + bucket: state.bucket, + key: state.key, + reason: inspect(reason), + message: "Failed to get current version of object - unknown reason" + }) + {:error, "Unknown error"} end end @impl true def download(state) do - Logger.info("downloading s3://#{state.bucket}/#{state.key}...") + Logger.info( + bucket: state.bucket, + key: state.key, + message: "Downloading object from S3" + ) with {:ok, %{body: body}} <- get_object(state) do - Logger.debug("downloaded s3://#{state.bucket}/#{state.key}!") + Logger.debug( + bucket: state.bucket, + key: state.key, + message: "Downloaded object from S3" + ) + {:ok, body} else {:error, reason} -> @@ -134,29 +152,53 @@ defmodule RemotePersistentTerm.Fetcher.S3 do defp aws_client_request(op, %{region: region, failover_regions: nil}), do: client().request(op, region: region) - defp aws_client_request(op, %{region: region, bucket: bucket, key: key, failover_regions: failover_regions}) + defp aws_client_request( + op, + %{ + region: region, + bucket: bucket, + key: key, + failover_regions: failover_regions + } = state + ) when is_list(failover_regions) do with {:error, reason} <- client().request(op, region: region) do - Logger.error( - "s3://#{bucket}/#{key} - Failed to fetch from primary region #{region}: #{inspect(reason)}, will try failover regions" - ) - - try_failover_regions(op, failover_regions, bucket, key) + Logger.error(%{ + bucket: bucket, + key: key, + region: region, + reason: inspect(reason), + message: "Failed to fetch from primary region, attempting failover regions" + }) + + try_failover_regions(op, failover_regions, state) end end - defp try_failover_regions(_op, [], _bucket, _key), do: {:error, "All regions failed"} + defp try_failover_regions(_op, [], _state), do: {:error, "All regions failed"} - defp try_failover_regions(op, [region | remaining_regions], bucket, key) do - Logger.info("s3://#{bucket}/#{key} - Trying failover region: #{region}") + defp try_failover_regions(op, [region | remaining_regions], state) do + Logger.info(%{ + bucket: state.bucket, + key: state.key, + region: region, + message: "Trying failover region" + }) case client().request(op, region: region) do {:ok, result} -> {:ok, result} {:error, reason} -> - Logger.error("s3://#{bucket}/#{key} - Failed to fetch from failover region #{region}: #{inspect(reason)}") - try_failover_regions(op, remaining_regions, bucket, key) + Logger.error(%{ + bucket: state.bucket, + key: state.key, + region: region, + reason: inspect(reason), + message: "Failed to fetch from failover region" + }) + + try_failover_regions(op, remaining_regions, state) end end diff --git a/test/remote_persistent_term/fetcher/s3_test.exs b/test/remote_persistent_term/fetcher/s3_test.exs index 1891545..fd14824 100644 --- a/test/remote_persistent_term/fetcher/s3_test.exs +++ b/test/remote_persistent_term/fetcher/s3_test.exs @@ -16,10 +16,16 @@ defmodule RemotePersistentTerm.Fetcher.S3Test do {:error, :unknown_error} end) - assert capture_log(fn -> - assert {:error, "Unknown error"} = S3.current_version(%S3{bucket: "bucket", key: "key"}) - end) =~ - "Elixir.RemotePersistentTerm.Fetcher.S3 - s3://bucket/key - unknown error: :unknown_error" + log = + capture_log(fn -> + assert {:error, "Unknown error"} = + S3.current_version(%S3{bucket: "bucket", key: "key"}) + end) + + assert log =~ "bucket: \"bucket\"" + assert log =~ "key: \"key\"" + assert log =~ "reason: \":unknown_error\"" + assert log =~ "Failed to get current version of object - unknown reason" end describe "init/1" do @@ -67,9 +73,13 @@ defmodule RemotePersistentTerm.Fetcher.S3Test do assert {:ok, "current-etag"} = result end) - assert log =~ "s3://#{@bucket}/#{@key} - Failed to fetch from primary region #{@region}" - assert log =~ "will try failover regions" - assert log =~ "s3://#{@bucket}/#{@key} - Trying failover region: failover-region-1" + assert log =~ "bucket: \"#{@bucket}\"" + assert log =~ "key: \"#{@key}\"" + assert log =~ "region: \"#{@region}\"" + assert log =~ "Failed to fetch from primary region, attempting failover regions" + assert log =~ "region: \"failover-region-1\"" + assert log =~ "Trying failover region" + assert log =~ "Found latest version of object with version: #{@version}" end test "download/1 tries first failover region when primary region fails" do @@ -97,9 +107,14 @@ defmodule RemotePersistentTerm.Fetcher.S3Test do assert {:ok, "content from failover region"} = result end) - assert log =~ "s3://#{@bucket}/#{@key} - Failed to fetch from primary region #{@region}" - assert log =~ "will try failover regions" - assert log =~ "s3://#{@bucket}/#{@key} - Trying failover region: failover-region-1" + assert log =~ "bucket: \"#{@bucket}\"" + assert log =~ "key: \"#{@key}\"" + assert log =~ "Downloading object from S3" + assert log =~ "region: \"#{@region}\"" + assert log =~ "Failed to fetch from primary region, attempting failover regions" + assert log =~ "region: \"failover-region-1\"" + assert log =~ "Trying failover region" + assert log =~ "Downloaded object from S3" end test "returns error when primary and all failover regions fail" do @@ -131,12 +146,17 @@ defmodule RemotePersistentTerm.Fetcher.S3Test do assert message =~ "All regions failed" end) - assert log =~ "s3://#{@bucket}/#{@key} - Failed to fetch from primary region #{@region}" - assert log =~ "will try failover regions" - assert log =~ "s3://#{@bucket}/#{@key} - Trying failover region: failover-region-1" - assert log =~ "s3://#{@bucket}/#{@key} - Failed to fetch from failover region failover-region-1" - assert log =~ "s3://#{@bucket}/#{@key} - Trying failover region: failover-region-2" - assert log =~ "s3://#{@bucket}/#{@key} - Failed to fetch from failover region failover-region-2" + assert log =~ "bucket: \"#{@bucket}\"" + assert log =~ "key: \"#{@key}\"" + assert log =~ "Downloading object from S3" + assert log =~ "region: \"#{@region}\"" + assert log =~ "Failed to fetch from primary region, attempting failover regions" + assert log =~ "region: \"failover-region-1\"" + assert log =~ "Trying failover region" + assert log =~ "reason: \"\\\"First failover region connection error\\\"\"" + assert log =~ "Failed to fetch from failover region" + assert log =~ "region: \"failover-region-2\"" + assert log =~ "reason: \"\\\"Second failover region connection error\\\"\"" end test "tries second failover region when first failover region fails" do @@ -167,10 +187,17 @@ defmodule RemotePersistentTerm.Fetcher.S3Test do assert {:ok, "content from second failover region"} = result end) - assert log =~ "s3://#{@bucket}/#{@key} - Failed to fetch from primary region #{@region}" - assert log =~ "s3://#{@bucket}/#{@key} - Trying failover region: failover-region-1" - assert log =~ "s3://#{@bucket}/#{@key} - Failed to fetch from failover region failover-region-1" - assert log =~ "s3://#{@bucket}/#{@key} - Trying failover region: failover-region-2" + assert log =~ "bucket: \"#{@bucket}\"" + assert log =~ "key: \"#{@key}\"" + assert log =~ "Downloading object from S3" + assert log =~ "region: \"#{@region}\"" + assert log =~ "Failed to fetch from primary region, attempting failover regions" + assert log =~ "region: \"failover-region-1\"" + assert log =~ "Trying failover region" + assert log =~ "reason: \"\\\"First failover region connection error\\\"\"" + assert log =~ "Failed to fetch from failover region" + assert log =~ "region: \"failover-region-2\"" + assert log =~ "Downloaded object from S3" end end end From 823971f094d3e168a5924aef1c6840fa294d2e3b Mon Sep 17 00:00:00 2001 From: Louis Date: Thu, 24 Apr 2025 13:19:24 +0200 Subject: [PATCH 5/6] Fix tests --- test/remote_persistent_term/fetcher/s3_test.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/remote_persistent_term/fetcher/s3_test.exs b/test/remote_persistent_term/fetcher/s3_test.exs index fd14824..f323997 100644 --- a/test/remote_persistent_term/fetcher/s3_test.exs +++ b/test/remote_persistent_term/fetcher/s3_test.exs @@ -79,7 +79,7 @@ defmodule RemotePersistentTerm.Fetcher.S3Test do assert log =~ "Failed to fetch from primary region, attempting failover regions" assert log =~ "region: \"failover-region-1\"" assert log =~ "Trying failover region" - assert log =~ "Found latest version of object with version: #{@version}" + assert log =~ "Found latest version of object" end test "download/1 tries first failover region when primary region fails" do From ab579194f20a48205481311b1170850120e2c15e Mon Sep 17 00:00:00 2001 From: Louis Date: Thu, 24 Apr 2025 13:26:02 +0200 Subject: [PATCH 6/6] Incremement version --- mix.exs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mix.exs b/mix.exs index 75d1aaa..864b6b5 100644 --- a/mix.exs +++ b/mix.exs @@ -2,7 +2,7 @@ defmodule RemotePersistentTerm.MixProject do use Mix.Project @name "RemotePersistentTerm" - @version "0.10.1" + @version "0.11.0" @repo_url "https://github.com/AppMonet/remote_persistent_term" def project do