diff --git a/lib/remote_persistent_term/fetcher/s3.ex b/lib/remote_persistent_term/fetcher/s3.ex index a463874..f2d16f2 100644 --- a/lib/remote_persistent_term/fetcher/s3.ex +++ b/lib/remote_persistent_term/fetcher/s3.ex @@ -6,13 +6,30 @@ defmodule RemotePersistentTerm.Fetcher.S3 do @behaviour RemotePersistentTerm.Fetcher + @type bucket :: String.t() + @type region :: String.t() + @type failover_bucket :: [bucket: bucket, region: region] + @type t :: %__MODULE__{ - bucket: String.t(), + bucket: bucket, key: String.t(), - region: String.t(), - failover_regions: [String.t()] | nil + region: region, + failover_buckets: [failover_bucket] | nil } - defstruct [:bucket, :key, :region, :failover_regions] + defstruct [:bucket, :key, :region, :failover_buckets] + + @failover_bucket_schema [ + bucket: [ + type: :string, + required: true, + doc: "The name of the failover S3 bucket." + ], + region: [ + type: :string, + required: true, + doc: "The AWS region of the failover S3 bucket." + ] + ] @opts_schema [ bucket: [ @@ -30,11 +47,11 @@ defmodule RemotePersistentTerm.Fetcher.S3 do required: true, doc: "The AWS region of the s3 bucket." ], - failover_regions: [ - type: {:list, :string}, + failover_buckets: [ + type: {:list, {:keyword_list, @failover_bucket_schema}}, required: false, - doc: - "A list of AWS regions to use if calls to the default region fail. They will be tried in order." + doc: "A list of failover_buckets to use as failover if the primary bucket fails. \n + The directory structure in failover buckets must match the primary bucket." ] ] @@ -58,7 +75,7 @@ defmodule RemotePersistentTerm.Fetcher.S3 do bucket: valid_opts[:bucket], key: valid_opts[:key], region: valid_opts[:region], - failover_regions: valid_opts[:failover_regions] + failover_buckets: valid_opts[:failover_buckets] }} end end @@ -118,9 +135,11 @@ defmodule RemotePersistentTerm.Fetcher.S3 do defp list_object_versions(state) do res = - state.bucket - |> ExAws.S3.get_bucket_object_versions(prefix: state.key) - |> aws_client_request(state) + aws_client_request( + &ExAws.S3.get_bucket_object_versions/2, + state, + prefix: state.key + ) with {:ok, %{body: %{versions: versions}}} <- res do {:ok, versions} @@ -128,9 +147,7 @@ defmodule RemotePersistentTerm.Fetcher.S3 do end defp get_object(state) do - state.bucket - |> ExAws.S3.get_object(state.key) - |> aws_client_request(state) + aws_client_request(&ExAws.S3.get_object/2, state, state.key) end defp find_latest([_ | _] = contents) do @@ -149,58 +166,66 @@ defmodule RemotePersistentTerm.Fetcher.S3 do defp find_latest(_), do: {:error, :not_found} - defp aws_client_request(op, %{region: region, failover_regions: nil}), - do: client().request(op, region: region) + defp aws_client_request(op, %{failover_buckets: nil} = state, opts) do + perform_request(op, state.bucket, state.region, opts) + end defp aws_client_request( op, %{ - region: region, - bucket: bucket, - key: key, - failover_regions: failover_regions - } = state - ) - when is_list(failover_regions) do - with {:error, reason} <- client().request(op, region: region) do + failover_buckets: [_|_] = failover_buckets + } = state, + opts + ) do + with {:error, reason} <- perform_request(op, state.bucket, state.region, opts) do Logger.error(%{ - bucket: bucket, - key: key, - region: region, + bucket: state.bucket, + key: state.key, + region: state.region, reason: inspect(reason), - message: "Failed to fetch from primary region, attempting failover regions" + message: "Failed to fetch from primary bucket, attempting failover buckets" }) - try_failover_regions(op, failover_regions, state) + try_failover_buckets(op, failover_buckets, opts, state) end end - defp try_failover_regions(_op, [], _state), do: {:error, "All regions failed"} + defp try_failover_buckets(_op, [], _opts, _state), do: {:error, "All buckets failed"} - defp try_failover_regions(op, [region | remaining_regions], state) do + defp try_failover_buckets( + op, + [[bucket: bucket, region: region] | remaining_buckets], + opts, + state + ) do Logger.info(%{ - bucket: state.bucket, + bucket: bucket, key: state.key, region: region, - message: "Trying failover region" + message: "Trying failover bucket" }) - case client().request(op, region: region) do + case perform_request(op, bucket, region, opts) do {:ok, result} -> {:ok, result} {:error, reason} -> Logger.error(%{ - bucket: state.bucket, + bucket: bucket, key: state.key, region: region, reason: inspect(reason), - message: "Failed to fetch from failover region" + message: "Failed to fetch from failover bucket" }) - try_failover_regions(op, remaining_regions, state) + try_failover_buckets(op, remaining_buckets, opts, state) end end + defp perform_request(op, bucket, region, opts) do + op.(bucket, opts) + |> client().request(region: region) + end + defp client, do: Application.get_env(:remote_persistent_term, :aws_client, ExAws) end diff --git a/mix.exs b/mix.exs index 864b6b5..e6cbd69 100644 --- a/mix.exs +++ b/mix.exs @@ -2,7 +2,7 @@ defmodule RemotePersistentTerm.MixProject do use Mix.Project @name "RemotePersistentTerm" - @version "0.11.0" + @version "0.12.0" @repo_url "https://github.com/AppMonet/remote_persistent_term" def project do @@ -32,7 +32,7 @@ defmodule RemotePersistentTerm.MixProject do {:telemetry, "~> 1.0"}, {:ex_doc, "~> 0.27", only: :dev, runtime: false}, {:ex_aws, "~> 2.1"}, - {:ex_aws_s3, "~> 2.0"}, + {:ex_aws_s3, "~> 2.5.7"}, {:configparser_ex, "~> 4.0", optional: true}, {:hackney, "~> 1.9"}, {:sweet_xml, "~> 0.6"}, diff --git a/mix.lock b/mix.lock index 8874ce5..0a9b629 100644 --- a/mix.lock +++ b/mix.lock @@ -7,12 +7,12 @@ "cowboy_telemetry": {:hex, :cowboy_telemetry, "0.4.0", "f239f68b588efa7707abce16a84d0d2acf3a0f50571f8bb7f56a15865aae820c", [:rebar3], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "7d98bac1ee4565d31b62d59f8823dfd8356a169e7fcbb83831b8a5397404c9de"}, "cowlib": {:hex, :cowlib, "2.14.0", "623791c56c1cc9df54a71a9c55147a401549917f00a2e48a6ae12b812c586ced", [:make, :rebar3], [], "hexpm", "0af652d1550c8411c3b58eed7a035a7fb088c0b86aff6bc504b0bc3b7f791aa2"}, "earmark_parser": {:hex, :earmark_parser, "1.4.44", "f20830dd6b5c77afe2b063777ddbbff09f9759396500cdbe7523efd58d7a339c", [:mix], [], "hexpm", "4778ac752b4701a5599215f7030989c989ffdc4f6df457c5f36938cc2d2a2750"}, - "ex_aws": {:hex, :ex_aws, "2.5.8", "0393cfbc5e4a9e7017845451a015d836a670397100aa4c86901980e2a2c5f7d4", [:mix], [{:configparser_ex, "~> 4.0", [hex: :configparser_ex, repo: "hexpm", optional: true]}, {:hackney, "~> 1.16", [hex: :hackney, repo: "hexpm", optional: true]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: true]}, {:jsx, "~> 2.8 or ~> 3.0", [hex: :jsx, repo: "hexpm", optional: true]}, {:mime, "~> 1.2 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:req, "~> 0.3", [hex: :req, repo: "hexpm", optional: true]}, {:sweet_xml, "~> 0.7", [hex: :sweet_xml, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "8f79777b7932168956c8cc3a6db41f5783aa816eb50de356aed3165a71e5f8c3"}, - "ex_aws_s3": {:hex, :ex_aws_s3, "2.5.6", "d135983bbd8b6df6350dfd83999437725527c1bea151e5055760bfc9b2d17c20", [:mix], [{:ex_aws, "~> 2.0", [hex: :ex_aws, repo: "hexpm", optional: false]}, {:sweet_xml, ">= 0.0.0", [hex: :sweet_xml, repo: "hexpm", optional: true]}], "hexpm", "9874e12847e469ca2f13a5689be04e546c16f63caf6380870b7f25bf7cb98875"}, + "ex_aws": {:hex, :ex_aws, "2.5.9", "8e2455172f0e5cbe2f56dd68de514f0dae6bb26d6b6e2f435a06434cf9dbb412", [:mix], [{:configparser_ex, "~> 4.0", [hex: :configparser_ex, repo: "hexpm", optional: true]}, {:hackney, "~> 1.16", [hex: :hackney, repo: "hexpm", optional: true]}, {:jason, "~> 1.1", [hex: :jason, repo: "hexpm", optional: true]}, {:jsx, "~> 2.8 or ~> 3.0", [hex: :jsx, repo: "hexpm", optional: true]}, {:mime, "~> 1.2 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:req, "~> 0.5.10 or ~> 0.6 or ~> 1.0", [hex: :req, repo: "hexpm", optional: true]}, {:sweet_xml, "~> 0.7", [hex: :sweet_xml, repo: "hexpm", optional: true]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "cbdb6ffb0e6c6368de05ed8641fe1376298ba23354674428e5b153a541f23359"}, + "ex_aws_s3": {:hex, :ex_aws_s3, "2.5.7", "e571424d2f345299753382f3a01b005c422b1a460a8bc3ed47659b3d3ef91e9e", [:mix], [{:ex_aws, "~> 2.0", [hex: :ex_aws, repo: "hexpm", optional: false]}, {:sweet_xml, ">= 0.0.0", [hex: :sweet_xml, repo: "hexpm", optional: true]}], "hexpm", "858e51241e50181e29aa2bc128fef548873a3a9cd580471f57eda5b64dec937f"}, "ex_doc": {:hex, :ex_doc, "0.37.3", "f7816881a443cd77872b7d6118e8a55f547f49903aef8747dbcb345a75b462f9", [:mix], [{:earmark_parser, "~> 1.4.42", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_c, ">= 0.1.0", [hex: :makeup_c, repo: "hexpm", optional: true]}, {:makeup_elixir, "~> 0.14 or ~> 1.0", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1 or ~> 1.0", [hex: :makeup_erlang, repo: "hexpm", optional: false]}, {:makeup_html, ">= 0.1.0", [hex: :makeup_html, repo: "hexpm", optional: true]}], "hexpm", "e6aebca7156e7c29b5da4daa17f6361205b2ae5f26e5c7d8ca0d3f7e18972233"}, "finch": {:hex, :finch, "0.19.0", "c644641491ea854fc5c1bbaef36bfc764e3f08e7185e1f084e35e0672241b76d", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:mint, "~> 1.6.2 or ~> 1.7", [hex: :mint, repo: "hexpm", optional: false]}, {:nimble_options, "~> 0.4 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:nimble_pool, "~> 1.1", [hex: :nimble_pool, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "fc5324ce209125d1e2fa0fcd2634601c52a787aff1cd33ee833664a5af4ea2b6"}, "hackney": {:hex, :hackney, "1.23.0", "55cc09077112bcb4a69e54be46ed9bc55537763a96cd4a80a221663a7eafd767", [:rebar3], [{:certifi, "~> 2.14.0", [hex: :certifi, repo: "hexpm", optional: false]}, {:idna, "~> 6.1.0", [hex: :idna, repo: "hexpm", optional: false]}, {:metrics, "~> 1.0.0", [hex: :metrics, repo: "hexpm", optional: false]}, {:mimerl, "~> 1.1", [hex: :mimerl, repo: "hexpm", optional: false]}, {:parse_trans, "3.4.1", [hex: :parse_trans, repo: "hexpm", optional: false]}, {:ssl_verify_fun, "~> 1.1.0", [hex: :ssl_verify_fun, repo: "hexpm", optional: false]}, {:unicode_util_compat, "~> 0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "6cd1c04cd15c81e5a493f167b226a15f0938a84fc8f0736ebe4ddcab65c0b44e"}, - "hpax": {:hex, :hpax, "1.0.2", "762df951b0c399ff67cc57c3995ec3cf46d696e41f0bba17da0518d94acd4aac", [:mix], [], "hexpm", "2f09b4c1074e0abd846747329eaa26d535be0eb3d189fa69d812bfb8bfefd32f"}, + "hpax": {:hex, :hpax, "1.0.3", "ed67ef51ad4df91e75cc6a1494f851850c0bd98ebc0be6e81b026e765ee535aa", [:mix], [], "hexpm", "8eab6e1cfa8d5918c2ce4ba43588e894af35dbd8e91e6e55c817bca5847df34a"}, "idna": {:hex, :idna, "6.1.1", "8a63070e9f7d0c62eb9d9fcb360a7de382448200fbbd1b106cc96d3d8099df8d", [:rebar3], [{:unicode_util_compat, "~> 0.7.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "92376eb7894412ed19ac475e4a86f7b413c1b9fbb5bd16dccd57934157944cea"}, "jason": {:hex, :jason, "1.4.4", "b9226785a9aa77b6857ca22832cffa5d5011a667207eb2a0ad56adb5db443b8a", [:mix], [{:decimal, "~> 1.0 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "c5eb0cab91f094599f94d55bc63409236a8ec69a21a67814529e8d5f6cc90b3b"}, "makeup": {:hex, :makeup, "1.2.1", "e90ac1c65589ef354378def3ba19d401e739ee7ee06fb47f94c687016e3713d1", [:mix], [{:nimble_parsec, "~> 1.4", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "d36484867b0bae0fea568d10131197a4c2e47056a6fbe84922bf6ba71c8d17ce"}, @@ -30,9 +30,9 @@ "parse_trans": {:hex, :parse_trans, "3.4.1", "6e6aa8167cb44cc8f39441d05193be6e6f4e7c2946cb2759f015f8c56b76e5ff", [:rebar3], [], "hexpm", "620a406ce75dada827b82e453c19cf06776be266f5a67cff34e1ef2cbb60e49a"}, "plug": {:hex, :plug, "1.17.0", "a0832e7af4ae0f4819e0c08dd2e7482364937aea6a8a997a679f2cbb7e026b2e", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "f6692046652a69a00a5a21d0b7e11fcf401064839d59d6b8787f23af55b1e6bc"}, "plug_cowboy": {:hex, :plug_cowboy, "2.7.3", "1304d36752e8bdde213cea59ef424ca932910a91a07ef9f3874be709c4ddb94b", [:mix], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:cowboy_telemetry, "~> 0.3", [hex: :cowboy_telemetry, repo: "hexpm", optional: false]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "77c95524b2aa5364b247fa17089029e73b951ebc1adeef429361eab0bb55819d"}, - "plug_crypto": {:hex, :plug_crypto, "2.1.0", "f44309c2b06d249c27c8d3f65cfe08158ade08418cf540fd4f72d4d6863abb7b", [:mix], [], "hexpm", "131216a4b030b8f8ce0f26038bc4421ae60e4bb95c5cf5395e1421437824c4fa"}, + "plug_crypto": {:hex, :plug_crypto, "2.1.1", "19bda8184399cb24afa10be734f84a16ea0a2bc65054e23a62bb10f06bc89491", [:mix], [], "hexpm", "6470bce6ffe41c8bd497612ffde1a7e4af67f36a15eea5f921af71cf3e11247c"}, "ranch": {:hex, :ranch, "1.8.1", "208169e65292ac5d333d6cdbad49388c1ae198136e4697ae2f474697140f201c", [:make, :rebar3], [], "hexpm", "aed58910f4e21deea992a67bf51632b6d60114895eb03bb392bb733064594dd0"}, - "req": {:hex, :req, "0.5.9", "09072dcd91a70c58734c4dd4fa878a9b6d36527291152885100ec33a5a07f1d6", [:mix], [{:brotli, "~> 0.3.1", [hex: :brotli, repo: "hexpm", optional: true]}, {:ezstd, "~> 1.0", [hex: :ezstd, repo: "hexpm", optional: true]}, {:finch, "~> 0.17", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mime, "~> 2.0.6 or ~> 2.1", [hex: :mime, repo: "hexpm", optional: false]}, {:nimble_csv, "~> 1.0", [hex: :nimble_csv, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "2f027043003275918f5e79e6a4e57b10cb17161a1ab41c959aa40ecfb2142e5a"}, + "req": {:hex, :req, "0.5.10", "a3a063eab8b7510785a467f03d30a8d95f66f5c3d9495be3474b61459c54376c", [:mix], [{:brotli, "~> 0.3.1", [hex: :brotli, repo: "hexpm", optional: true]}, {:ezstd, "~> 1.0", [hex: :ezstd, repo: "hexpm", optional: true]}, {:finch, "~> 0.17", [hex: :finch, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}, {:mime, "~> 2.0.6 or ~> 2.1", [hex: :mime, repo: "hexpm", optional: false]}, {:nimble_csv, "~> 1.0", [hex: :nimble_csv, repo: "hexpm", optional: true]}, {:plug, "~> 1.0", [hex: :plug, repo: "hexpm", optional: true]}], "hexpm", "8a604815743f8a2d3b5de0659fa3137fa4b1cffd636ecb69b30b2b9b2c2559be"}, "ssl_verify_fun": {:hex, :ssl_verify_fun, "1.1.7", "354c321cf377240c7b8716899e182ce4890c5938111a1296add3ec74cf1715df", [:make, :mix, :rebar3], [], "hexpm", "fe4c190e8f37401d30167c8c405eda19469f34577987c76dde613e838bbc67f8"}, "sweet_xml": {:hex, :sweet_xml, "0.7.5", "803a563113981aaac202a1dbd39771562d0ad31004ddbfc9b5090bdcd5605277", [:mix], [], "hexpm", "193b28a9b12891cae351d81a0cead165ffe67df1b73fe5866d10629f4faefb12"}, "telemetry": {:hex, :telemetry, "1.3.0", "fedebbae410d715cf8e7062c96a1ef32ec22e764197f70cda73d82778d61e7a2", [:rebar3], [], "hexpm", "7015fc8919dbe63764f4b4b87a95b7c0996bd539e0d499be6ec9d7f3875b79e6"}, diff --git a/test/remote_persistent_term/fetcher/s3_test.exs b/test/remote_persistent_term/fetcher/s3_test.exs index f323997..9e7d34d 100644 --- a/test/remote_persistent_term/fetcher/s3_test.exs +++ b/test/remote_persistent_term/fetcher/s3_test.exs @@ -8,7 +8,10 @@ defmodule RemotePersistentTerm.Fetcher.S3Test do @bucket "test-bucket" @key "test-key" @region "test-region" - @failover_regions ["failover-region-1", "failover-region-2"] + @failover_buckets [ + [bucket: "failover-bucket-1", region: "failover-region-1"], + [bucket: "failover-bucket-2", region: "failover-region-2"] + ] @version "F76V.weh4uOlU15f7a2OLHPgCLXkDpm4" test "Unknown error returns an error for current_version/1" do @@ -37,25 +40,48 @@ defmodule RemotePersistentTerm.Fetcher.S3Test do assert {:ok, %S3{bucket: bucket, key: key, region: region}} == S3.init(bucket: bucket, key: key, region: region) end + + test "with failover buckets" do + bucket = "my-bucket" + key = "my-key" + region = "my-region" + + failover_buckets = [ + [bucket: "backup-bucket", region: "backup-region"], + [bucket: "dr-bucket", region: "dr-region"] + ] + + assert {:ok, + %S3{bucket: bucket, key: key, region: region, failover_buckets: failover_buckets}} == + S3.init( + bucket: bucket, + key: key, + region: region, + failover_buckets: failover_buckets + ) + end end - describe "failover_regions" do - test "current_identifiers/1 tries first failover region when primary region fails" do - # Setup state with failover regions + describe "failover_buckets" do + test "current_version/1 tries first failover bucket when primary bucket fails" do + # Setup state with failover buckets state = %S3{ bucket: @bucket, key: @key, region: @region, - failover_regions: @failover_regions + failover_buckets: @failover_buckets } - # Mock the AWS client to fail for primary region but succeed for first failover region - expect(AwsClientMock, :request, 2, fn _op, opts -> - case opts do - [region: @region] -> - {:error, "Primary region connection error"} + # Mock the AWS client to fail for primary bucket but succeed for first failover bucket + expect(AwsClientMock, :request, 2, fn operation, opts -> + op_bucket = operation.bucket + region = Keyword.get(opts, :region) + + cond do + op_bucket == @bucket && region == @region -> + {:error, "Primary bucket connection error"} - [region: "failover-region-1"] -> + op_bucket == "failover-bucket-1" && region == "failover-region-1" -> {:ok, %{ body: %{ @@ -64,6 +90,9 @@ defmodule RemotePersistentTerm.Fetcher.S3Test do ] } }} + + true -> + {:error, "Unexpected bucket or region"} end end) @@ -76,66 +105,80 @@ defmodule RemotePersistentTerm.Fetcher.S3Test do assert log =~ "bucket: \"#{@bucket}\"" assert log =~ "key: \"#{@key}\"" assert log =~ "region: \"#{@region}\"" - assert log =~ "Failed to fetch from primary region, attempting failover regions" + assert log =~ "Failed to fetch from primary bucket, attempting failover buckets" + assert log =~ "bucket: \"failover-bucket-1\"" assert log =~ "region: \"failover-region-1\"" - assert log =~ "Trying failover region" + assert log =~ "Trying failover bucket" assert log =~ "Found latest version of object" end - test "download/1 tries first failover region when primary region fails" do + test "download/1 tries first failover bucket when primary bucket fails" do state = %S3{ bucket: @bucket, key: @key, region: @region, - failover_regions: @failover_regions + failover_buckets: @failover_buckets } - # Mock the AWS client to fail for primary region but succeed for first failover region - expect(AwsClientMock, :request, 2, fn _op, opts -> - case opts do - [region: @region] -> - {:error, "Primary region connection error"} + # Mock the AWS client to fail for primary bucket but succeed for first failover bucket + expect(AwsClientMock, :request, 2, fn operation, opts -> + op_bucket = operation.bucket + region = Keyword.get(opts, :region) + + cond do + op_bucket == @bucket && region == @region -> + {:error, "Primary bucket connection error"} - [region: "failover-region-1"] -> - {:ok, %{body: "content from failover region"}} + op_bucket == "failover-bucket-1" && region == "failover-region-1" -> + {:ok, %{body: "content from failover bucket"}} + + true -> + {:error, "Unexpected bucket or region"} end end) log = capture_log(fn -> result = S3.download(state) - assert {:ok, "content from failover region"} = result + assert {:ok, "content from failover bucket"} = result end) assert log =~ "bucket: \"#{@bucket}\"" assert log =~ "key: \"#{@key}\"" assert log =~ "Downloading object from S3" assert log =~ "region: \"#{@region}\"" - assert log =~ "Failed to fetch from primary region, attempting failover regions" + assert log =~ "Failed to fetch from primary bucket, attempting failover buckets" + assert log =~ "bucket: \"failover-bucket-1\"" assert log =~ "region: \"failover-region-1\"" - assert log =~ "Trying failover region" + assert log =~ "Trying failover bucket" assert log =~ "Downloaded object from S3" end - test "returns error when primary and all failover regions fail" do + test "returns error when primary and all failover buckets fail" do state = %S3{ bucket: @bucket, key: @key, region: @region, - failover_regions: @failover_regions + failover_buckets: @failover_buckets } - # Mock the AWS client to fail for all regions - expect(AwsClientMock, :request, 3, fn _op, opts -> - case opts do - [region: @region] -> - {:error, "Primary region connection error"} + # Mock the AWS client to fail for all buckets + expect(AwsClientMock, :request, 3, fn operation, opts -> + op_bucket = operation.bucket + region = Keyword.get(opts, :region) + + cond do + op_bucket == @bucket && region == @region -> + {:error, "Primary bucket connection error"} - [region: "failover-region-1"] -> - {:error, "First failover region connection error"} + op_bucket == "failover-bucket-1" && region == "failover-region-1" -> + {:error, "First failover bucket connection error"} - [region: "failover-region-2"] -> - {:error, "Second failover region connection error"} + op_bucket == "failover-bucket-2" && region == "failover-region-2" -> + {:error, "Second failover bucket connection error"} + + true -> + {:error, "Unexpected bucket or region"} end end) @@ -143,59 +186,69 @@ defmodule RemotePersistentTerm.Fetcher.S3Test do capture_log(fn -> result = S3.download(state) assert {:error, message} = result - assert message =~ "All regions failed" + assert message =~ "All buckets failed" end) assert log =~ "bucket: \"#{@bucket}\"" assert log =~ "key: \"#{@key}\"" assert log =~ "Downloading object from S3" assert log =~ "region: \"#{@region}\"" - assert log =~ "Failed to fetch from primary region, attempting failover regions" + assert log =~ "Failed to fetch from primary bucket, attempting failover buckets" + assert log =~ "bucket: \"failover-bucket-1\"" assert log =~ "region: \"failover-region-1\"" - assert log =~ "Trying failover region" - assert log =~ "reason: \"\\\"First failover region connection error\\\"\"" - assert log =~ "Failed to fetch from failover region" + assert log =~ "Trying failover bucket" + assert log =~ "reason: \"\\\"First failover bucket connection error\\\"\"" + assert log =~ "Failed to fetch from failover bucket" + assert log =~ "bucket: \"failover-bucket-2\"" assert log =~ "region: \"failover-region-2\"" - assert log =~ "reason: \"\\\"Second failover region connection error\\\"\"" + assert log =~ "reason: \"\\\"Second failover bucket connection error\\\"\"" end - test "tries second failover region when first failover region fails" do + test "tries second failover bucket when first failover bucket fails" do state = %S3{ bucket: @bucket, key: @key, region: @region, - failover_regions: @failover_regions + failover_buckets: @failover_buckets } - # Mock the AWS client to fail for primary and first failover region but succeed for second failover region - expect(AwsClientMock, :request, 3, fn _op, opts -> - case opts do - [region: @region] -> - {:error, "Primary region connection error"} + # Mock the AWS client to fail for primary and first failover bucket but succeed for second failover bucket + expect(AwsClientMock, :request, 3, fn operation, opts -> + op_bucket = operation.bucket + region = Keyword.get(opts, :region) + + cond do + op_bucket == @bucket && region == @region -> + {:error, "Primary bucket connection error"} + + op_bucket == "failover-bucket-1" && region == "failover-region-1" -> + {:error, "First failover bucket connection error"} - [region: "failover-region-1"] -> - {:error, "First failover region connection error"} + op_bucket == "failover-bucket-2" && region == "failover-region-2" -> + {:ok, %{body: "content from second failover bucket"}} - [region: "failover-region-2"] -> - {:ok, %{body: "content from second failover region"}} + true -> + {:error, "Unexpected bucket or region"} end end) log = capture_log(fn -> result = S3.download(state) - assert {:ok, "content from second failover region"} = result + assert {:ok, "content from second failover bucket"} = result end) assert log =~ "bucket: \"#{@bucket}\"" assert log =~ "key: \"#{@key}\"" assert log =~ "Downloading object from S3" assert log =~ "region: \"#{@region}\"" - assert log =~ "Failed to fetch from primary region, attempting failover regions" + assert log =~ "Failed to fetch from primary bucket, attempting failover buckets" + assert log =~ "bucket: \"failover-bucket-1\"" assert log =~ "region: \"failover-region-1\"" - assert log =~ "Trying failover region" - assert log =~ "reason: \"\\\"First failover region connection error\\\"\"" - assert log =~ "Failed to fetch from failover region" + assert log =~ "Trying failover bucket" + assert log =~ "reason: \"\\\"First failover bucket connection error\\\"\"" + assert log =~ "Failed to fetch from failover bucket" + assert log =~ "bucket: \"failover-bucket-2\"" assert log =~ "region: \"failover-region-2\"" assert log =~ "Downloaded object from S3" end