Add aws fallback region for rpt s3

lib/remote_persistent_term/fetcher/s3.ex

@@ -9,9 +9,10 @@ defmodule RemotePersistentTerm.Fetcher.S3 do
   @type t :: %__MODULE__{
           bucket: String.t(),
           key: String.t(),
-          region: String.t()
+          region: String.t(),
+          failover_regions: [String.t()] | nil
         }
-  defstruct [:bucket, :key, :region]
+  defstruct [:bucket, :key, :region, :failover_regions]
 
   @opts_schema [
     bucket: [
@@ -28,6 +29,12 @@ defmodule RemotePersistentTerm.Fetcher.S3 do
       type: :string,
       required: true,
       doc: "The AWS region of the s3 bucket."
+    ],
+    failover_regions: [
+      type: {:list, :string},
+      required: false,
+      doc:
+        "A list of AWS regions to use if calls to the default region fail. They will be tried in order."
     ]
   ]
 
@@ -50,7 +57,8 @@ defmodule RemotePersistentTerm.Fetcher.S3 do
        %__MODULE__{
          bucket: valid_opts[:bucket],
          key: valid_opts[:key],
-         region: valid_opts[:region]
+         region: valid_opts[:region],
+         failover_regions: valid_opts[:failover_regions]
        }}
     end
   end
@@ -60,7 +68,10 @@ defmodule RemotePersistentTerm.Fetcher.S3 do
     with {:ok, versions} <- list_object_versions(state),
          {:ok, %{etag: etag, version_id: version}} <- find_latest(versions) do
       Logger.info(
-        "found latest version of s3://#{state.bucket}/#{state.key}: #{etag} with version: #{version}"
+        bucket: state.bucket,
+        key: state.key,
+        version: version,
+        message: "Found latest version of object"
       )
 
       {:ok, etag}
@@ -72,17 +83,32 @@ defmodule RemotePersistentTerm.Fetcher.S3 do
         {:error, "could not find s3://#{state.bucket}/#{state.key}"}
 
       {:error, reason} ->
-        Logger.error("#{__MODULE__} - unknown error: #{inspect(reason)}")
+        Logger.error(%{
+          bucket: state.bucket,
+          key: state.key,
+          reason: inspect(reason),
+          message: "Failed to get current version of object - unknown reason"
+        })
+
         {:error, "Unknown error"}
     end
   end
 
   @impl true
   def download(state) do
-    Logger.info("downloading s3://#{state.bucket}/#{state.key}...")
+    Logger.info(
+      bucket: state.bucket,
+      key: state.key,
+      message: "Downloading object from S3"
+    )
 
     with {:ok, %{body: body}} <- get_object(state) do
-      Logger.debug("downloaded s3://#{state.bucket}/#{state.key}!")
+      Logger.debug(
+        bucket: state.bucket,
+        key: state.key,
+        message: "Downloaded object from S3"
+      )
+
       {:ok, body}
     else
       {:error, reason} ->
@@ -94,7 +120,7 @@ defmodule RemotePersistentTerm.Fetcher.S3 do
     res =
       state.bucket
       |> ExAws.S3.get_bucket_object_versions(prefix: state.key)
-      |> aws_client_request(state.region)
+      |> aws_client_request(state)
 
     with {:ok, %{body: %{versions: versions}}} <- res do
       {:ok, versions}
@@ -104,7 +130,7 @@ defmodule RemotePersistentTerm.Fetcher.S3 do
   defp get_object(state) do
     state.bucket
     |> ExAws.S3.get_object(state.key)
-    |> aws_client_request(state.region)
+    |> aws_client_request(state)
   end
 
   defp find_latest([_ | _] = contents) do
@@ -123,8 +149,57 @@ defmodule RemotePersistentTerm.Fetcher.S3 do
 
   defp find_latest(_), do: {:error, :not_found}
 
-  defp aws_client_request(op, region) do
-    client().request(op, region: region)
+  defp aws_client_request(op, %{region: region, failover_regions: nil}),
+    do: client().request(op, region: region)
+
+  defp aws_client_request(
+         op,
+         %{
+           region: region,
+           bucket: bucket,
+           key: key,
+           failover_regions: failover_regions
+         } = state
+       )
+       when is_list(failover_regions) do
+    with {:error, reason} <- client().request(op, region: region) do
+      Logger.error(%{
+        bucket: bucket,
+        key: key,
+        region: region,
+        reason: inspect(reason),
+        message: "Failed to fetch from primary region, attempting failover regions"
+      })
+
+      try_failover_regions(op, failover_regions, state)
+    end
+  end
+
+  defp try_failover_regions(_op, [], _state), do: {:error, "All regions failed"}
+
+  defp try_failover_regions(op, [region | remaining_regions], state) do
+    Logger.info(%{
+      bucket: state.bucket,
+      key: state.key,
+      region: region,
+      message: "Trying failover region"
+    })
+
+    case client().request(op, region: region) do
+      {:ok, result} ->
+        {:ok, result}
+
+      {:error, reason} ->
+        Logger.error(%{
+          bucket: state.bucket,
+          key: state.key,
+          region: region,
+          reason: inspect(reason),
+          message: "Failed to fetch from failover region"
+        })
+
+        try_failover_regions(op, remaining_regions, state)
+    end
   end
 
   defp client, do: Application.get_env(:remote_persistent_term, :aws_client, ExAws)

mix.exs

@@ -2,7 +2,7 @@ defmodule RemotePersistentTerm.MixProject do
   use Mix.Project
 
   @name "RemotePersistentTerm"
-  @version "0.10.1"
+  @version "0.11.0"
   @repo_url "https://github.com/AppMonet/remote_persistent_term"
 
   def project do

test/remote_persistent_term/fetcher/s3_test.exs

@@ -5,15 +5,27 @@ defmodule RemotePersistentTerm.Fetcher.S3Test do
   setup :verify_on_exit!
   import ExUnit.CaptureLog
 
+  @bucket "test-bucket"
+  @key "test-key"
+  @region "test-region"
+  @failover_regions ["failover-region-1", "failover-region-2"]
+  @version "F76V.weh4uOlU15f7a2OLHPgCLXkDpm4"
+
   test "Unknown error returns an error for current_version/1" do
     expect(AwsClientMock, :request, fn _op, _opts ->
       {:error, :unknown_error}
     end)
 
-    assert capture_log(fn ->
-             assert {:error, "Unknown error"} = S3.current_version(%S3{bucket: "bucket"})
-           end) =~
-             "Elixir.RemotePersistentTerm.Fetcher.S3 - unknown error: :unknown_error"
+    log =
+      capture_log(fn ->
+        assert {:error, "Unknown error"} =
+                 S3.current_version(%S3{bucket: "bucket", key: "key"})
+      end)
+
+    assert log =~ "bucket: \"bucket\""
+    assert log =~ "key: \"key\""
+    assert log =~ "reason: \":unknown_error\""
+    assert log =~ "Failed to get current version of object - unknown reason"
   end
 
   describe "init/1" do
@@ -26,4 +38,166 @@ defmodule RemotePersistentTerm.Fetcher.S3Test do
                S3.init(bucket: bucket, key: key, region: region)
     end
   end
+
+  describe "failover_regions" do
+    test "current_identifiers/1 tries first failover region when primary region fails" do
+      # Setup state with failover regions
+      state = %S3{
+        bucket: @bucket,
+        key: @key,
+        region: @region,
+        failover_regions: @failover_regions
+      }
+
+      # Mock the AWS client to fail for primary region but succeed for first failover region
+      expect(AwsClientMock, :request, 2, fn _op, opts ->
+        case opts do
+          [region: @region] ->
+            {:error, "Primary region connection error"}
+
+          [region: "failover-region-1"] ->
+            {:ok,
+             %{
+               body: %{
+                 versions: [
+                   %{version_id: @version, etag: "current-etag", is_latest: "true"}
+                 ]
+               }
+             }}
+        end
+      end)
+
+      log =
+        capture_log(fn ->
+          result = S3.current_version(state)
+          assert {:ok, "current-etag"} = result
+        end)
+
+      assert log =~ "bucket: \"#{@bucket}\""
+      assert log =~ "key: \"#{@key}\""
+      assert log =~ "region: \"#{@region}\""
+      assert log =~ "Failed to fetch from primary region, attempting failover regions"
+      assert log =~ "region: \"failover-region-1\""
+      assert log =~ "Trying failover region"
+      assert log =~ "Found latest version of object"
+    end
+
+    test "download/1 tries first failover region when primary region fails" do
+      state = %S3{
+        bucket: @bucket,
+        key: @key,
+        region: @region,
+        failover_regions: @failover_regions
+      }
+
+      # Mock the AWS client to fail for primary region but succeed for first failover region
+      expect(AwsClientMock, :request, 2, fn _op, opts ->
+        case opts do
+          [region: @region] ->
+            {:error, "Primary region connection error"}
+
+          [region: "failover-region-1"] ->
+            {:ok, %{body: "content from failover region"}}
+        end
+      end)
+
+      log =
+        capture_log(fn ->
+          result = S3.download(state)
+          assert {:ok, "content from failover region"} = result
+        end)
+
+      assert log =~ "bucket: \"#{@bucket}\""
+      assert log =~ "key: \"#{@key}\""
+      assert log =~ "Downloading object from S3"
+      assert log =~ "region: \"#{@region}\""
+      assert log =~ "Failed to fetch from primary region, attempting failover regions"
+      assert log =~ "region: \"failover-region-1\""
+      assert log =~ "Trying failover region"
+      assert log =~ "Downloaded object from S3"
+    end
+
+    test "returns error when primary and all failover regions fail" do
+      state = %S3{
+        bucket: @bucket,
+        key: @key,
+        region: @region,
+        failover_regions: @failover_regions
+      }
+
+      # Mock the AWS client to fail for all regions
+      expect(AwsClientMock, :request, 3, fn _op, opts ->
+        case opts do
+          [region: @region] ->
+            {:error, "Primary region connection error"}
+
+          [region: "failover-region-1"] ->
+            {:error, "First failover region connection error"}
+
+          [region: "failover-region-2"] ->
+            {:error, "Second failover region connection error"}
+        end
+      end)
+
+      log =
+        capture_log(fn ->
+          result = S3.download(state)
+          assert {:error, message} = result
+          assert message =~ "All regions failed"
+        end)
+
+      assert log =~ "bucket: \"#{@bucket}\""
+      assert log =~ "key: \"#{@key}\""
+      assert log =~ "Downloading object from S3"
+      assert log =~ "region: \"#{@region}\""
+      assert log =~ "Failed to fetch from primary region, attempting failover regions"
+      assert log =~ "region: \"failover-region-1\""
+      assert log =~ "Trying failover region"
+      assert log =~ "reason: \"\\\"First failover region connection error\\\"\""
+      assert log =~ "Failed to fetch from failover region"
+      assert log =~ "region: \"failover-region-2\""
+      assert log =~ "reason: \"\\\"Second failover region connection error\\\"\""
+    end
+
+    test "tries second failover region when first failover region fails" do
+      state = %S3{
+        bucket: @bucket,
+        key: @key,
+        region: @region,
+        failover_regions: @failover_regions
+      }
+
+      # Mock the AWS client to fail for primary and first failover region but succeed for second failover region
+      expect(AwsClientMock, :request, 3, fn _op, opts ->
+        case opts do
+          [region: @region] ->
+            {:error, "Primary region connection error"}
+
+          [region: "failover-region-1"] ->
+            {:error, "First failover region connection error"}
+
+          [region: "failover-region-2"] ->
+            {:ok, %{body: "content from second failover region"}}
+        end
+      end)
+
+      log =
+        capture_log(fn ->
+          result = S3.download(state)
+          assert {:ok, "content from second failover region"} = result
+        end)
+
+      assert log =~ "bucket: \"#{@bucket}\""
+      assert log =~ "key: \"#{@key}\""
+      assert log =~ "Downloading object from S3"
+      assert log =~ "region: \"#{@region}\""
+      assert log =~ "Failed to fetch from primary region, attempting failover regions"
+      assert log =~ "region: \"failover-region-1\""
+      assert log =~ "Trying failover region"
+      assert log =~ "reason: \"\\\"First failover region connection error\\\"\""
+      assert log =~ "Failed to fetch from failover region"
+      assert log =~ "region: \"failover-region-2\""
+      assert log =~ "Downloaded object from S3"
+    end
+  end
 end