Merge remote-tracking branch 'upstream/main'

# Conflicts:
#	mix.lock

.github/dependabot.yml

@@ -1,14 +1,14 @@
 version: 2
 updates:
-- package-ecosystem: mix
-  directory: "/"
-  schedule:
-    interval: daily
-  open-pull-requests-limit: 10
-  ignore:
-  - dependency-name: ex_doc
-    versions:
-    - 0.24.2
-  - dependency-name: hackney
-    versions:
-    - 1.17.1
+  - package-ecosystem: mix
+    directory: "/"
+    schedule:
+      interval: daily
+    open-pull-requests-limit: 10
+    ignore:
+      - dependency-name: ex_doc
+        versions:
+          - 0.24.2
+      - dependency-name: hackney
+        versions:
+          - 1.17.1

.github/workflows/on-push.yml

@@ -6,27 +6,26 @@ env:
   AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
 jobs:
   build:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-20.04
     strategy:
       matrix:
-        otp: ['23.3.3']
-        elixir: ['1.7.4', '1.8.2', '1.9.4', '1.10.4', '1.11.4', '1.12.3', '1.13.2']
+        otp: ["23.3.4.6"]
+        elixir: ["1.10.4", "1.11.4", "1.12.3", "1.13.4", "1.14.2"]
     steps:
       - uses: actions/checkout@v2
-      - uses: erlef/setup-elixir@v1
+      - uses: erlef/setup-beam@v1
         with:
           otp-version: ${{matrix.otp}}
           elixir-version: ${{matrix.elixir}}
+          version-type: strict
       - uses: rrainn/[email protected]
         with:
-            port: 8000
-            cors: '*'
-      - run: mix deps.update --all
-        if: matrix.elixir == '1.13.2'
+          port: 8000
+          cors: "*"
       - run: mix deps.get
       - run: mix compile
       - run: mix format --check-formatted
-        if: matrix.elixir == '1.13.2' # Only check formatting with the latest verison
+        if: matrix.elixir == '1.14.2' # Only check formatting with the latest verison
       - run: mix dialyzer
-        if: matrix.elixir == '1.13.2'
+        if: matrix.elixir == '1.14.2'
       - run: mix test

.tool-versions

@@ -0,0 +1 @@
+elixir 1.14.2

CHANGELOG.md

@@ -1,4 +1,25 @@
+v2.4.0
+- Increase minimum elixir version to 1.10
+- Add `error_parser` field to operations. This may be optionally populated by services which
+need to do service-specific error handling prior to falling back to the default ExAws handling.
+
+V2.3.4
+- Fix crash in authentication for regions without SSO service (#894)
+- Service endpoint updates
+
+v2.3.3
+- Imporve resiliency/recovery when authentication token queries fail
+- Use `default` profile for `:aws_cli` config when `AWS_PROFILE is undefined
+- Include service in telemetry events
+- Fix crash generating auth headers for request with empty path
+
+v2.3.2
+- Fix type for IMDSv2 header
+- Make IMDSv2 optional, with fallback to v1
+- Fix spec for `Config.new/2`
+
 v2.3.1
+- Support container task role credentials in token provider
 - Fix issue with ECS instance meta data introduced in 2.3.0
 - Fix typespec on `ExAws.Request.HttpClient.request/5`
 

README.md

@@ -13,13 +13,6 @@ A flexible easy to use set of AWS APIs.
 
 Available Services: https://github.com/ex-aws?q=service&type=&language=
 
-## Un-Deprecation Notice
-
-ExAws is now actively maintained again :). It's going to take me a while to work through
-all the outstanding issues and PRs, so please bear with me.
-
-- Bernard
-
 ## Getting Started
 
 ExAws v2.0 breaks out every service into its own package. To use the S3

config/config.exs

@@ -1,4 +1,4 @@
-use Mix.Config
+import Config
 
 config :logger, :console,
   level: :debug,

config/dev.exs

@@ -1,4 +1,4 @@
-use Mix.Config
+import Config
 
 config :ex_aws,
   debug_requests: true,

config/prod.exs

@@ -1 +1 @@
-use Mix.Config
+import Config

config/test.exs

@@ -1,4 +1,4 @@
-use Mix.Config
+import Config
 
 config :logger, level: :warn
 

lib/ex_aws.ex

@@ -62,6 +62,7 @@ defmodule ExAws do
 
     * `:result` - the request result: `:ok` or `:error`
     * `:attempt` - the attempt number
+    * `:service` - the AWS service
     * `:options` - extra options given to the repo operation under
       `:telemetry_options`
 

lib/ex_aws/config.ex

@@ -36,7 +36,7 @@ defmodule ExAws.Config do
     4. Finally, any configuration overrides are merged in
 
   """
-  @spec new(atom, keyword) :: %{}
+  @spec new(atom, keyword) :: map()
   def new(service, opts \\ []) do
     overrides = Map.new(opts)
 

lib/ex_aws/config/auth_cache.ex

@@ -62,8 +62,6 @@ defmodule ExAws.Config.AuthCache do
   end
 
   def refresh_awscli_config(profile, expiration, ets) do
-    Process.send_after(self(), {:refresh_awscli_config, profile, expiration}, expiration)
-
     auth = ExAws.Config.awscli_auth_credentials(profile)
 
     auth =
@@ -72,14 +70,32 @@ defmodule ExAws.Config.AuthCache do
           auth
 
         adapter ->
-          adapter.adapt_auth_config(auth, profile, expiration)
+          attempt_credentials_refresh(adapter, auth, profile, expiration)
       end
 
+    Process.send_after(self(), {:refresh_awscli_config, profile, expiration}, expiration)
     :ets.insert(ets, {{:awscli, profile}, auth})
 
     auth
   end
 
+  defp attempt_credentials_refresh(adapter, auth, profile, expiration, retries \\ 6) do
+    case adapter.adapt_auth_config(auth, profile, expiration) do
+      {:error, error} when retries == 1 ->
+        Process.send_after(self(), {:refresh_awscli_config, profile, expiration}, expiration)
+
+        raise "Could't get credentials from auth adapter after 6 retries, last error was #{inspect(error)}"
+
+      {:error, _error} ->
+        Process.sleep(:rand.uniform(5_000))
+        attempt_credentials_refresh(adapter, auth, profile, expiration, retries - 1)
+
+      # Always store a map on AuthCache
+      auth when is_map(auth) ->
+        auth
+    end
+  end
+
   defp refresh_auth_if_required([], config) do
     GenServer.call(__MODULE__, {:refresh_auth, config}, 30_000)
   end

lib/ex_aws/config/defaults.ex

@@ -14,6 +14,7 @@ defmodule ExAws.Config.Defaults do
       base_backoff_in_ms: 10,
       max_backoff_in_ms: 10_000
     ],
+    require_imds_v2: false,
     normalize_path: true
   }
 

lib/ex_aws/credentials_ini/file.ex

@@ -12,19 +12,28 @@ if Code.ensure_loaded?(ConfigParser) do
     def security_credentials(profile_name) do
       config_credentials = profile_from_config(profile_name)
       shared_credentials = profile_from_shared_credentials(profile_name)
-      config = ExAws.Config.http_config(:sso)
 
       case config_credentials do
         %{
           sso_start_url: sso_start_url,
           sso_account_id: sso_account_id,
           sso_role_name: sso_role_name
         } ->
+          config = ExAws.Config.http_config(:sso)
+
           case get_sso_role_credentials(sso_start_url, sso_account_id, sso_role_name, config) do
             {:ok, sso_creds} -> {:ok, Map.merge(sso_creds, shared_credentials)}
             {:error, _} = err -> err
           end
 
+        %{credential_process: credential_process} ->
+          config = ExAws.Config.http_config(:sso)
+
+          case get_credentials_from_process(credential_process, config) do
+            {:ok, credentials} -> {:ok, Map.merge(credentials, shared_credentials)}
+            {:error, _} = err -> err
+          end
+
         _ ->
           {:ok, Map.merge(config_credentials, shared_credentials)}
       end
@@ -67,8 +76,7 @@ if Code.ensure_loaded?(ConfigParser) do
     end
 
     defp check_sso_expiration(expires_at_str) do
-      with {_, {:ok, expires_at, _}} <- {:timestamp, DateTime.from_iso8601(expires_at_str)},
-           {_, :gt} <- {:expires, DateTime.compare(expires_at, DateTime.utc_now())} do
+      with {:ok, _} <- check_expiration(expires_at_str) do
         :ok
       else
         {:timestamp, {:error, err}} ->
@@ -127,6 +135,89 @@ if Code.ensure_loaded?(ConfigParser) do
       end
     end
 
+    defp get_credentials_from_process(credential_process, config) do
+      with {_, {:ok, process_result}} <-
+             {:process, execute_process(credential_process)},
+           {_, {:ok, %{"Version" => 1} = result}} <-
+             {:decode, config[:json_codec].decode(process_result)},
+           {_, {:ok, expiration}} <-
+             {:expiration, check_credentials_expiration(result)},
+           {_, {:ok, reformatted_creds}} <-
+             {:rename, format_result(result, expiration)} do
+        {:ok, reformatted_creds}
+      else
+        {:process, {:error, error}} -> {:error, "Could not execute process: #{error}"}
+        {:decode, _} -> {:error, "Credentials process results contains invalid json"}
+        {:expiration, error} -> error
+        {:rename, error} -> error
+      end
+    end
+
+    defp execute_process(credential_process) do
+      with [command | args] <- String.split(credential_process),
+           {result, 0} <- System.cmd(command, args, stderr_to_stdout: true) do
+        {:ok, result}
+      else
+        [] -> {:error, "Could not read command from config file : #{credential_process}"}
+        {error, exit_code} -> {:error, "Exit code : #{exit_code} - #{error}"}
+      end
+    end
+
+    defp format_result(result, nil) do
+      with {_, access_key} when not is_nil(access_key) <-
+             {:accessKey, Map.get(result, "AccessKeyId")},
+           {_, secret_access_key} when not is_nil(secret_access_key) <-
+             {:secretAccess, Map.get(result, "SecretAccessKey")} do
+        {:ok,
+         %{
+           access_key_id: access_key,
+           secret_access_key: secret_access_key
+         }}
+      else
+        {missing, _} -> {:error, "#{missing} is missing from credentials process response"}
+      end
+    end
+
+    defp format_result(result, expiration) do
+      with {_, access_key} when not is_nil(access_key) <-
+             {:accessKey, Map.get(result, "AccessKeyId")},
+           {_, secret_access_key} when not is_nil(secret_access_key) <-
+             {:secretAccess, Map.get(result, "SecretAccessKey")},
+           {_, session_token} when not is_nil(session_token) <-
+             {:sessionToken, Map.get(result, "SessionToken")} do
+        {:ok,
+         %{
+           access_key_id: access_key,
+           expiration: DateTime.to_unix(expiration),
+           secret_access_key: secret_access_key,
+           security_token: session_token
+         }}
+      else
+        {missing, _} -> {:error, "#{missing} is missing from credentials process response"}
+      end
+    end
+
+    defp check_credentials_expiration(%{"Expiration" => expiration_str}) do
+      with {:ok, expiration} <- check_expiration(expiration_str) do
+        {:ok, expiration}
+      else
+        {:timestamp, {:error, err}} ->
+          {:error, "Process returned invalid expiration format: #{err}"}
+
+        {:expires, _} ->
+          {:error, "Process returned expired credentials"}
+      end
+    end
+
+    defp check_credentials_expiration(_), do: {:ok, nil}
+
+    defp check_expiration(expiration_str) do
+      with {_, {:ok, expiration, _}} <- {:timestamp, DateTime.from_iso8601(expiration_str)},
+           {_, :gt} <- {:expires, DateTime.compare(expiration, DateTime.utc_now())} do
+        {:ok, expiration}
+      end
+    end
+
     def parse_ini_file({:ok, contents}, :system) do
       parse_ini_file({:ok, contents}, profile_name_from_env())
     end
@@ -180,19 +271,22 @@ if Code.ensure_loaded?(ConfigParser) do
     end
 
     defp profile_from_config(profile_name) do
-      section =
-        case profile_name do
-          :system -> "profile #{profile_name_from_env()}"
-          "default" -> "default"
-          other -> "profile #{other}"
-        end
+      section = profile_from_name(profile_name)
 
       System.user_home()
       |> Path.join(".aws/config")
       |> File.read()
       |> parse_ini_file(section)
     end
 
+    defp profile_from_name(:system) do
+      profile_name_from_env()
+      |> profile_from_name()
+    end
+
+    defp profile_from_name("default"), do: "default"
+    defp profile_from_name(other), do: "profile #{other}"
+
     defp profile_name_from_env() do
       System.get_env("AWS_PROFILE") || "default"
     end