Merge remote-tracking branch 'upstream/main'

# Conflicts:
#	lib/ex_aws/config.ex
#	mix.exs
#	mix.lock

.github/workflows/on-push.yml

@@ -7,25 +7,72 @@ env:
 jobs:
   build:
     runs-on: ubuntu-20.04
+    # See https://hexdocs.pm/elixir/compatibility-and-deprecations.html#compatibility-between-elixir-and-erlang-otp
     strategy:
       matrix:
-        otp: ["23.3.4.6"]
-        elixir: ["1.10.4", "1.11.4", "1.12.3", "1.13.4", "1.14.2"]
+        include:
+          - pair:
+              otp: 26.x
+              elixir: 1.16.x
+              lint: lint
+          - pair:
+              otp: 26.x
+              elixir: 1.15.x
+          - pair:
+              otp: 25.x
+              elixir: 1.15.x
+          - pair:
+              otp: 25.x
+              elixir: 1.14.x
+          - pair:
+              otp: 25.x
+              elixir: 1.13.x
+          - pair:
+              otp: 24.x
+              elixir: 1.15.x
+          - pair:
+              otp: 24.x
+              elixir: 1.14.x
+          - pair:
+              otp: 24.x
+              elixir: 1.13.x
+          - pair:
+              otp: 24.x
+              elixir: 1.12.x
+          - pair:
+              otp: 23.x
+              elixir: 1.14.x
+          - pair:
+              otp: 23.x
+              elixir: 1.13.x
+          - pair:
+              otp: 23.x
+              elixir: 1.12.x
+
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - uses: erlef/setup-beam@v1
         with:
-          otp-version: ${{matrix.otp}}
-          elixir-version: ${{matrix.elixir}}
-          version-type: strict
-      - uses: rrainn/[email protected]
+          otp-version: ${{matrix.pair.otp}}
+          elixir-version: ${{matrix.pair.elixir}}
+      - uses: rrainn/[email protected]
         with:
           port: 8000
           cors: "*"
+      - uses: actions/cache@v3
+        with:
+          path: |
+            deps
+            _build
+          key: ${{ runner.os }}-mix-${{matrix.pair.otp}}-${{matrix.pair.elixir}}-${{ hashFiles('**/mix.lock') }}
+          restore-keys: |
+            ${{ runner.os }}-mix-${{matrix.pair.otp}}-${{matrix.pair.elixir}}-
       - run: mix deps.get
       - run: mix compile
+      - run: mix deps.unlock --check-unused
+        if: ${{matrix.lint}} # Only check formatting with the latest verison
       - run: mix format --check-formatted
-        if: matrix.elixir == '1.14.2' # Only check formatting with the latest verison
+        if: ${{matrix.lint}}
       - run: mix dialyzer
-        if: matrix.elixir == '1.14.2'
+        if: ${{matrix.lint}}
       - run: mix test

.tool-versions

@@ -1 +1,2 @@
-elixir 1.14.2
+elixir 1.16.3-otp-26
+erlang 26.2.5

CHANGELOG.md

@@ -1,3 +1,53 @@
+v2.5.4 (2024-06-04)
+- Endpoint and region updates
+- Add support for Elixir 1.16
+- Drop support for Elixir 1.11
+- Add support for `sso_session` in `.aws/config`
+- Add support for EventBridge Pipes service
+- Add support for Pinpoint Mobiletargeting service
+- Add docs for running DynamoDB locally
+- Allow iodata body in s3 requests
+
+v2.5.3 (2024-03-20)
+- Add mappings for Chime
+
+v2.5.2 (2024-03-19)
+- Endpoint updates
+
+v2.5.1 (2024-01-11)
+- Endpoint updates
+- Documentation updates
+- Fix request error handling when error type is a proplist
+- Fixes to ensure Req and Finch clients return the same success fields as hackney
+- Add basic support for Personalise service
+- Add basic support for Bedrock service
+
+v2.5.0
+- [Breaking] Bump minimum elixir version to 1.11
+- Replace retired HTTPotion with Req as default web client
+- Endpoint updates
+
+v2.4.4
+- Endpoint updates
+- Add new endpoints for Chime SDK Media Pipelines API
+
+v2.4.3
+- Switch default behaviour of credential refreshing so that it must be explicitly enabled with
+`refreshable: true`. Having it as the default behaviour was causing breaking issues with ignoring 
+overridesin places they shouldn't have been.
+- Update endpoints
+
+v2.4.2
+- Add name, request, and response data to telemetry
+- Force refresh of auth config during long-running streaming operations to avoid failed auth
+- Update endpoints
+- Update docs
+
+v2.4.1
+- Add support for credentials_process in AWS credentials config
+- Service endpoint updates
+- Switch to `Config` from `Mix.Config`
+
 v2.4.0
 - Increase minimum elixir version to 1.10
 - Add `error_parser` field to operations. This may be optionally populated by services which

PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,5 @@
+Before opening a PR, please make sure you have:
+
+* Run `mix format` using a recent version of Elixir
+* Run `mix dialyzer` to make sure the typing is correct
+* Run `mix test` to ensure no tests have broken (also please make sure you've added tests for your particular change, where appropriate).

README.md

@@ -260,7 +260,13 @@ config :ex_aws, :retries,
 ## Testing
 
 If you want to run `mix test`, you'll need to have a local `dynamodb` running
-on port 8000.  See [Setting up DynamoDB Local](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/DynamoDBLocal.DownloadingAndRunning.html).
+on port 8000:
+
+```console
+docker run --rm -d -p 8000:8000 amazon/dynamodb-local -jar DynamoDBLocal.jar -port 8000
+```
+
+For more info please see [Setting up DynamoDB Local](https://docs.aws.amazon.com/amazondynamodb/latest/developerguide/DynamoDBLocal.DownloadingAndRunning.html).
 
 The redirect test will intentionally cause a warning to be issued.
 

config/test.exs

@@ -1,11 +1,11 @@
 import Config
 
-config :logger, level: :warn
+config :logger, level: :warning
 
 config :ex_aws,
   json_codec: Test.JSONCodec,
-  access_key_id: ["test_key_id"],
-  secret_access_key: ["secret_access_key"]
+  access_key_id: "testkeyid",
+  secret_access_key: "secretaccesskey"
 
 config :ex_aws, :kinesis,
   scheme: "https://",

lib/ex_aws/config.ex

@@ -2,9 +2,18 @@ defmodule ExAws.Config do
   @moduledoc """
   Generates the configuration for a service.
 
-  It starts with the defaults for a given environment
-  and then merges in the common config from the ex_aws config root,
-  and then finally any config specified for the particular service.
+  It starts with the defaults for a given environment and then merges in the
+  common config from the ex_aws config root, and then finally any config
+  specified for the particular service.
+
+  ## Refreshable fields
+
+  Some fields are marked as refreshable. These fields will be fetched through
+  the auth cache even if they are passed in as overrides. This is so stale
+  credentials aren't used, for example, with long running streams.
+
+  This behaviour must be explicitly enabled by passing `refreshable: true` as an option
+  to Config.new/2
   """
 
   # TODO: Add proper documentation?
@@ -25,6 +34,25 @@ defmodule ExAws.Config do
     :telemetry_options
   ]
 
+  @instance_role_config [
+    :access_key_id,
+    :secret_access_key,
+    :security_token,
+    :s3_auth_version
+  ]
+
+  @awscli_config [
+    :source_profile,
+    :role_arn,
+    :access_key_id,
+    :secret_access_key,
+    :region,
+    :security_token,
+    :role_session_name,
+    :external_id,
+    :s3_auth_version
+  ]
+
   @type t :: %{} | Keyword.t()
 
   @doc """
@@ -42,8 +70,8 @@ defmodule ExAws.Config do
 
     service
     |> build_base(overrides)
-    |> retrieve_runtime_config
-    |> parse_host_for_region
+    |> retrieve_runtime_config()
+    |> parse_host_for_region()
   end
 
   @doc """
@@ -70,10 +98,48 @@ defmodule ExAws.Config do
 
     defaults = ExAws.Config.Defaults.get(service, region)
 
-    defaults
-    |> Map.merge(common_config)
-    |> Map.merge(service_config)
-    |> Map.merge(overrides)
+    config =
+      defaults
+      |> Map.merge(common_config)
+      |> Map.merge(service_config)
+      |> add_refreshable_metadata(overrides)
+
+    # (Maybe) do not allow overrides for refreshable config.
+    overrides =
+      if refreshable = config[:refreshable] do
+        Enum.reduce(refreshable, overrides, fn
+          :awscli, overrides -> Map.drop(overrides, @awscli_config)
+          :instance_role, overrides -> Map.drop(overrides, @instance_role_config)
+        end)
+      else
+        overrides
+      end
+
+    Map.merge(config, overrides)
+  end
+
+  # :awscli and :instance_role both read creds from ExAws.Config.AuthCache which
+  # is "refreshable". This is useful for long running streams where the creds can
+  # change while the stream is still running.
+  defp add_refreshable_metadata(config, %{refreshable: true}) do
+    refreshable =
+      Enum.flat_map(config, fn {_k, v} -> List.wrap(v) end)
+      |> Enum.reduce([], fn
+        {:awscli, _, _}, acc -> [:awscli | acc]
+        :instance_role, acc -> [:instance_role | acc]
+        _, acc -> acc
+      end)
+      |> Enum.uniq()
+
+    if refreshable != [] do
+      Map.put(config, :refreshable, refreshable)
+    else
+      config
+    end
+  end
+
+  defp add_refreshable_metadata(config, _overrides) do
+    config
   end
 
   def retrieve_runtime_config(config) do
@@ -96,6 +162,9 @@ defmodule ExAws.Config do
       {:headers, headers}, config ->
         Map.put(config, :headers, headers)
 
+      {:refreshable, refreshable}, config ->
+        Map.put(config, :refreshable, refreshable)
+
       {k, v}, config ->
         case retrieve_runtime_value(v, config) do
           %{} = result -> Map.merge(config, result)
@@ -111,23 +180,13 @@ defmodule ExAws.Config do
   def retrieve_runtime_value(:instance_role, config) do
     config
     |> ExAws.Config.AuthCache.get()
-    |> Map.take([:access_key_id, :secret_access_key, :s3_auth_version, :security_token])
+    |> Map.take(@instance_role_config)
     |> valid_map_or_nil
   end
 
   def retrieve_runtime_value({:awscli, profile, expiration}, _) do
     ExAws.Config.AuthCache.get(profile, expiration * 1000)
-    |> Map.take([
-      :source_profile,
-      :role_arn,
-      :access_key_id,
-      :secret_access_key,
-      :s3_auth_version,
-      :region,
-      :security_token,
-      :role_session_name,
-      :external_id
-    ])
+    |> Map.take(@awscli_config)
     |> valid_map_or_nil
   end
 

lib/ex_aws/config/defaults.ex

@@ -38,6 +38,16 @@ defmodule ExAws.Config.Defaults do
     |> Map.merge(defaults(:lex))
   end
 
+  def defaults(:"personalize-runtime") do
+    %{service_override: :personalize}
+    |> Map.merge(defaults(:personalize))
+  end
+
+  def defaults(:"personalize-events") do
+    %{service_override: :personalize}
+    |> Map.merge(defaults(:personalize))
+  end
+
   def defaults(:sagemaker_runtime) do
     %{service_override: :sagemaker}
     |> Map.merge(defaults(:sagemaker))
@@ -73,6 +83,17 @@ defmodule ExAws.Config.Defaults do
     |> Map.merge(defaults(:geo))
   end
 
+  def defaults(chime_service)
+      when chime_service in [
+             :"chime-sdk-media-pipelines",
+             :"chime-sdk-identity",
+             :"chime-sdk-meetings",
+             :"chime-sdk-voice"
+           ] do
+    %{service_override: :chime}
+    |> Map.merge(defaults(:chime))
+  end
+
   def defaults(_) do
     Map.merge(
       %{
@@ -91,7 +112,7 @@ defmodule ExAws.Config.Defaults do
   end
 
   @partitions [
-    {~r/^(us|eu|af|ap|sa|ca|me)\-\w+\-\d+$/, "aws"},
+    {~r/^(us|eu|af|ap|sa|ca|me)\-\w+-\d?-?\w+$/, "aws"},
     {~r/^cn\-\w+\-\d+$/, "aws-cn"},
     {~r/^us\-gov\-\w+\-\d+$/, "aws-us-gov"}
   ]
@@ -140,7 +161,6 @@ defmodule ExAws.Config.Defaults do
   defp do_host(partition, service_slug, region) do
     partition = @partition_data |> Map.fetch!(partition)
     partition_name = partition["partition"]
-
     service = service_map(service_slug)
 
     partition