Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Smithy Identity auth refactor for Rest XML S3 client #3262

Merged
merged 37 commits into from
Feb 12, 2025
Merged
Show file tree
Hide file tree
Changes from 21 commits
Commits
Show all changes
37 commits
Select commit Hold shift + click to select a range
f0bf448
s3 support
sbera87 Jan 10, 2025
1b5d497
more changes for s3
sbera87 Jan 13, 2025
e6638d3
codegen changes
sbera87 Jan 16, 2025
b8676c7
codegen changes 2
sbera87 Jan 16, 2025
1aec690
first version s3 client
sbera87 Jan 16, 2025
39e841f
fix s3 legacy constructors
sbera87 Jan 17, 2025
ffa61d3
s3 express signer
sbera87 Jan 18, 2025
fa017e4
s3 express related changes
sbera87 Jan 19, 2025
30cba54
s3 identity resolver changes
sbera87 Jan 20, 2025
df0f1f8
s3 resolver + constructor updates
sbera87 Jan 20, 2025
c2ae2e3
s3 client operations updates
sbera87 Jan 20, 2025
3a8f644
merged from main
sbera87 Jan 20, 2025
d421218
pending merge conflict
sbera87 Jan 21, 2025
1aef2a8
# This is a combination of 21 commits.
sbera87 Jan 22, 2025
98e45f4
Codegen updates
sbera87 Jan 31, 2025
64d8d51
multi auth support + sigv4a fixes
sbera87 Jan 28, 2025
35474c3
sigv4a working
sbera87 Jan 30, 2025
6032dc0
merge changes from main
sbera87 Jan 31, 2025
5af3458
merge from main
sbera87 Jan 31, 2025
733c1e8
merge from main
sbera87 Jan 31, 2025
e531edc
Warning fixes
sbera87 Jan 31, 2025
bc8503a
remove reference member for proper move constructor functionality
sbera87 Feb 4, 2025
27a5e8b
structural changes for PR comments
sbera87 Feb 5, 2025
473c27c
cleanup
sbera87 Feb 10, 2025
abd92a8
relocate files
sbera87 Feb 10, 2025
40732db
Backward compatibility for S3 express endpoint provider
sbera87 Feb 10, 2025
20f79cc
resolve authscheme structural change
sbera87 Feb 11, 2025
1e5146c
S3 express signer support for old and new interfaces
sbera87 Feb 11, 2025
3f154ac
s3 copy constructor fix
sbera87 Feb 11, 2025
0940beb
copy constructor fix + test
sbera87 Feb 11, 2025
9e5302f
include header for test
sbera87 Feb 11, 2025
a08a906
fix for linker errors
sbera87 Feb 11, 2025
b6145a0
s3 express resolve endpoint updates
sbera87 Feb 11, 2025
cc4b2df
address identity supplier backward compatibility
sbera87 Feb 11, 2025
290750f
updates in s3express signer
sbera87 Feb 11, 2025
72ca83b
fixes for pipeline
sbera87 Feb 11, 2025
2eeab3f
dry run fixes
sbera87 Feb 11, 2025
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 24 additions & 8 deletions generated/src/aws-cpp-sdk-s3/include/aws/s3/S3Client.h
Original file line number Diff line number Diff line change
Expand Up @@ -4,14 +4,22 @@
*/

#pragma once

sbiscigl marked this conversation as resolved.
Show resolved Hide resolved
#include <aws/s3/S3_EXPORTS.h>
#include <aws/core/client/ClientConfiguration.h>
#include <aws/core/client/AWSClient.h>
#include <aws/core/client/AWSClientAsyncCRTP.h>
#include <aws/core/auth/AWSAuthSigner.h>
#include <aws/core/utils/xml/XmlSerializer.h>
#include <aws/core/utils/DNS.h>
#include <aws/s3/S3ServiceClientModel.h>
#include <smithy/client/AwsSmithyClient.h>
#include <smithy/identity/auth/built-in/SigV4MultiAuthResolver.h>
#include <smithy/identity/auth/built-in/SigV4AuthScheme.h>
#include <smithy/identity/auth/built-in/SigV4aAuthScheme.h>
#include <smithy/identity/auth/built-in/S3ExpressSigV4AuthScheme.h>
#include <smithy/client/serializer/XmlOutcomeSerializer.h>
#include <aws/core/utils/DNS.h>

#include <aws/s3/S3ErrorMarshaller.h>



// TODO: temporary fix for naming conflicts on Windows.
#ifdef _WIN32
Expand All @@ -32,19 +40,28 @@ namespace Aws
static const char SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY[] = "x-amz-server-side-encryption-customer-key";
static const char SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5[] = "x-amz-server-side-encryption-customer-key-MD5";
} // SS3Headers
AWS_S3_API extern const char SERVICE_NAME[];

//max expiration for presigned urls in s3 is 7 days.
static const unsigned MAX_EXPIRATION_SECONDS = 7 * 24 * 60 * 60;

/**
* <p/>
*/
class AWS_S3_API S3Client : public Aws::Client::AWSXMLClient, public Aws::Client::ClientWithAsyncTemplateMethods<S3Client>
class AWS_S3_API S3Client : public smithy::client::AwsSmithyClientT<Aws::S3::SERVICE_NAME,
Aws::S3::S3ClientConfiguration,
smithy::SigV4MultiAuthSchemeResolver<>,
Aws::Crt::Variant<smithy::SigV4AuthScheme,smithy::S3ExpressSigV4AuthScheme,smithy::SigV4aAuthScheme>,
S3EndpointProviderBase,
smithy::client::XmlOutcomeSerializer,
smithy::client::XmlOutcome,
Aws::Client::S3ErrorMarshaller>,
Aws::Client::ClientWithAsyncTemplateMethods<S3Client>
{
public:
typedef Aws::Client::AWSXMLClient BASECLASS;
static const char* GetServiceName();
static const char* GetAllocationTag();
inline const char* GetServiceClientName() const override { return "S3"; }

typedef S3ClientConfiguration ClientConfigurationType;
typedef S3EndpointProvider EndpointProviderType;
Expand Down Expand Up @@ -6973,8 +6990,7 @@ namespace Aws
private:
friend class Aws::Client::ClientWithAsyncTemplateMethods<S3Client>;
void init(const S3ClientConfiguration& clientConfiguration);
S3ClientConfiguration m_clientConfiguration;
std::shared_ptr<S3EndpointProviderBase> m_endpointProvider;

};

} // namespace S3
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@
#pragma once

#include <aws/s3/S3_EXPORTS.h>
#include <aws/s3/S3ExpressIdentityProvider.h>
#include <aws/core/client/GenericClientConfiguration.h>
#include <aws/core/auth/signer/AWSAuthV4Signer.h>

Expand Down Expand Up @@ -58,10 +57,6 @@ namespace Aws
bool useArnRegion = false;
Client::AWSAuthV4Signer::PayloadSigningPolicy payloadSigningPolicy = Client::AWSAuthV4Signer::PayloadSigningPolicy::RequestDependent;
bool disableS3ExpressAuth = false;
using IdentityProviderSupplier = std::function<std::shared_ptr<S3ExpressIdentityProvider> (const S3Client &)>;
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

No longer needed here as the provider is passed in the smithy constructor

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

you cant remove this because it is part of the public interface i.e.

auto main() -> int {
  SDKOptions options{};
  InitAPI(options);
  {
    S3ClientConfiguration configuration{};
    configuration.identityProviderSupplier = 
      [](const S3Client &client) -> std::shared_ptr<S3ExpressIdentityProvider>  {
         //...
      }
    S3Client client{configuration};
  }
  ShutdownAPI(options);
  return 0;
}

that is valid code and our customers have been using that to override the identity provider.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ok, will restore this and reuse it internally

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

on the latest commit the above code still break for for backwards compatibility. reproduction in full:

#include <aws/core/Aws.h>
#include <aws/s3/S3Client.h>

using namespace Aws;
using namespace Aws::S3;

auto main() -> int {
    SDKOptions options{};
    InitAPI(options);
    {
        S3ClientConfiguration configuration{};
        configuration.identityProviderSupplier =
          [](const S3Client &client) -> std::shared_ptr<S3ExpressIdentityProvider> {
              return Aws::MakeShared<DefaultS3ExpressIdentityProvider>("log_tag", client);
          };
        S3Client client{configuration};
    }
    ShutdownAPI(options);
    return 0;
}

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I have to support both the old and new one in this case too, even though this supplier can't be used in the auth schemes in smithy. Added in next commit and added this compilation test as well.

IdentityProviderSupplier identityProviderSupplier = [](const S3Client &client) -> std::shared_ptr<S3ExpressIdentityProvider> {
return Aws::MakeShared<DefaultS3ExpressIdentityProvider>("S3ClientConfiguration", client);
};
private:
void LoadS3SpecificConfig(const Aws::String& profileName);
};
Expand Down
71 changes: 0 additions & 71 deletions generated/src/aws-cpp-sdk-s3/include/aws/s3/S3ExpressIdentity.h

This file was deleted.

This file was deleted.

Original file line number Diff line number Diff line change
@@ -0,0 +1,96 @@
/**
* Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
* SPDX-License-Identifier: Apache-2.0.
*/

#pragma once

#include <aws/core/utils/memory/stl/AWSSet.h>
#include <aws/core/utils/memory/stl/AWSString.h>
#include <aws/core/utils/ConcurrentCache.h>
#include <smithy/identity/resolver/AwsIdentityResolverBase.h>
#include <smithy/identity/identity/AwsCredentialIdentity.h>
#include <thread>
#include <condition_variable>

namespace Aws {
namespace Http {
struct ServiceSpecificParameters;
}

namespace S3 {
class S3Client;
class S3ExpressIdentityResolver : public smithy::IdentityResolverBase<smithy::AwsCredentialIdentityBase> {
public:
explicit S3ExpressIdentityResolver(const S3Client& s3Client);
explicit S3ExpressIdentityResolver(const S3Client& s3Client, const std::shared_ptr<Aws::Auth::AWSCredentialsProvider> credentialProvider);
ResolveIdentityFutureOutcome getIdentity(
const IdentityProperties& identityProperties,
const AdditionalParameters& additionalParameters) override;
virtual smithy::AwsCredentialIdentity GetS3ExpressIdentity(const std::shared_ptr<Aws::Http::ServiceSpecificParameters> &serviceSpecificParameters) = 0;
smithy::AwsCredentialIdentity GetCredentialsFromBucket(const Aws::String& bucketName) const;

private:
const Aws::S3::S3Client& m_s3Client;
mutable std::mutex m_bucketNameMapMutex;
Aws::Map<Aws::String, std::shared_ptr<std::mutex>> m_bucketNameMutex;

protected:
std::shared_ptr<std::mutex> GetMutexForBucketName(const Aws::String& bucketName);
mutable std::shared_ptr<Aws::Auth::AWSCredentialsProvider> m_credsProvider;
};

class DefaultS3ExpressIdentityResolver : public S3ExpressIdentityResolver {
public:
explicit DefaultS3ExpressIdentityResolver(const S3Client& s3Client);
explicit DefaultS3ExpressIdentityResolver(
const S3Client& s3Client,
std::shared_ptr<Utils::ConcurrentCache<Aws::String, smithy::AwsCredentialIdentity>> credentialsCache);
explicit DefaultS3ExpressIdentityResolver(const S3Client& s3Client, const std::shared_ptr<Aws::Auth::AWSCredentialsProvider> credentialProvider);
DefaultS3ExpressIdentityResolver(const DefaultS3ExpressIdentityResolver& other) = delete;
DefaultS3ExpressIdentityResolver(DefaultS3ExpressIdentityResolver&& other) noexcept = delete;
DefaultS3ExpressIdentityResolver& operator=(const DefaultS3ExpressIdentityResolver& other) = delete;
DefaultS3ExpressIdentityResolver& operator=(DefaultS3ExpressIdentityResolver&& other) noexcept = delete;
virtual ~DefaultS3ExpressIdentityResolver() override = default;
smithy::AwsCredentialIdentity GetS3ExpressIdentity(const std::shared_ptr<Aws::Http::ServiceSpecificParameters> &serviceSpecificParameters) override;

private:
mutable std::shared_ptr<Aws::Utils::ConcurrentCache<Aws::String, smithy::AwsCredentialIdentity>> m_credentialsCache;
};

class DefaultAsyncS3ExpressIdentityResolver : public S3ExpressIdentityResolver {
public:
explicit DefaultAsyncS3ExpressIdentityResolver(
const S3Client& s3Client,
std::chrono::minutes refreshPeriod = std::chrono::minutes(1));

explicit DefaultAsyncS3ExpressIdentityResolver(
const S3Client& s3Client,
std::shared_ptr<Utils::ConcurrentCache<Aws::String, smithy::AwsCredentialIdentity>> credentialsCache,
std::chrono::minutes refreshPeriod = std::chrono::minutes(1));

DefaultAsyncS3ExpressIdentityResolver(const DefaultAsyncS3ExpressIdentityResolver& other) = delete;
DefaultAsyncS3ExpressIdentityResolver(DefaultAsyncS3ExpressIdentityResolver&& other) noexcept = delete;
DefaultAsyncS3ExpressIdentityResolver& operator=(
const DefaultAsyncS3ExpressIdentityResolver& other) = delete;
DefaultAsyncS3ExpressIdentityResolver& operator=(DefaultAsyncS3ExpressIdentityResolver&& other) noexcept = delete;
virtual ~DefaultAsyncS3ExpressIdentityResolver() override;
smithy::AwsCredentialIdentity GetS3ExpressIdentity(const std::shared_ptr<Aws::Http::ServiceSpecificParameters> &serviceSpecificParameters) override;


private:
void refreshIdentities(std::chrono::minutes refreshPeriod);
void threadSafeKeyInsert(const Aws::String& key);
bool threadSafeKeyHas(const Aws::String& key);
void threadSafeKeyEmpty();

mutable std::shared_ptr<Aws::Utils::ConcurrentCache<Aws::String, smithy::AwsCredentialIdentity>> m_credentialsCache;
Aws::Set<Aws::String> m_keysUsed;
mutable std::mutex m_keysUsedMutex;
mutable bool m_shouldStopBackgroundRefresh;
Aws::UniquePtr<std::thread> m_backgroundRefreshThread;
mutable std::mutex m_shutDownMutex;
mutable std::condition_variable m_shutdownCondition;
};
}
}
Loading
Loading