diff --git a/src/aws-cpp-sdk-core/include/aws/core/client/ClientConfiguration.h b/src/aws-cpp-sdk-core/include/aws/core/client/ClientConfiguration.h index 360b39a1622..e198ee5c1a1 100644 --- a/src/aws-cpp-sdk-core/include/aws/core/client/ClientConfiguration.h +++ b/src/aws-cpp-sdk-core/include/aws/core/client/ClientConfiguration.h @@ -602,6 +602,18 @@ namespace Aws * First available auth scheme will be used for each operation. */ Aws::Vector authPreferences; + + /** + * Buffer size in bytes that will be used to content encode + * bodies using aws-chunked. Changing this is useful when you + * want to minimize memory use while uploading to S3. Size MUST + * be greater than 8KB otherwise S3 will reject the request. + * + * Defaults to 64KiB. + * + * https://docs.aws.amazon.com/AmazonS3/latest/API/sigv4-streaming.html + */ + size_t awsChunkedBufferSize = 64UL * 1024; }; /** diff --git a/src/aws-cpp-sdk-core/include/smithy/client/AwsSmithyClientBase.h b/src/aws-cpp-sdk-core/include/smithy/client/AwsSmithyClientBase.h index 5d39ad8d75a..09087f225cd 100644 --- a/src/aws-cpp-sdk-core/include/smithy/client/AwsSmithyClientBase.h +++ b/src/aws-cpp-sdk-core/include/smithy/client/AwsSmithyClientBase.h @@ -107,7 +107,8 @@ namespace client m_interceptors({ Aws::MakeShared("AwsSmithyClientBase", *m_clientConfig), Aws::MakeShared("AwsSmithyClientBase", - m_httpClient->IsDefaultAwsHttpClient() ? Aws::Client::HttpClientChunkedMode::DEFAULT : m_clientConfig->httpClientChunkedMode) + m_httpClient->IsDefaultAwsHttpClient() ? Aws::Client::HttpClientChunkedMode::DEFAULT : m_clientConfig->httpClientChunkedMode, + m_clientConfig->awsChunkedBufferSize) }) { diff --git a/src/aws-cpp-sdk-core/include/smithy/client/features/ChunkingInterceptor.h b/src/aws-cpp-sdk-core/include/smithy/client/features/ChunkingInterceptor.h index 0b377593fb8..e31c11231b9 100644 --- a/src/aws-cpp-sdk-core/include/smithy/client/features/ChunkingInterceptor.h +++ b/src/aws-cpp-sdk-core/include/smithy/client/features/ChunkingInterceptor.h @@ -152,10 +152,18 @@ class AwsChunkedIOStream : public Aws::IOStream { class ChunkingInterceptor : public smithy::interceptor::Interceptor { public: explicit ChunkingInterceptor(Aws::Client::HttpClientChunkedMode httpClientChunkedMode) - : m_httpClientChunkedMode(httpClientChunkedMode) {} + : m_httpClientChunkedMode(httpClientChunkedMode), m_bufferSize(AWS_DATA_BUFFER_SIZE) {} + explicit ChunkingInterceptor(Aws::Client::HttpClientChunkedMode httpClientChunkedMode, size_t bufferSize) + : m_httpClientChunkedMode(httpClientChunkedMode), m_bufferSize(bufferSize) {} ~ChunkingInterceptor() override = default; ModifyRequestOutcome ModifyBeforeSigning(smithy::interceptor::InterceptorContext& context) override { + if (m_bufferSize < 8 * 1024) { + return Aws::Client::AWSError{Aws::Client::CoreErrors::VALIDATION, + "ValidationErrorException", + "aws-chunked buffer must be over 8KiB to content encode", + false}; + } auto request = context.GetTransmitRequest(); if (!ShouldApplyChunking(request, context)) { @@ -188,6 +196,12 @@ class ChunkingInterceptor : public smithy::interceptor::Interceptor { } ModifyRequestOutcome ModifyBeforeTransmit(smithy::interceptor::InterceptorContext& context) override { + if (m_bufferSize < 8 * 1024) { + return Aws::Client::AWSError{Aws::Client::CoreErrors::VALIDATION, + "ValidationErrorException", + "aws-chunked buffer must be over 8KiB to content encode", + false}; + } auto request = context.GetTransmitRequest(); if (!ShouldApplyChunking(request, context)) { @@ -199,8 +213,7 @@ class ChunkingInterceptor : public smithy::interceptor::Interceptor { return request; } - auto chunkedBody = Aws::MakeShared( - ALLOCATION_TAG, request.get(), originalBody); + auto chunkedBody = Aws::MakeShared(ALLOCATION_TAG, request.get(), originalBody, m_bufferSize); request->AddContentBody(chunkedBody); return request; @@ -233,6 +246,7 @@ class ChunkingInterceptor : public smithy::interceptor::Interceptor { } Aws::Client::HttpClientChunkedMode m_httpClientChunkedMode; + size_t m_bufferSize; }; } // namespace features diff --git a/src/aws-cpp-sdk-core/source/client/AWSClient.cpp b/src/aws-cpp-sdk-core/source/client/AWSClient.cpp index 4dbc7895352..3a0f2ca52ae 100644 --- a/src/aws-cpp-sdk-core/source/client/AWSClient.cpp +++ b/src/aws-cpp-sdk-core/source/client/AWSClient.cpp @@ -141,7 +141,9 @@ AWSClient::AWSClient(const Aws::Client::ClientConfiguration& configuration, m_requestCompressionConfig(configuration.requestCompressionConfig), m_userAgentInterceptor{Aws::MakeShared(AWS_CLIENT_LOG_TAG, configuration, m_retryStrategy->GetStrategyName(), m_serviceName)}, m_interceptors{Aws::MakeShared(AWS_CLIENT_LOG_TAG), Aws::MakeShared(AWS_CLIENT_LOG_TAG, - m_httpClient->IsDefaultAwsHttpClient() ? Aws::Client::HttpClientChunkedMode::DEFAULT : configuration.httpClientChunkedMode), m_userAgentInterceptor} + m_httpClient->IsDefaultAwsHttpClient() ? Aws::Client::HttpClientChunkedMode::DEFAULT : configuration.httpClientChunkedMode, + configuration.awsChunkedBufferSize), + m_userAgentInterceptor} { } @@ -168,7 +170,9 @@ AWSClient::AWSClient(const Aws::Client::ClientConfiguration& configuration, m_requestCompressionConfig(configuration.requestCompressionConfig), m_userAgentInterceptor{Aws::MakeShared(AWS_CLIENT_LOG_TAG, configuration, m_retryStrategy->GetStrategyName(), m_serviceName)}, m_interceptors{Aws::MakeShared(AWS_CLIENT_LOG_TAG, configuration), Aws::MakeShared(AWS_CLIENT_LOG_TAG, - m_httpClient->IsDefaultAwsHttpClient() ? Aws::Client::HttpClientChunkedMode::DEFAULT : configuration.httpClientChunkedMode), m_userAgentInterceptor} + m_httpClient->IsDefaultAwsHttpClient() ? Aws::Client::HttpClientChunkedMode::DEFAULT : configuration.httpClientChunkedMode, + configuration.awsChunkedBufferSize), + m_userAgentInterceptor} { } diff --git a/src/aws-cpp-sdk-core/source/smithy/client/AwsSmithyClientBase.cpp b/src/aws-cpp-sdk-core/source/smithy/client/AwsSmithyClientBase.cpp index f185bfacc74..66b9484ee3c 100644 --- a/src/aws-cpp-sdk-core/source/smithy/client/AwsSmithyClientBase.cpp +++ b/src/aws-cpp-sdk-core/source/smithy/client/AwsSmithyClientBase.cpp @@ -107,7 +107,8 @@ void AwsSmithyClientBase::baseCopyAssign(const AwsSmithyClientBase& other, m_interceptors = Aws::Vector>{ Aws::MakeShared("AwsSmithyClientBase", *m_clientConfig), Aws::MakeShared("AwsSmithyClientBase", - m_httpClient->IsDefaultAwsHttpClient() ? Aws::Client::HttpClientChunkedMode::DEFAULT : m_clientConfig->httpClientChunkedMode) + m_httpClient->IsDefaultAwsHttpClient() ? Aws::Client::HttpClientChunkedMode::DEFAULT : m_clientConfig->httpClientChunkedMode, + m_clientConfig->awsChunkedBufferSize) }; baseCopyInit(); diff --git a/tests/aws-cpp-sdk-core-tests/utils/stream/ChunkingInterceptorTest.cpp b/tests/aws-cpp-sdk-core-tests/utils/stream/ChunkingInterceptorTest.cpp index 742219e3505..5173d2d85fc 100644 --- a/tests/aws-cpp-sdk-core-tests/utils/stream/ChunkingInterceptorTest.cpp +++ b/tests/aws-cpp-sdk-core-tests/utils/stream/ChunkingInterceptorTest.cpp @@ -159,4 +159,19 @@ TEST_F(ChunkingInterceptorTest, ShouldNotApplyChunkingForCustomHttpClient) { EXPECT_EQ(request, result.GetResult()); EXPECT_FALSE(request->HasHeader(Aws::Http::AWS_TRAILER_HEADER)); EXPECT_FALSE(request->HasHeader(Aws::Http::TRANSFER_ENCODING_HEADER)); -} \ No newline at end of file +} + +TEST_F(ChunkingInterceptorTest, ShouldFailWhenBufferIsTooShort) { + const Aws::Client::ClientConfiguration config; + ChunkingInterceptor interceptor(config.httpClientChunkedMode, 7UL * 1024); + + // Create interceptor context with a mock request + const MockRequest mockRequest; + smithy::interceptor::InterceptorContext context(mockRequest); + auto respnse = interceptor.ModifyBeforeSigning(context); + EXPECT_FALSE(respnse.IsSuccess()); + EXPECT_STREQ("aws-chunked buffer must be over 8KiB to content encode", respnse.GetError().GetMessage().c_str()); + respnse = interceptor.ModifyBeforeTransmit(context); + EXPECT_FALSE(respnse.IsSuccess()); + EXPECT_STREQ("aws-chunked buffer must be over 8KiB to content encode", respnse.GetError().GetMessage().c_str()); +} diff --git a/tests/aws-cpp-sdk-s3-integration-tests/BucketAndObjectOperationTest.cpp b/tests/aws-cpp-sdk-s3-integration-tests/BucketAndObjectOperationTest.cpp index eb493780695..54488f806c9 100644 --- a/tests/aws-cpp-sdk-s3-integration-tests/BucketAndObjectOperationTest.cpp +++ b/tests/aws-cpp-sdk-s3-integration-tests/BucketAndObjectOperationTest.cpp @@ -96,6 +96,7 @@ namespace static std::string BASE_CONTENT_ENCODING_BUCKET_NAME = "contentencoding"; static std::string BASE_CROSS_REGION_BUCKET_NAME = "crossregion"; static std::string BASE_ENDPOINT_OVERRIDE_BUCKET_NAME = "endpointoverride"; + static std::string BASE_STREAM_SIZE_BUCKET_NAME = "streamsize"; static const char* ALLOCATION_TAG = "BucketAndObjectOperationTest"; static const char* TEST_OBJ_KEY = "TestObjectKey"; static const char* TEST_NEWLINE_KEY = "TestNewlineKey"; @@ -143,6 +144,7 @@ namespace std::ref(BASE_CONTENT_ENCODING_BUCKET_NAME), std::ref(BASE_CROSS_REGION_BUCKET_NAME), std::ref(BASE_ENDPOINT_OVERRIDE_BUCKET_NAME), + std::ref(BASE_STREAM_SIZE_BUCKET_NAME) }; for (auto& testBucketName : TEST_BUCKETS) { @@ -2715,4 +2717,29 @@ namespace .WithKey(objectKey)); AWS_EXPECT_SUCCESS(getObjectResponse); } + + TEST_F(BucketAndObjectOperationTest, ShouldSuccessfullyUploadObjectForSmallerBufferSize) { + const String fullBucketName = CalculateBucketName(BASE_STREAM_SIZE_BUCKET_NAME.c_str()); + SCOPED_TRACE(Aws::String("FullBucketName ") + fullBucketName); + CreateBucketRequest createBucketRequest; + createBucketRequest.SetBucket(fullBucketName); + createBucketRequest.SetACL(BucketCannedACL::private_); + CreateBucketOutcome createBucketOutcome = CreateBucket(createBucketRequest); + AWS_ASSERT_SUCCESS(createBucketOutcome); + + S3ClientConfiguration configuration{}; + configuration.region = Aws::Region::US_EAST_1; + // Set aws-chunked buffer to 12KB + configuration.awsChunkedBufferSize = 1024 * 12; + const S3Client shortStreamClient{configuration}; + + auto request = PutObjectRequest().WithBucket(fullBucketName).WithKey("the-jack-bros"); + + // Create a 24KB body + const Aws::String body(24L * 1024, 'a'); + request.SetBody(Aws::MakeShared(ALLOCATION_TAG, body)); + + const auto response = shortStreamClient.PutObject(request); + AWS_EXPECT_SUCCESS(response); + } }