Merge branch 'checksum-callback' of github.com:awslabs/aws-c-s3 into …

…checksum-callback

.github/ISSUE_TEMPLATE/bug-report.yml

@@ -12,6 +12,14 @@ body:
       description: What is the problem? A clear and concise description of the bug.
     validations:
       required: true
+  - type: checkboxes
+    id: regression
+    attributes:
+      label: Regression Issue
+      description: What is a regression? If it worked in a previous version but doesn't in the latest version, it's considered a regression. In this case, please provide specific version number in the report.
+      options:
+        - label: Select this option if this issue appears to be a regression.
+          required: false
   - type: textarea
     id: expected
     attributes:

.github/workflows/ci.yml

@@ -5,8 +5,13 @@ on:
     branches-ignore:
       - 'main'
 
+# cancel in-progress builds after a new commit
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
 env:
-  BUILDER_VERSION: v0.9.62
+  BUILDER_VERSION: v0.9.64
   BUILDER_SOURCE: releases
   BUILDER_HOST: https://d19elf31gohf1l.cloudfront.net
   PACKAGE_NAME: aws-c-s3

.github/workflows/issue-regression-labeler.yml

@@ -0,0 +1,32 @@
+# Apply potential regression label on issues
+name: issue-regression-label
+on:
+  issues:
+    types: [opened, edited]
+jobs:
+  add-regression-label:
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write
+    steps:
+    - name: Fetch template body
+      id: check_regression
+      uses: actions/github-script@v7
+      env: 
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        TEMPLATE_BODY: ${{ github.event.issue.body }}
+      with:
+        script: |
+          const regressionPattern = /\[x\] Select this option if this issue appears to be a regression\./i;
+          const template = `${process.env.TEMPLATE_BODY}`
+          const match = regressionPattern.test(template);
+          core.setOutput('is_regression', match);
+    - name: Manage regression label
+      env:
+        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+      run: |
+        if [ "${{ steps.check_regression.outputs.is_regression }}" == "true" ]; then
+          gh issue edit ${{ github.event.issue.number }} --add-label "potential-regression" -R ${{ github.repository }}
+        else
+          gh issue edit ${{ github.event.issue.number }} --remove-label "potential-regression" -R ${{ github.repository }}
+        fi

.gitignore

@@ -68,3 +68,6 @@ cmake-build*
 # js package locks irrelevant to the overall package's security
 benchmarks/benchmarks-stack/benchmarks-stack/package-lock.json
 benchmarks/dashboard-stack/package-lock.json
+
+# virtual environment
+.venv/

CMakeLists.txt

@@ -1,11 +1,7 @@
 
-cmake_minimum_required(VERSION 3.1)
+cmake_minimum_required(VERSION 3.9)
 project(aws-c-s3 C)
 
-if (POLICY CMP0069)
-    cmake_policy(SET CMP0069 NEW) # Enable LTO/IPO if available in the compiler, see AwsCFlags
-endif()
-
 option(ASSERT_LOCK_HELD "Enable ASSERT_SYNCED_DATA_LOCK_HELD for checking thread issue" OFF)
 option(ENABLE_MOCK_SERVER_TESTS "Whether to run the integration tests that rely on pre-configured mock server" OFF)
 option(ENABLE_MRAP_TESTS "Whether to run the integration tests that rely on pre-configured multi-region access point" OFF)

README.md

@@ -23,7 +23,7 @@ This library is licensed under the Apache 2.0 License.
 
 ### Building
 
-CMake 3.1+ is required to build.
+CMake 3.9+ is required to build.
 
 `<install-path>` must be an absolute path in the following instructions.
 

include/aws/s3/private/s3_checksums.h

@@ -11,26 +11,48 @@
 
 struct aws_s3_checksum;
 
+/* List to check the checksum algorithm to use based on the priority. */
+static const enum aws_s3_checksum_algorithm s_checksum_algo_priority_list[] = {
+    AWS_SCA_CRC64NVME,
+    AWS_SCA_CRC32C,
+    AWS_SCA_CRC32,
+    AWS_SCA_SHA1,
+    AWS_SCA_SHA256,
+};
+AWS_STATIC_ASSERT(AWS_ARRAY_SIZE(s_checksum_algo_priority_list) == (AWS_SCA_END - AWS_SCA_INIT + 1));
+
 struct aws_checksum_vtable {
     void (*destroy)(struct aws_s3_checksum *checksum);
     int (*update)(struct aws_s3_checksum *checksum, const struct aws_byte_cursor *buf);
-    int (*finalize)(struct aws_s3_checksum *checksum, struct aws_byte_buf *out, size_t truncate_to);
+    int (*finalize)(struct aws_s3_checksum *checksum, struct aws_byte_buf *out);
 };
 
 struct aws_s3_checksum {
     struct aws_allocator *allocator;
     struct aws_checksum_vtable *vtable;
-    void *impl;
     size_t digest_size;
     enum aws_s3_checksum_algorithm algorithm;
     bool good;
+    union {
+        struct aws_hash *hash;
+        uint32_t crc_val_32bit;
+        uint64_t crc_val_64bit;
+    } impl;
 };
 
-struct checksum_config {
+struct checksum_config_storage {
+    struct aws_allocator *allocator;
+    struct aws_byte_buf full_object_checksum;
+    bool has_full_object_checksum;
+
+    aws_s3_meta_request_full_object_checksum_fn *full_object_checksum_callback;
+    void *user_data;
+
     enum aws_s3_checksum_location location;
     enum aws_s3_checksum_algorithm checksum_algorithm;
     bool validate_response_checksum;
     struct {
+        bool crc64nvme;
         bool crc32c;
         bool crc32;
         bool sha1;
@@ -85,25 +107,26 @@ struct aws_input_stream *aws_chunk_stream_new(
  * Get the size of the checksum output corresponding to the aws_s3_checksum_algorithm enum value.
  */
 AWS_S3_API
-size_t aws_get_digest_size_from_algorithm(enum aws_s3_checksum_algorithm algorithm);
+size_t aws_get_digest_size_from_checksum_algorithm(enum aws_s3_checksum_algorithm algorithm);
 
 /**
- * Get the header name corresponding to the aws_s3_checksum_algorithm enum value.
+ * Get header name to use for algorithm (e.g. "x-amz-checksum-crc32")
  */
 AWS_S3_API
-const struct aws_byte_cursor *aws_get_http_header_name_from_algorithm(enum aws_s3_checksum_algorithm algorithm);
+struct aws_byte_cursor aws_get_http_header_name_from_checksum_algorithm(enum aws_s3_checksum_algorithm algorithm);
 
 /**
- * Get the multipart upload header name corresponding to the aws_s3_checksum_algorithm enum value.
+ * Get algorithm's name (e.g. "CRC32"), to be used as the value of headers like `x-amz-checksum-algorithm`
  */
 AWS_S3_API
-const struct aws_byte_cursor *aws_get_create_mpu_header_name_from_algorithm(enum aws_s3_checksum_algorithm algorithm);
+struct aws_byte_cursor aws_get_checksum_algorithm_name(enum aws_s3_checksum_algorithm algorithm);
 
 /**
- * Get the complete multipart upload name corresponding to the aws_s3_checksum_algorithm enum value.
+ * Get the name of checksum algorithm to be used as the details of the parts were uploaded. Referring to
+ * https://docs.aws.amazon.com/AmazonS3/latest/API/API_CompletedPart.html#AmazonS3-Type-CompletedPart
  */
 AWS_S3_API
-const struct aws_byte_cursor *aws_get_complete_mpu_name_from_algorithm(enum aws_s3_checksum_algorithm algorithm);
+struct aws_byte_cursor aws_get_completed_part_name_from_checksum_algorithm(enum aws_s3_checksum_algorithm algorithm);
 
 /**
  * create a new aws_checksum corresponding to the aws_s3_checksum_algorithm enum value.
@@ -121,8 +144,7 @@ int aws_checksum_compute(
     struct aws_allocator *allocator,
     enum aws_s3_checksum_algorithm algorithm,
     const struct aws_byte_cursor *input,
-    struct aws_byte_buf *output,
-    size_t truncate_to);
+    struct aws_byte_buf *output);
 
 /**
  * Cleans up and deallocates checksum.
@@ -141,9 +163,15 @@ int aws_checksum_update(struct aws_s3_checksum *checksum, const struct aws_byte_
  * Allocation of output is the caller's responsibility.
  */
 AWS_S3_API
-int aws_checksum_finalize(struct aws_s3_checksum *checksum, struct aws_byte_buf *output, size_t truncate_to);
+int aws_checksum_finalize(struct aws_s3_checksum *checksum, struct aws_byte_buf *output);
+
+AWS_S3_API
+void aws_checksum_config_storage_init(
+    struct aws_allocator *allocator,
+    struct checksum_config_storage *internal_config,
+    const struct aws_s3_checksum_config *config);
 
 AWS_S3_API
-void checksum_config_init(struct checksum_config *internal_config, const struct aws_s3_checksum_config *config);
+void aws_checksum_config_storage_cleanup(struct checksum_config_storage *internal_config);
 
 #endif /* AWS_S3_CHECKSUMS_H */

include/aws/s3/private/s3_client_impl.h

@@ -169,6 +169,10 @@ struct aws_s3_client_vtable {
 
     struct aws_parallel_input_stream *(
         *parallel_input_stream_new_from_file)(struct aws_allocator *allocator, struct aws_byte_cursor file_name);
+
+    struct aws_http_stream *(*http_connection_make_request)(
+        struct aws_http_connection *client_connection,
+        const struct aws_http_make_request_options *options);
 };
 
 struct aws_s3_upload_part_timeout_stats {

include/aws/s3/private/s3_meta_request_impl.h

@@ -272,13 +272,18 @@ struct aws_s3_meta_request {
     const bool should_compute_content_md5;
 
     /* deep copy of the checksum config. */
-    struct checksum_config checksum_config;
+    struct checksum_config_storage checksum_config;
 
     /* checksum found in either a default get request, or in the initial head request of a multipart get */
     struct aws_byte_buf meta_request_level_response_header_checksum;
 
     /* running checksum of all the parts of a default get, or ranged get meta request*/
     struct aws_s3_checksum *meta_request_level_running_response_sum;
+
+    /* The receiving file handler */
+    FILE *recv_file;
+    struct aws_string *recv_filepath;
+    bool recv_file_delete_on_failure;
 };
 
 /* Info for each part, that we need to remember until we send CompleteMultipartUpload */
@@ -345,6 +350,14 @@ void aws_s3_meta_request_init_signing_date_time_default(
     struct aws_s3_meta_request *meta_request,
     struct aws_date_time *date_time);
 
+AWS_S3_API
+void aws_s3_meta_request_sign_request_default_impl(
+    struct aws_s3_meta_request *meta_request,
+    struct aws_s3_request *request,
+    aws_signing_complete_fn *on_signing_complete,
+    void *user_data,
+    bool disable_s3_express_signing);
+
 AWS_S3_API
 void aws_s3_meta_request_sign_request_default(
     struct aws_s3_meta_request *meta_request,
@@ -452,6 +465,13 @@ bool aws_s3_meta_request_checksum_config_has_algorithm(
     struct aws_s3_meta_request *meta_request,
     enum aws_s3_checksum_algorithm algorithm);
 
+void aws_s3_meta_request_schedule_prepare_request_default_impl(
+    struct aws_s3_meta_request *meta_request,
+    struct aws_s3_request *request,
+    bool parallel,
+    aws_s3_meta_request_prepare_request_callback_fn *callback,
+    void *user_data);
+
 AWS_EXTERN_C_END
 
 #endif /* AWS_S3_META_REQUEST_IMPL_H */

include/aws/s3/private/s3_request.h

@@ -212,6 +212,9 @@ struct aws_s3_request {
 
         /* The metrics for the request telemetry */
         struct aws_s3_request_metrics *metrics;
+
+        /* The request is required to have the unsigned payload */
+        uint32_t require_streaming_unsigned_payload_header : 1;
     } send_data;
 
     /* When true, response headers from the request will be stored in the request's response_headers variable. */

include/aws/s3/private/s3_request_messages.h

@@ -17,7 +17,7 @@ struct aws_byte_buf;
 struct aws_byte_cursor;
 struct aws_string;
 struct aws_array_list;
-struct checksum_config;
+struct checksum_config_storage;
 
 AWS_EXTERN_C_BEGIN
 
@@ -52,13 +52,9 @@ struct aws_input_stream *aws_s3_message_util_assign_body(
     struct aws_allocator *allocator,
     struct aws_byte_buf *byte_buf,
     struct aws_http_message *out_message,
-    const struct checksum_config *checksum_config,
+    const struct checksum_config_storage *checksum_config,
     struct aws_byte_buf *out_checksum);
 
-/* Return true if checksum headers has been set. */
-AWS_S3_API
-bool aws_s3_message_util_check_checksum_header(struct aws_http_message *message);
-
 /* Create an HTTP request for an S3 Ranged Get Object Request, using the given request as a basis */
 AWS_S3_API
 struct aws_http_message *aws_s3_ranged_get_object_message_new(
@@ -80,7 +76,7 @@ AWS_S3_API
 struct aws_http_message *aws_s3_create_multipart_upload_message_new(
     struct aws_allocator *allocator,
     struct aws_http_message *base_message,
-    const struct checksum_config *checksum_config);
+    const struct checksum_config_storage *checksum_config);
 
 /* Create an HTTP request for an S3 Put Object request, using the original request as a basis.  Creates and assigns a
  * body stream using the passed in buffer.  If multipart is not needed, part number and upload_id can be 0 and NULL,
@@ -93,7 +89,7 @@ struct aws_http_message *aws_s3_upload_part_message_new(
     uint32_t part_number,
     const struct aws_string *upload_id,
     bool should_compute_content_md5,
-    const struct checksum_config *checksum_config,
+    const struct checksum_config_storage *checksum_config,
     struct aws_byte_buf *encoded_checksum_output);
 
 /* Create an HTTP request for an S3 UploadPartCopy request, using the original request as a basis.
@@ -120,7 +116,7 @@ struct aws_http_message *aws_s3_complete_multipart_message_new(
     struct aws_byte_buf *body_buffer,
     const struct aws_string *upload_id,
     const struct aws_array_list *parts,
-    const struct checksum_config *checksum_config);
+    const struct checksum_config_storage *checksum_config);
 
 AWS_S3_API
 struct aws_http_message *aws_s3_abort_multipart_upload_message_new(

include/aws/s3/private/s3_util.h

@@ -61,44 +61,17 @@ extern const struct aws_byte_cursor g_request_validation_mode;
 AWS_S3_API
 extern const struct aws_byte_cursor g_enabled;
 
+/**
+ * The checksum-algorithm header name used for CopyObject and CreateMultipartUpload
+ */
 AWS_S3_API
-extern const struct aws_byte_cursor g_create_mpu_checksum_header_name;
-
-AWS_S3_API
-extern const struct aws_byte_cursor g_crc32c_header_name;
-
-AWS_S3_API
-extern const struct aws_byte_cursor g_crc32_header_name;
-
-AWS_S3_API
-extern const struct aws_byte_cursor g_sha1_header_name;
-
-AWS_S3_API
-extern const struct aws_byte_cursor g_sha256_header_name;
-
-AWS_S3_API
-extern const struct aws_byte_cursor g_crc32c_create_mpu_header_name;
-
-AWS_S3_API
-extern const struct aws_byte_cursor g_crc32_create_mpu_header_name;
-
-AWS_S3_API
-extern const struct aws_byte_cursor g_sha1_create_mpu_header_name;
-
-AWS_S3_API
-extern const struct aws_byte_cursor g_sha256_create_mpu_header_name;
-
-AWS_S3_API
-extern const struct aws_byte_cursor g_crc32c_complete_mpu_name;
-
-AWS_S3_API
-extern const struct aws_byte_cursor g_crc32_complete_mpu_name;
-
-AWS_S3_API
-extern const struct aws_byte_cursor g_sha1_complete_mpu_name;
+extern const struct aws_byte_cursor g_checksum_algorithm_header_name;
 
+/**
+ * The checksum-algorithm header name used for PutObject, UploadParts and PutObject*
+ */
 AWS_S3_API
-extern const struct aws_byte_cursor g_sha256_complete_mpu_name;
+extern const struct aws_byte_cursor g_sdk_checksum_algorithm_header_name;
 
 AWS_S3_API
 extern const struct aws_byte_cursor g_s3_client_version;

include/aws/s3/s3.h

@@ -46,6 +46,9 @@ enum aws_s3_errors {
     AWS_ERROR_S3EXPRESS_CREATE_SESSION_FAILED,
     AWS_ERROR_S3_INTERNAL_PART_SIZE_MISMATCH_RETRYING_WITH_RANGE,
     AWS_ERROR_S3_REQUEST_HAS_COMPLETED,
+    AWS_ERROR_S3_RECV_FILE_ALREADY_EXISTS,
+    AWS_ERROR_S3_RECV_FILE_NOT_FOUND,
+    AWS_ERROR_S3_REQUEST_TIMEOUT,
 
     AWS_ERROR_S3_END_RANGE = AWS_ERROR_ENUM_END_RANGE(AWS_C_S3_PACKAGE_ID)
 };