[file-asset-apis] Refactor Md5 middleware logic (#1180)

This PR makes the following changes: - removes `ChecksumValidation` arguments to the s3 configuration - These don't seem necessary and didn't resolve the main issue here #1157 - Refactored the s3 middleware md5 step to only remove checksum headers instead of all headers - #1157 (comment)
terascope · Feb 19, 2025 · bd54462 · bd54462
1 parent 9787db1
commit bd54462
Show file tree

Hide file tree

Showing 8 changed files with 18 additions and 27 deletions.
diff --git a/asset/asset.json b/asset/asset.json
@@ -1,6 +1,6 @@
 {
     "name": "file",
-    "version": "3.0.4",
+    "version": "3.0.5",
     "description": "A set of processors for working with files",
     "minimum_teraslice_version": "2.0.0"
 }
diff --git a/asset/package.json b/asset/package.json
@@ -1,7 +1,7 @@
 {
     "name": "file",
     "displayName": "Asset",
-    "version": "3.0.4",
+    "version": "3.0.5",
     "private": true,
     "description": "A set of processors for working with files",
     "repository": {
@@ -21,7 +21,7 @@
         "test": "yarn --cwd ../ test"
     },
     "dependencies": {
-        "@terascope/file-asset-apis": "~1.0.3",
+        "@terascope/file-asset-apis": "~1.0.4",
         "@terascope/job-components": "~1.9.5",
         "csvtojson": "~2.0.10",
         "fs-extra": "~11.3.0",

diff --git a/package.json b/package.json
@@ -1,7 +1,7 @@
 {
     "name": "file-assets-bundle",
     "displayName": "File Assets Bundle",
-    "version": "3.0.4",
+    "version": "3.0.5",
     "private": true,
     "description": "A set of processors for working with files",
     "repository": "https://github.com/terascope/file-assets.git",
@@ -33,7 +33,7 @@
     },
     "devDependencies": {
         "@terascope/eslint-config": "~1.1.6",
-        "@terascope/file-asset-apis": "~1.0.3",
+        "@terascope/file-asset-apis": "~1.0.4",
         "@terascope/job-components": "~1.9.5",
         "@terascope/scripts": "~1.10.3",
         "@types/fs-extra": "~11.0.4",

diff --git a/packages/file-asset-apis/package.json b/packages/file-asset-apis/package.json
@@ -1,7 +1,7 @@
 {
     "name": "@terascope/file-asset-apis",
     "displayName": "File Asset Apis",
-    "version": "1.0.3",
+    "version": "1.0.4",
     "description": "file reader and sender apis",
     "homepage": "https://github.com/terascope/file-assets",
     "repository": "[email protected]:terascope/file-assets.git",

diff --git a/packages/file-asset-apis/src/s3/createS3Client.ts b/packages/file-asset-apis/src/s3/createS3Client.ts
@@ -66,9 +66,6 @@ export async function genFinalS3ClientConfig(config: S3ClientConfig): Promise<Ba
         throw new Error(`S3 endpoint ${config.endpoint} cannot be https if sslEnabled is false`);
     }
 
-    config.responseChecksumValidation = 'WHEN_REQUIRED';
-    config.requestChecksumCalculation = 'WHEN_REQUIRED';
-
     if (!config.credentials) {
         config.credentials = createCredentialsObject(config);
     }

diff --git a/packages/file-asset-apis/src/s3/s3-helpers.ts b/packages/file-asset-apis/src/s3/s3-helpers.ts
@@ -130,22 +130,22 @@ export async function deleteS3Objects(
                 const body = request.body as string;
                 /// Check to see if the command is of the right type
                 if (context.commandName === 'DeleteObjectsCommand') {
+                    // Remove any checksum headers
+                    Object.keys(headers).forEach((header) => {
+                        if (
+                            header.toLowerCase().startsWith('x-amz-checksum-')
+                            || header.toLowerCase().startsWith('x-amz-sdk-checksum-')
+                        ) {
+                            delete headers[header];
+                        }
+                    });
                     /// Ensure there is a body to make a hash from
                     if (typeof body === 'string' && body) {
                         const md5Hash = crypto.createHash('md5').update(body, 'utf8')
                             .digest('base64');
                         headers['Content-MD5'] = md5Hash;
                     }
                     request.headers = headers;
-
-                    Object.entries(request.headers).forEach(
-                        ([key, value]: [string, string]): void => {
-                            if (!request.headers) {
-                                request.headers = {};
-                            }
-                            (request.headers as Record<string, string>)[key] = value;
-                        }
-                    );
                 }
                 return next(args);
             },

diff --git a/packages/file-asset-apis/test/s3/createS3Client-spec.ts b/packages/file-asset-apis/test/s3/createS3Client-spec.ts
@@ -43,8 +43,6 @@ describe('createS3Client', () => {
                         + '-----END CERTIFICATE-----',
                     forcePathStyle: true,
                     bucketEndpoint: false,
-                    responseChecksumValidation: 'WHEN_REQUIRED',
-                    requestChecksumCalculation: 'WHEN_REQUIRED',
                     maxAttempts: 3,
                     requestHandler: {
                         metadata: { handlerProtocol: 'http/1.1' },
@@ -78,8 +76,6 @@ describe('createS3Client', () => {
                     sslEnabled: false,
                     forcePathStyle: true,
                     bucketEndpoint: false,
-                    responseChecksumValidation: 'WHEN_REQUIRED',
-                    requestChecksumCalculation: 'WHEN_REQUIRED',
                     maxAttempts: 3,
                     credentials: { accessKeyId: 'minioadmin', secretAccessKey: 'minioadmin' }
                 });
@@ -140,8 +136,6 @@ describe('createS3Client', () => {
                     sslEnabled: false,
                     forcePathStyle: true,
                     bucketEndpoint: false,
-                    responseChecksumValidation: 'WHEN_REQUIRED',
-                    requestChecksumCalculation: 'WHEN_REQUIRED',
                     maxAttempts: 3,
                     credentials: { accessKeyId: 'minioadmin', secretAccessKey: 'minioadmin' }
                 });

diff --git a/yarn.lock b/yarn.lock
@@ -2455,7 +2455,7 @@ __metadata:
   languageName: node
   linkType: hard
 
-"@terascope/file-asset-apis@npm:~1.0.3, @terascope/file-asset-apis@workspace:packages/file-asset-apis":
+"@terascope/file-asset-apis@npm:~1.0.4, @terascope/file-asset-apis@workspace:packages/file-asset-apis":
   version: 0.0.0-use.local
   resolution: "@terascope/file-asset-apis@workspace:packages/file-asset-apis"
   dependencies:
@@ -5719,7 +5719,7 @@ __metadata:
   resolution: "file-assets-bundle@workspace:."
   dependencies:
     "@terascope/eslint-config": "npm:~1.1.6"
-    "@terascope/file-asset-apis": "npm:~1.0.3"
+    "@terascope/file-asset-apis": "npm:~1.0.4"
     "@terascope/job-components": "npm:~1.9.5"
     "@terascope/scripts": "npm:~1.10.3"
     "@types/fs-extra": "npm:~11.0.4"
@@ -5786,7 +5786,7 @@ __metadata:
   version: 0.0.0-use.local
   resolution: "file@workspace:asset"
   dependencies:
-    "@terascope/file-asset-apis": "npm:~1.0.3"
+    "@terascope/file-asset-apis": "npm:~1.0.4"
     "@terascope/job-components": "npm:~1.9.5"
     csvtojson: "npm:~2.0.10"
     fs-extra: "npm:~11.3.0"