From 140ab0cd0be95f6c9d681d7e375521e17c256960 Mon Sep 17 00:00:00 2001 From: Thodoris Greasidis Date: Tue, 9 Jan 2024 12:48:07 +0200 Subject: [PATCH] Bump the aws-sdk to v3 Change-type: major --- .../device-types/build-info-facade.ts | 4 +- src/features/device-types/storage/s3.ts | 80 ++++++------------ test/test-lib/aws-mock.ts | 84 +++++++++++-------- 3 files changed, 78 insertions(+), 90 deletions(-) diff --git a/src/features/device-types/build-info-facade.ts b/src/features/device-types/build-info-facade.ts index a3f0b5288..e04a05a9e 100644 --- a/src/features/device-types/build-info-facade.ts +++ b/src/features/device-types/build-info-facade.ts @@ -65,7 +65,9 @@ export const getDeviceTypeJson = multiCacheMemoizee( ); const deviceType = response && response.Body - ? (JSON.parse(response.Body.toString()) as DeviceTypeJson) + ? (JSON.parse( + await response.Body.transformToString(), + ) as DeviceTypeJson) : undefined; if (deviceType) { deviceType.buildId = buildId; diff --git a/src/features/device-types/storage/s3.ts b/src/features/device-types/storage/s3.ts index a547a7908..a74aa05a3 100644 --- a/src/features/device-types/storage/s3.ts +++ b/src/features/device-types/storage/s3.ts @@ -1,9 +1,7 @@ -import AWSWrapper from './aws-sdk-wrapper.cjs'; +import { S3 } from '@aws-sdk/client-s3'; import _ from 'lodash'; import path from 'path'; -const { AWS } = AWSWrapper; - import { IMAGE_STORAGE_ACCESS_KEY, IMAGE_STORAGE_BUCKET as S3_BUCKET, @@ -14,75 +12,53 @@ import { export const getKey = (...parts: string[]): string => parts.join('/'); -class UnauthenticatedS3Facade { - constructor(private s3: AWS.S3) {} - - public headObject( - params: AWS.S3.Types.HeadObjectRequest, - ): ReturnType { - return this.s3.makeUnauthenticatedRequest('headObject', params); - } - - public getObject( - params: AWS.S3.Types.GetObjectRequest, - ): ReturnType { - return this.s3.makeUnauthenticatedRequest('getObject', params); - } - - public listObjectsV2( - params: AWS.S3.Types.ListObjectsV2Request, - ): ReturnType { - return this.s3.makeUnauthenticatedRequest('listObjectsV2', params); - } -} - -function createS3Client() { - if (!IMAGE_STORAGE_ACCESS_KEY || !IMAGE_STORAGE_SECRET_KEY) { - return new UnauthenticatedS3Facade( - new AWS.S3({ - endpoint: IMAGE_STORAGE_ENDPOINT, - s3ForcePathStyle: IMAGE_STORAGE_FORCE_PATH_STYLE, - signatureVersion: 'v4', +const s3Client = new S3({ + // The transformation for endpoint is not implemented. + // Refer to UPGRADING.md on aws-sdk-js-v3 for changes needed. + // Please create/upvote feature request on aws-sdk-js-codemod for endpoint. + endpoint: IMAGE_STORAGE_ENDPOINT, + + region: 'us-east-1', + + // The key s3ForcePathStyle is renamed to forcePathStyle. + forcePathStyle: IMAGE_STORAGE_FORCE_PATH_STYLE, + + ...(!IMAGE_STORAGE_ACCESS_KEY || !IMAGE_STORAGE_SECRET_KEY + ? { + // makes the requests being unauthenticated + signer: { sign: async (request) => request }, + } + : { + credentials: { + accessKeyId: IMAGE_STORAGE_ACCESS_KEY, + secretAccessKey: IMAGE_STORAGE_SECRET_KEY, + }, }), - ); - } - return new AWS.S3({ - endpoint: IMAGE_STORAGE_ENDPOINT, - s3ForcePathStyle: IMAGE_STORAGE_FORCE_PATH_STYLE, - signatureVersion: 'v4', - accessKeyId: IMAGE_STORAGE_ACCESS_KEY, - secretAccessKey: IMAGE_STORAGE_SECRET_KEY, - }); -} - -const s3Client = createS3Client(); +}); async function getFileInfo(s3Path: string) { - const req = s3Client.headObject({ + return await s3Client.headObject({ Bucket: S3_BUCKET, Key: s3Path, }); - return await req.promise(); } export async function getFile(s3Path: string) { - const req = s3Client.getObject({ + return await s3Client.getObject({ Bucket: S3_BUCKET, Key: s3Path, }); - return await req.promise(); } export async function getFolderSize( folder: string, marker?: string, ): Promise { - const req = s3Client.listObjectsV2({ + const res = await s3Client.listObjectsV2({ Bucket: S3_BUCKET, Prefix: `${folder}/`, ContinuationToken: marker, }); - const res = await req.promise(); const size = _.sumBy(res.Contents, 'Size'); const nextMarker = res.NextContinuationToken; @@ -97,15 +73,13 @@ export async function listFolders( folder: string, marker?: string, ): Promise { - const req = s3Client.listObjectsV2({ + const res = await s3Client.listObjectsV2({ Bucket: S3_BUCKET, Prefix: `${folder}/`, Delimiter: '/', ContinuationToken: marker, }); - const res = await req.promise(); - const objects = _(res.CommonPrefixes) .map(({ Prefix }) => Prefix) // only keep the folder paths (which are ending with `/`) diff --git a/test/test-lib/aws-mock.ts b/test/test-lib/aws-mock.ts index 04555ecdd..6f577dc40 100644 --- a/test/test-lib/aws-mock.ts +++ b/test/test-lib/aws-mock.ts @@ -1,4 +1,11 @@ -import type AWS from 'aws-sdk'; +import type { + GetObjectCommandInput, + GetObjectCommandOutput, + HeadObjectCommandInput, + ListObjectsV2CommandInput, + S3, + S3ClientConfig, +} from '@aws-sdk/client-s3'; import Bluebird from 'bluebird'; import { assert } from 'chai'; import _ from 'lodash'; @@ -14,22 +21,27 @@ import listObjectsV2Mocks from '../fixtures/s3/listObjectsV2.json' assert { type // AWS S3 Client getObject results have a Buffer on their Body prop // and a Date on their LastModified prop so we have to reconstruct // them from the strings that the mock object holds -const getObjectMocks: Dictionary = _.mapValues( +const getObjectMocks: Dictionary = _.mapValues( $getObjectMocks, ( getObjectMock: (typeof $getObjectMocks)[keyof typeof $getObjectMocks], - ): AWS.S3.Types.GetObjectOutput => { + ): GetObjectCommandOutput => { return { ...getObjectMock, Body: 'Body' in getObjectMock && getObjectMock.Body - ? Buffer.from(getObjectMock.Body) + ? ({ + async transformToString() { + return Buffer.from(getObjectMock.Body).toString(); + }, + } as GetObjectCommandOutput['Body']) : undefined, LastModified: 'LastModified' in getObjectMock && getObjectMock.LastModified ? new Date(getObjectMock.LastModified) : undefined, + $metadata: {}, }; }, ); @@ -58,50 +70,52 @@ const toReturnType = any>(result: { } as unknown as ReturnType; }; -interface UnauthenticatedRequestParams { - [key: string]: any; -} +// interface UnauthenticatedRequestParams { +// [key: string]: any; +// } class S3Mock { - constructor(params: AWS.S3.Types.ClientConfiguration) { + constructor(params: S3ClientConfig) { assert( - params.accessKeyId === IMAGE_STORAGE_ACCESS_KEY, + 'accessKeyId' in params && + params.accessKeyId === IMAGE_STORAGE_ACCESS_KEY, 'S3 access key not matching', ); assert( - params.secretAccessKey === IMAGE_STORAGE_SECRET_KEY, + 'secretAccessKey' in params && + params.secretAccessKey === IMAGE_STORAGE_SECRET_KEY, 'S3 secret key not matching', ); } - public makeUnauthenticatedRequest( - operation: string, - params?: UnauthenticatedRequestParams, - ): AWS.Request { - if (operation === 'headObject') { - return this.headObject(params as AWS.S3.Types.HeadObjectRequest); - } - if (operation === 'getObject') { - return this.getObject(params as AWS.S3.Types.GetObjectRequest); - } - if (operation === 'listObjectsV2') { - return this.listObjectsV2(params as AWS.S3.Types.ListObjectsV2Request); - } - throw new Error(`AWS Mock: Operation ${operation} isn't implemented`); - } + // public makeUnauthenticatedRequest( + // operation: string, + // params?: UnauthenticatedRequestParams, + // ): AWS.Request { + // if (operation === 'headObject') { + // return this.headObject(params as HeadObjectCommandInput); + // } + // if (operation === 'getObject') { + // return this.getObject(params as GetObjectCommandInput); + // } + // if (operation === 'listObjectsV2') { + // return this.listObjectsV2(params as ListObjectsV2CommandInput); + // } + // throw new Error(`AWS Mock: Operation ${operation} isn't implemented`); + // } public headObject( - params: AWS.S3.Types.HeadObjectRequest, - ): ReturnType { + params: HeadObjectCommandInput, + ): ReturnType { const mock = getObjectMocks[params.Key as keyof typeof getObjectMocks]; if (mock) { const trimmedMock = _.omit(mock, 'Body', 'ContentRange', 'TagCount'); - return toReturnType(trimmedMock); + return toReturnType(trimmedMock); } // treat not found IGNORE file mocks as 404 if (_.endsWith(params.Key, '/IGNORE')) { - return toReturnType( + return toReturnType( Bluebird.reject(new NotFoundError()), ); } @@ -111,21 +125,19 @@ class S3Mock { ); } - public getObject( - params: AWS.S3.Types.GetObjectRequest, - ): ReturnType { + public getObject(params: GetObjectCommandInput): ReturnType { const mock = getObjectMocks[params.Key as keyof typeof getObjectMocks]; if (!mock) { throw new Error( `aws mock: getObject could not find a mock for ${params.Key}`, ); } - return toReturnType(mock); + return toReturnType(mock); } public listObjectsV2( - params: AWS.S3.Types.ListObjectsV2Request, - ): ReturnType { + params: ListObjectsV2CommandInput, + ): ReturnType { const mock = listObjectsV2Mocks[params.Prefix as keyof typeof listObjectsV2Mocks]; if (!mock) { @@ -133,7 +145,7 @@ class S3Mock { `aws mock: listObjectsV2 could not find a mock for ${params.Prefix}`, ); } - return toReturnType(mock); + return toReturnType(mock); } }