diff --git a/jest.cjs b/jest.cjs index e6f646e7..07243baa 100644 --- a/jest.cjs +++ b/jest.cjs @@ -3,10 +3,10 @@ module.exports.makePackageConfig = () => { ...require("@jspsych/config/jest").makePackageConfig(__dirname), coverageThreshold: { global: { - branches: 80, - functions: 80, - lines: 80, - statements: -10, + branches: 100, + functions: 100, + lines: 100, + statements: 0, }, }, }; diff --git a/packages/data/src/error.ts b/packages/data/src/error.ts new file mode 100644 index 00000000..ade94873 --- /dev/null +++ b/packages/data/src/error.ts @@ -0,0 +1,25 @@ +/** Error for when AWS response is missing attribute. */ +export class AWSMissingAttrError extends Error { + /** + * Error for when AWS response is missing attribute. + * + * @param awsAttrName - Name of missing AWS response attribute. + */ + public constructor(awsAttrName: string) { + super(`Response from AWS send is missing an attribute: ${awsAttrName}`); + this.name = "AWSMissingAttrError"; + } +} + +/** Error when the upload part retries run out. */ +export class UploadPartError extends Error { + /** + * Error when the upload part retries run out. + * + * @param error - Error object generated by upload part function. + */ + public constructor(error?: Error) { + super(`Upload part failed after 3 attempts: ${error}`); + this.name = "UploadPartError"; + } +} diff --git a/packages/data/src/lookitS3.spec.ts b/packages/data/src/lookitS3.spec.ts new file mode 100644 index 00000000..bd31083b --- /dev/null +++ b/packages/data/src/lookitS3.spec.ts @@ -0,0 +1,92 @@ +import { + CompleteMultipartUploadCommand, + CreateMultipartUploadCommand, + UploadPartCommand, +} from "@aws-sdk/client-s3"; +import LookitS3 from "./lookitS3"; + +let mockSendRtn: { UploadId?: string; ETag?: string }; +const largeBlob = new Blob(["x".repeat(LookitS3.minUploadSize + 1)]); + +jest.mock("@aws-sdk/client-s3", () => ({ + S3Client: jest.fn().mockImplementation(() => ({ + send: jest.fn().mockReturnValue(mockSendRtn), + })), + CreateMultipartUploadCommand: jest.fn().mockImplementation(), + UploadPartCommand: jest.fn().mockImplementation(), + CompleteMultipartUploadCommand: jest.fn().mockImplementation(), +})); + +afterEach(() => { + jest.clearAllMocks(); +}); + +test("Upload file to S3", async () => { + mockSendRtn = { UploadId: "upload id", ETag: "etag" }; + const s3 = new LookitS3("key value"); + + await s3.createUpload(); + expect(s3.percentUploadComplete).toBeNaN; + + s3.onDataAvailable(largeBlob); + expect(s3.percentUploadComplete).toBe(0); + + await s3.completeUpload(); + expect(s3.percentUploadComplete).toBe(100); + + expect(UploadPartCommand).toHaveBeenCalledTimes(2); + expect(CreateMultipartUploadCommand).toHaveBeenCalledTimes(1); + expect(CompleteMultipartUploadCommand).toHaveBeenCalledTimes(1); +}); + +test("Upload file to S3 multiple parts", async () => { + mockSendRtn = { UploadId: "upload id", ETag: "etag" }; + const s3 = new LookitS3("key value"); + + await s3.createUpload(); + + for (let i = 0; i < 2; i++) { + for (let j = 0; j < 100; j++) { + s3.onDataAvailable(largeBlob.slice(0, largeBlob.size / 100)); + } + } + + await s3.completeUpload(); + + /** + * UploadPartCommand is called once for every 100 blobs added. We do this + * twice. Add one more call for when we completeUpload. + */ + expect(UploadPartCommand).toHaveBeenCalledTimes(3); + + expect(CreateMultipartUploadCommand).toHaveBeenCalledTimes(1); + expect(CompleteMultipartUploadCommand).toHaveBeenCalledTimes(1); +}); + +test("Upload to S3 missing upload id", () => { + mockSendRtn = { ETag: "etag" }; + const s3 = new LookitS3("key value"); + expect(async () => { + await s3.createUpload(); + }).rejects.toThrow( + "Response from AWS send is missing an attribute: UploadId", + ); + + expect(CreateMultipartUploadCommand).toHaveBeenCalledTimes(1); +}); + +test("Upload to S3 missing Etag", async () => { + mockSendRtn = { UploadId: "upload id" }; + const s3 = new LookitS3("key value"); + await s3.createUpload(); + s3.onDataAvailable(largeBlob); + expect(async () => { + await s3.completeUpload(); + }).rejects.toThrow( + "Upload part failed after 3 attempts: AWSMissingAttrError: Response from AWS send is missing an attribute: ETag", + ); + + expect(UploadPartCommand).toHaveBeenCalledTimes(2); + expect(CreateMultipartUploadCommand).toHaveBeenCalledTimes(1); + expect(CompleteMultipartUploadCommand).toHaveBeenCalledTimes(0); +}); diff --git a/packages/data/src/lookitS3.ts b/packages/data/src/lookitS3.ts index b2cf304b..f0b4371c 100644 --- a/packages/data/src/lookitS3.ts +++ b/packages/data/src/lookitS3.ts @@ -4,18 +4,20 @@ import { S3Client, UploadPartCommand, } from "@aws-sdk/client-s3"; +import { AWSMissingAttrError, UploadPartError } from "./error"; import { Env } from "./types"; -/** Provides functionality to upload videos incremetally to an AWS S3 Bucket. */ +/** Provides functionality to upload videos incrementally to an AWS S3 Bucket. */ class LookitS3 { private blobParts: Blob[]; - private promises: Promise<{ PartNumber: number; ETag?: string }>[]; + private promises: Promise<{ PartNumber: number; ETag: string }>[]; private partNumber: number; private partsUploaded: number; private s3: S3Client; - private uploadId?: string; + private uploadId: string = ""; private env: Env; private key: string; + public static readonly minUploadSize: number = 5 * 1024 * 1024; /** * Provide file name to initiate a new upload to a S3 bucket. The AWS secrets @@ -62,7 +64,7 @@ class LookitS3 { * @returns Percent uploaded. */ public get percentUploadComplete() { - return Math.floor((this.partsUploaded / this.partNumber) * 100); + return Math.floor((this.partsUploaded / this.promises.length) * 100); } /** @@ -84,7 +86,12 @@ class LookitS3 { Key: this.key, ContentType: "video/webm", // TO DO: check browser support for type/codec and set the actual value here }); + const response = await this.s3.send(command); + if (!response.UploadId) { + throw new AWSMissingAttrError("UploadId"); + } + this.uploadId = response.UploadId; this.logRecordingEvent(`Connection established.`); } @@ -99,25 +106,24 @@ class LookitS3 { */ private async uploadPart(blob: Blob, partNumber: number) { let retry = 0; - let err; - - if (!this.uploadId) { - throw Error("no upload id."); - } + let err: Error | undefined = undefined; + const input = { + Body: blob, + Bucket: this.env.bucket, + Key: this.key, + PartNumber: partNumber, + UploadId: this.uploadId, + }; + const command = new UploadPartCommand(input); while (retry < 3) { try { - const input = { - Body: blob, - Bucket: this.env.bucket, - Key: this.key, - PartNumber: partNumber, - UploadId: this.uploadId, - }; - const command = new UploadPartCommand(input); const response = await this.s3.send(command); - this.logRecordingEvent(`Uploaded file part ${partNumber}.`); + if (!response.ETag) { + throw new AWSMissingAttrError("ETag"); + } + this.logRecordingEvent(`Uploaded file part ${partNumber}.`); this.partsUploaded++; return { @@ -128,11 +134,12 @@ class LookitS3 { this.logRecordingEvent( `Error uploading part ${partNumber}.\nError: ${_err}`, ); - err = _err; + err = _err as Error; retry += 1; } } - throw Error(`Upload part failed after 3 attempts.\nError: ${err}`); + + throw new UploadPartError(err); } /** @@ -142,10 +149,6 @@ class LookitS3 { public async completeUpload() { this.addUploadPartPromise(); - if (!this.uploadId) { - throw Error("No upload id"); - } - const input = { Bucket: this.env.bucket, Key: this.key, @@ -169,7 +172,7 @@ class LookitS3 { public onDataAvailable(blob: Blob) { this.blobParts.push(blob); - if (this.blobPartsSize > 5 * (1024 * 1024)) { + if (this.blobPartsSize > LookitS3.minUploadSize) { this.addUploadPartPromise(); } }