S3 tests

jest.cjs

@@ -3,10 +3,10 @@ module.exports.makePackageConfig = () => {
     ...require("@jspsych/config/jest").makePackageConfig(__dirname),
     coverageThreshold: {
       global: {
-        branches: 80,
-        functions: 80,
-        lines: 80,
-        statements: -10,
+        branches: 100,
+        functions: 100,
+        lines: 100,
+        statements: 0,
       },
     },
   };

packages/data/src/error.ts

@@ -0,0 +1,25 @@
+/** Error for when AWS response is missing attribute. */
+export class AWSMissingAttrError extends Error {
+  /**
+   * Error for when AWS response is missing attribute.
+   *
+   * @param awsAttrName - Name of missing AWS response attribute.
+   */
+  public constructor(awsAttrName: string) {
+    super(`Response from AWS send is missing an attribute: ${awsAttrName}`);
+    this.name = "AWSMissingAttrError";
+  }
+}
+
+/** Error when the upload part retries run out. */
+export class UploadPartError extends Error {
+  /**
+   * Error when the upload part retries run out.
+   *
+   * @param error - Error object generated by upload part function.
+   */
+  public constructor(error?: Error) {
+    super(`Upload part failed after 3 attempts: ${error}`);
+    this.name = "UploadPartError";
+  }
+}

packages/data/src/lookitS3.spec.ts

@@ -0,0 +1,92 @@
+import {
+  CompleteMultipartUploadCommand,
+  CreateMultipartUploadCommand,
+  UploadPartCommand,
+} from "@aws-sdk/client-s3";
+import LookitS3 from "./lookitS3";
+
+let mockSendRtn: { UploadId?: string; ETag?: string };
+const largeBlob = new Blob(["x".repeat(LookitS3.minUploadSize + 1)]);
+
+jest.mock("@aws-sdk/client-s3", () => ({
+  S3Client: jest.fn().mockImplementation(() => ({
+    send: jest.fn().mockReturnValue(mockSendRtn),
+  })),
+  CreateMultipartUploadCommand: jest.fn().mockImplementation(),
+  UploadPartCommand: jest.fn().mockImplementation(),
+  CompleteMultipartUploadCommand: jest.fn().mockImplementation(),
+}));
+
+afterEach(() => {
+  jest.clearAllMocks();
+});
+
+test("Upload file to S3", async () => {
+  mockSendRtn = { UploadId: "upload id", ETag: "etag" };
+  const s3 = new LookitS3("key value");
+
+  await s3.createUpload();
+  expect(s3.percentUploadComplete).toBeNaN;
+
+  s3.onDataAvailable(largeBlob);
+  expect(s3.percentUploadComplete).toBe(0);
+
+  await s3.completeUpload();
+  expect(s3.percentUploadComplete).toBe(100);
+
+  expect(UploadPartCommand).toHaveBeenCalledTimes(2);
+  expect(CreateMultipartUploadCommand).toHaveBeenCalledTimes(1);
+  expect(CompleteMultipartUploadCommand).toHaveBeenCalledTimes(1);
+});
+
+test("Upload file to S3 multiple parts", async () => {
+  mockSendRtn = { UploadId: "upload id", ETag: "etag" };
+  const s3 = new LookitS3("key value");
+
+  await s3.createUpload();
+
+  for (let i = 0; i < 2; i++) {
+    for (let j = 0; j < 100; j++) {
+      s3.onDataAvailable(largeBlob.slice(0, largeBlob.size / 100));
+    }
+  }
+
+  await s3.completeUpload();
+
+  /**
+   * UploadPartCommand is called once for every 100 blobs added. We do this
+   * twice. Add one more call for when we completeUpload.
+   */
+  expect(UploadPartCommand).toHaveBeenCalledTimes(3);
+
+  expect(CreateMultipartUploadCommand).toHaveBeenCalledTimes(1);
+  expect(CompleteMultipartUploadCommand).toHaveBeenCalledTimes(1);
+});
+
+test("Upload to S3 missing upload id", () => {
+  mockSendRtn = { ETag: "etag" };
+  const s3 = new LookitS3("key value");
+  expect(async () => {
+    await s3.createUpload();
+  }).rejects.toThrow(
+    "Response from AWS send is missing an attribute: UploadId",
+  );
+
+  expect(CreateMultipartUploadCommand).toHaveBeenCalledTimes(1);
+});
+
+test("Upload to S3 missing Etag", async () => {
+  mockSendRtn = { UploadId: "upload id" };
+  const s3 = new LookitS3("key value");
+  await s3.createUpload();
+  s3.onDataAvailable(largeBlob);
+  expect(async () => {
+    await s3.completeUpload();
+  }).rejects.toThrow(
+    "Upload part failed after 3 attempts: AWSMissingAttrError: Response from AWS send is missing an attribute: ETag",
+  );
+
+  expect(UploadPartCommand).toHaveBeenCalledTimes(2);
+  expect(CreateMultipartUploadCommand).toHaveBeenCalledTimes(1);
+  expect(CompleteMultipartUploadCommand).toHaveBeenCalledTimes(0);
+});

packages/data/src/lookitS3.ts

@@ -4,18 +4,20 @@ import {
   S3Client,
   UploadPartCommand,
 } from "@aws-sdk/client-s3";
+import { AWSMissingAttrError, UploadPartError } from "./error";
 import { Env } from "./types";
 
-/** Provides functionality to upload videos incremetally to an AWS S3 Bucket. */
+/** Provides functionality to upload videos incrementally to an AWS S3 Bucket. */
 class LookitS3 {
   private blobParts: Blob[];
-  private promises: Promise<{ PartNumber: number; ETag?: string }>[];
+  private promises: Promise<{ PartNumber: number; ETag: string }>[];
   private partNumber: number;
   private partsUploaded: number;
   private s3: S3Client;
-  private uploadId?: string;
+  private uploadId: string = "";
   private env: Env;
   private key: string;
+  public static readonly minUploadSize: number = 5 * 1024 * 1024;
 
   /**
    * Provide file name to initiate a new upload to a S3 bucket. The AWS secrets
@@ -62,7 +64,7 @@ class LookitS3 {
    * @returns Percent uploaded.
    */
   public get percentUploadComplete() {
-    return Math.floor((this.partsUploaded / this.partNumber) * 100);
+    return Math.floor((this.partsUploaded / this.promises.length) * 100);
   }
 
   /**
@@ -84,7 +86,12 @@ class LookitS3 {
       Key: this.key,
       ContentType: "video/webm", // TO DO: check browser support for type/codec and set the actual value here
     });
+
     const response = await this.s3.send(command);
+    if (!response.UploadId) {
+      throw new AWSMissingAttrError("UploadId");
+    }
+
     this.uploadId = response.UploadId;
     this.logRecordingEvent(`Connection established.`);
   }
@@ -99,25 +106,24 @@ class LookitS3 {
    */
   private async uploadPart(blob: Blob, partNumber: number) {
     let retry = 0;
-    let err;
-
-    if (!this.uploadId) {
-      throw Error("no upload id.");
-    }
+    let err: Error | undefined = undefined;
+    const input = {
+      Body: blob,
+      Bucket: this.env.bucket,
+      Key: this.key,
+      PartNumber: partNumber,
+      UploadId: this.uploadId,
+    };
+    const command = new UploadPartCommand(input);
 
     while (retry < 3) {
       try {
-        const input = {
-          Body: blob,
-          Bucket: this.env.bucket,
-          Key: this.key,
-          PartNumber: partNumber,
-          UploadId: this.uploadId,
-        };
-        const command = new UploadPartCommand(input);
         const response = await this.s3.send(command);
-        this.logRecordingEvent(`Uploaded file part ${partNumber}.`);
+        if (!response.ETag) {
+          throw new AWSMissingAttrError("ETag");
+        }
 
+        this.logRecordingEvent(`Uploaded file part ${partNumber}.`);
         this.partsUploaded++;
 
         return {
@@ -128,11 +134,12 @@ class LookitS3 {
         this.logRecordingEvent(
           `Error uploading part ${partNumber}.\nError: ${_err}`,
         );
-        err = _err;
+        err = _err as Error;
         retry += 1;
       }
     }
-    throw Error(`Upload part failed after 3 attempts.\nError: ${err}`);
+
+    throw new UploadPartError(err);
   }
 
   /**
@@ -142,10 +149,6 @@ class LookitS3 {
   public async completeUpload() {
     this.addUploadPartPromise();
 
-    if (!this.uploadId) {
-      throw Error("No upload id");
-    }
-
     const input = {
       Bucket: this.env.bucket,
       Key: this.key,
@@ -169,7 +172,7 @@ class LookitS3 {
   public onDataAvailable(blob: Blob) {
     this.blobParts.push(blob);
 
-    if (this.blobPartsSize > 5 * (1024 * 1024)) {
+    if (this.blobPartsSize > LookitS3.minUploadSize) {
       this.addUploadPartPromise();
     }
   }