From 574a4331a46a3726c13dfc5d093889d940a59e86 Mon Sep 17 00:00:00 2001 From: Becky Gilbert Date: Thu, 24 Oct 2024 11:28:56 -0700 Subject: [PATCH 1/6] move AWS credentials from .env to lookit-api template, get credentials from document, throw errors for expired credentials --- packages/data/rollup.config.mjs | 3 - packages/data/src/errors.ts | 34 +++++++- packages/data/src/lookitS3.spec.ts | 11 +++ packages/data/src/lookitS3.ts | 103 ++++++++++++++++++----- packages/data/src/lookitS3config.spec.ts | 22 ++++- packages/data/src/types.ts | 9 ++ 6 files changed, 153 insertions(+), 29 deletions(-) diff --git a/packages/data/rollup.config.mjs b/packages/data/rollup.config.mjs index 010f204b..1d7b64a6 100644 --- a/packages/data/rollup.config.mjs +++ b/packages/data/rollup.config.mjs @@ -1,5 +1,4 @@ import { nodeResolve } from "@rollup/plugin-node-resolve"; -import dotenv from "rollup-plugin-dotenv"; import { makeRollupConfig } from "../../rollup.mjs"; export default makeRollupConfig("chsData").map((config) => { @@ -8,8 +7,6 @@ export default makeRollupConfig("chsData").map((config) => { plugins: [ // Resolve node dependencies to be used in a browser. nodeResolve({ browser: true, preferBuiltins: false }), - // Add support for .env files - dotenv(), ...config.plugins, ], }; diff --git a/packages/data/src/errors.ts b/packages/data/src/errors.ts index 6200db53..8e3b88aa 100644 --- a/packages/data/src/errors.ts +++ b/packages/data/src/errors.ts @@ -30,13 +30,18 @@ export class AWSConfigError extends Error { * AWS cofiguration error. This could be due to incorrect credentials, bucket * name, and/or region. * - * @param errorMsg - Message property of error object from the AWS response. + * @param error - Error object from the AWS response. */ - public constructor(errorMsg: string) { - super(`AWS configuration error: ${errorMsg}`); + public constructor(error: unknown) { + let err_msg = ""; + if (error instanceof Error) { + err_msg = error.message; + } + super(`AWS configuration error: ${err_msg}`); this.name = "AWSConfigError"; } } + /** Error for when URL from Django API is not formatted as expected. */ export class URLWrongError extends Error { /** Throw error when URL is not formatted correctly. */ @@ -45,3 +50,26 @@ export class URLWrongError extends Error { this.name = "URLWrongError"; } } + +/** Error for when temporary AWS credentials are not found on the page. */ +export class MissingCredentials extends Error { + /** Throw error when AWS credentials are not found. */ + public constructor() { + super("AWS credentials for video uploading not found."); + this.name = "MissingCredentials"; + } +} + +/** Error for when temporary AWS credentials have expired. */ +export class ExpiredCredentials extends Error { + /** Throw error when temporary AWS credentials have expired. */ + public constructor() { + super( + "The video upload credentials have expired. Please re-start the experiment on the CHS website.", + ); + alert( + "Your credentials have expired. Please re-start the experiment on the CHS website.", + ); + this.name = "ExpiredCredentials"; + } +} diff --git a/packages/data/src/lookitS3.spec.ts b/packages/data/src/lookitS3.spec.ts index 6287ec4d..fc75df91 100644 --- a/packages/data/src/lookitS3.spec.ts +++ b/packages/data/src/lookitS3.spec.ts @@ -17,6 +17,17 @@ jest.mock("@aws-sdk/client-s3", () => ({ CompleteMultipartUploadCommand: jest.fn().mockImplementation(), })); +// Mock the retrieval of AWS variables passed from lookit-api into the jsPsych study template. The variables are stored in a script element that is retrieved with document.getElementById in the LookitS3 constructor. This mock is currently needed for all tests in this file, but could potentially cause problems if document.getElementById is used for other purposes during these tests. +beforeEach(() => { + Object.assign(document, { + getElementById: jest.fn().mockImplementation(() => { + return { + textContent: '{"JSPSYCH_S3_REGION":"region","JSPSYCH_S3_ACCESS_KEY_ID":"keyId","JSPSYCH_S3_SECRET_ACCESS_KEY":"key","JSPSYCH_S3_BUCKET":"bucket","JSPSYCH_S3_SESSION_TOKEN":"token","JSPSYCH_S3_EXPIRATION":"datestring"}' + } + }) + }); +}); + afterEach(() => { jest.clearAllMocks(); }); diff --git a/packages/data/src/lookitS3.ts b/packages/data/src/lookitS3.ts index cfb7c6ce..104ca0dc 100644 --- a/packages/data/src/lookitS3.ts +++ b/packages/data/src/lookitS3.ts @@ -4,7 +4,14 @@ import { S3Client, UploadPartCommand, } from "@aws-sdk/client-s3"; -import { AWSConfigError, AWSMissingAttrError, UploadPartError } from "./errors"; +import { + AWSConfigError, + AWSMissingAttrError, + ExpiredCredentials, + MissingCredentials, + UploadPartError, +} from "./errors"; +import { awsVars } from "./types"; /** Provides functionality to upload videos incrementally to an AWS S3 Bucket. */ class LookitS3 { @@ -15,7 +22,8 @@ class LookitS3 { private s3: S3Client; private uploadId: string = ""; private key: string; - private bucket: string = process.env.JSPSYCH_S3_BUCKET; + private bucket: string; + private awsVars: awsVars; private complete: boolean = false; public static readonly minUploadSize: number = 5 * 1024 * 1024; @@ -29,20 +37,29 @@ class LookitS3 { public constructor(key: string) { this.key = key; try { + this.awsVars = JSON.parse( + document.getElementById("aws-vars")!.textContent!, + ); + } catch { + throw new MissingCredentials(); + } + try { + this.bucket = this.awsVars.JSPSYCH_S3_BUCKET; this.s3 = new S3Client({ - region: process.env.JSPSYCH_S3_REGION, + region: this.awsVars.JSPSYCH_S3_REGION, credentials: { - accessKeyId: process.env.JSPSYCH_S3_ACCESS_KEY_ID, - secretAccessKey: process.env.JSPSYCH_S3_SECRET_ACCESS_KEY, + accessKeyId: this.awsVars.JSPSYCH_S3_ACCESS_KEY_ID, + secretAccessKey: this.awsVars.JSPSYCH_S3_SECRET_ACCESS_KEY, + sessionToken: this.awsVars.JSPSYCH_S3_SESSION_TOKEN, }, }); } catch (e) { - console.error(`Error setting up S3 client: ${e}`); - let err_msg = ""; - if (e instanceof Error) { - err_msg = e.message; + this.logRecordingEvent(`Error setting up the S3 client.\nError: ${e}`); + if (this.awsExpiredToken(e)) { + throw new ExpiredCredentials(); + } else { + throw new AWSConfigError(e); } - throw new AWSConfigError(err_msg); } } @@ -86,13 +103,23 @@ class LookitS3 { ContentType: "video/webm", // TO DO: check browser support for type/codec and set the actual value here }); - const response = await this.s3.send(command); - if (!response.UploadId) { - throw new AWSMissingAttrError("UploadId"); + try { + const response = await this.s3.send(command); + if (!response.UploadId) { + throw new AWSMissingAttrError("UploadId"); + } + this.uploadId = response.UploadId; + this.logRecordingEvent(`Connection established.`); + } catch (error) { + this.logRecordingEvent( + `Error creating upload ${this.key}.\nError: ${error}`, + ); + if (this.awsExpiredToken(error)) { + throw new ExpiredCredentials(); + } else { + throw error; + } } - - this.uploadId = response.UploadId; - this.logRecordingEvent(`Connection established.`); } /** @@ -133,8 +160,12 @@ class LookitS3 { this.logRecordingEvent( `Error uploading part ${partNumber}.\nError: ${_err}`, ); - err = _err as Error; - retry += 1; + if (this.awsExpiredToken(_err)) { + throw new ExpiredCredentials(); + } else { + err = _err as Error; + retry += 1; + } } } @@ -157,10 +188,20 @@ class LookitS3 { UploadId: this.uploadId, }; const command = new CompleteMultipartUploadCommand(input); - const response = await this.s3.send(command); - this.complete = true; - - this.logRecordingEvent(`Upload complete: ${response.Location}`); + try { + const response = await this.s3.send(command); + this.complete = true; + this.logRecordingEvent(`Upload complete: ${response.Location}`); + } catch (error) { + this.logRecordingEvent( + `Error completing upload ${this.key}.\nError: ${error}`, + ); + if (this.awsExpiredToken(error)) { + throw new ExpiredCredentials(); + } else { + throw error; + } + } } /** @@ -197,6 +238,24 @@ class LookitS3 { public get uploadInProgress(): boolean { return this.uploadId !== "" && !this.complete; } + + /** + * Check whether an AWS S3 error is due to expired credentials/token. + * + * @param error - Error returned from S3. + * @returns Boolean indicating whether or not this is an Expired Token error. + */ + private awsExpiredToken(error: unknown) { + if ( + error instanceof Object && + "name" in error && + error.name === "ExpiredTokenException" + ) { + return true; + } else { + return false; + } + } } export default LookitS3; diff --git a/packages/data/src/lookitS3config.spec.ts b/packages/data/src/lookitS3config.spec.ts index eb9eedbd..25c7d71c 100644 --- a/packages/data/src/lookitS3config.spec.ts +++ b/packages/data/src/lookitS3config.spec.ts @@ -1,3 +1,4 @@ +import { AWSConfigError } from "./errors"; import LookitS3 from "./lookitS3"; // This is in a separate file because imports can only be mocked once per file, at the top level (not inside test functions), and the config failure test requires a different mock than the rest of the lookitS3 tests. @@ -7,8 +8,27 @@ jest.mock("@aws-sdk/client-s3", () => ({ }), })); +// Mock the retrieval of AWS variables passed from lookit-api into the jsPsych study template. The variables are stored in a script element that is retrieved with document.getElementById in the LookitS3 constructor. This mock is currently needed for all tests in this file, but could potentially cause problems if document.getElementById is used for other purposes during these tests. +beforeEach(() => { + Object.assign(document, { + getElementById: jest.fn().mockImplementation(() => { + return { + textContent: + '{"JSPSYCH_S3_REGION":"region","JSPSYCH_S3_ACCESS_KEY_ID":"keyId","JSPSYCH_S3_SECRET_ACCESS_KEY":"key","JSPSYCH_S3_BUCKET":"bucket","JSPSYCH_S3_SESSION_TOKEN":"token","JSPSYCH_S3_EXPIRATION":"datestring"}', + }; + }), + }); +}); + +afterEach(() => { + jest.clearAllMocks(); +}); + test.only("Lookit S3 constructor throws error when S3 Client initialization fails", () => { expect(() => { new LookitS3("key value"); - }).toThrow(); + }).toThrow(AWSConfigError); + expect(() => { + new LookitS3("key value"); + }).toThrow("AWS configuration error: Error"); }); diff --git a/packages/data/src/types.ts b/packages/data/src/types.ts index 1f800d29..56b4ed64 100644 --- a/packages/data/src/types.ts +++ b/packages/data/src/types.ts @@ -140,3 +140,12 @@ export interface LookitWindow extends Window { sessionRecorder: unknown; }; } + +export interface awsVars { + JSPSYCH_S3_REGION: string; + JSPSYCH_S3_ACCESS_KEY_ID: string; + JSPSYCH_S3_SECRET_ACCESS_KEY: string; + JSPSYCH_S3_BUCKET: string; + JSPSYCH_S3_SESSION_TOKEN: string; + JSPSYCH_S3_EXPIRATION: string; +} From eac2333c9918bb7fb1262518bb326ca86db7a1a9 Mon Sep 17 00:00:00 2001 From: Becky Gilbert Date: Fri, 25 Oct 2024 12:58:10 -0700 Subject: [PATCH 2/6] move creation of input and command into completeUpload try/catch in case one of the uploadPart promises throws an error, and catch an ExpiredCredentials error from uploadPart in the completeUpload catch block --- packages/data/src/lookitS3.ts | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/packages/data/src/lookitS3.ts b/packages/data/src/lookitS3.ts index 104ca0dc..b5ac716e 100644 --- a/packages/data/src/lookitS3.ts +++ b/packages/data/src/lookitS3.ts @@ -179,16 +179,16 @@ class LookitS3 { public async completeUpload() { this.addUploadPartPromise(); - const input = { - Bucket: this.bucket, - Key: this.key, - MultipartUpload: { - Parts: await Promise.all(this.promises), - }, - UploadId: this.uploadId, - }; - const command = new CompleteMultipartUploadCommand(input); try { + const input = { + Bucket: this.bucket, + Key: this.key, + MultipartUpload: { + Parts: await Promise.all(this.promises), + }, + UploadId: this.uploadId, + }; + const command = new CompleteMultipartUploadCommand(input); const response = await this.s3.send(command); this.complete = true; this.logRecordingEvent(`Upload complete: ${response.Location}`); @@ -196,7 +196,7 @@ class LookitS3 { this.logRecordingEvent( `Error completing upload ${this.key}.\nError: ${error}`, ); - if (this.awsExpiredToken(error)) { + if (this.awsExpiredToken(error) || error instanceof ExpiredCredentials) { throw new ExpiredCredentials(); } else { throw error; From e157220108100bed8b64a423f6216f9f0dfec66a Mon Sep 17 00:00:00 2001 From: Becky Gilbert Date: Fri, 25 Oct 2024 13:05:06 -0700 Subject: [PATCH 3/6] add tests for missing and expired credentials in LookitS3 constructor --- packages/data/src/lookitS3config.spec.ts | 55 +++++++++++++++++++++++- 1 file changed, 53 insertions(+), 2 deletions(-) diff --git a/packages/data/src/lookitS3config.spec.ts b/packages/data/src/lookitS3config.spec.ts index 25c7d71c..e566f074 100644 --- a/packages/data/src/lookitS3config.spec.ts +++ b/packages/data/src/lookitS3config.spec.ts @@ -1,6 +1,24 @@ -import { AWSConfigError } from "./errors"; +import * as awsSdk from "@aws-sdk/client-s3"; +import { + AWSConfigError, + ExpiredCredentials, + MissingCredentials, +} from "./errors"; import LookitS3 from "./lookitS3"; +/** Mock for the AWS expired token error */ +class MockExpiredTokenError extends Error { + /** + * Constructor + * + * @param message - Error message. + */ + public constructor(message: string) { + super(message); + this.name = "ExpiredTokenException"; + } +} + // This is in a separate file because imports can only be mocked once per file, at the top level (not inside test functions), and the config failure test requires a different mock than the rest of the lookitS3 tests. jest.mock("@aws-sdk/client-s3", () => ({ S3Client: jest.fn().mockImplementation(() => { @@ -24,7 +42,7 @@ afterEach(() => { jest.clearAllMocks(); }); -test.only("Lookit S3 constructor throws error when S3 Client initialization fails", () => { +test("Lookit S3 constructor throws error when S3 Client initialization fails", () => { expect(() => { new LookitS3("key value"); }).toThrow(AWSConfigError); @@ -32,3 +50,36 @@ test.only("Lookit S3 constructor throws error when S3 Client initialization fail new LookitS3("key value"); }).toThrow("AWS configuration error: Error"); }); + +test("Lookit S3 constructor throws missing credentials error when AWS variables are not found", () => { + Object.assign(document, { + getElementById: jest.fn().mockImplementation(() => { + return undefined; + }), + }); + expect(() => { + new LookitS3("key value"); + }).toThrow(MissingCredentials); + expect(() => { + new LookitS3("key value"); + }).toThrow("AWS credentials for video uploading not found."); +}); + +test("Lookit S3 constructor throws expired credentials error when AWS token is expired", () => { + (awsSdk.S3Client as jest.Mock).mockImplementation(() => { + throw new MockExpiredTokenError(""); + }); + window.alert = jest.fn(); + expect(() => { + new LookitS3("key value"); + }).toThrow(ExpiredCredentials); + expect(() => { + new LookitS3("key value"); + }).toThrow( + "The video upload credentials have expired. Please re-start the experiment on the CHS website.", + ); + expect(window.alert).toHaveBeenCalledTimes(2); + expect(window.alert).toHaveBeenCalledWith( + "Your credentials have expired. Please re-start the experiment on the CHS website.", + ); +}); From aefa71fa22597d6fb02b08a979c6686be9a1c3db Mon Sep 17 00:00:00 2001 From: Becky Gilbert Date: Fri, 25 Oct 2024 13:06:43 -0700 Subject: [PATCH 4/6] add tests for expired credentials errors in createUpload, uploadPart, and completeUpload --- packages/data/src/lookitS3.spec.ts | 96 +++++++++++++++++++++++++++++- 1 file changed, 93 insertions(+), 3 deletions(-) diff --git a/packages/data/src/lookitS3.spec.ts b/packages/data/src/lookitS3.spec.ts index fc75df91..537ecdfb 100644 --- a/packages/data/src/lookitS3.spec.ts +++ b/packages/data/src/lookitS3.spec.ts @@ -3,8 +3,22 @@ import { CreateMultipartUploadCommand, UploadPartCommand, } from "@aws-sdk/client-s3"; +import { ExpiredCredentials } from "./errors"; import LookitS3 from "./lookitS3"; +/** Mock for the AWS expired token error that can be received from s3.send. */ +class MockExpiredTokenError extends Error { + /** + * Constructor + * + * @param message - Error message. + */ + public constructor(message: string) { + super(message); + this.name = "ExpiredTokenException"; + } +} + let mockSendRtn: { UploadId?: string; ETag?: string }; const largeBlob = new Blob(["x".repeat(LookitS3.minUploadSize + 1)]); @@ -22,9 +36,10 @@ beforeEach(() => { Object.assign(document, { getElementById: jest.fn().mockImplementation(() => { return { - textContent: '{"JSPSYCH_S3_REGION":"region","JSPSYCH_S3_ACCESS_KEY_ID":"keyId","JSPSYCH_S3_SECRET_ACCESS_KEY":"key","JSPSYCH_S3_BUCKET":"bucket","JSPSYCH_S3_SESSION_TOKEN":"token","JSPSYCH_S3_EXPIRATION":"datestring"}' - } - }) + textContent: + '{"JSPSYCH_S3_REGION":"region","JSPSYCH_S3_ACCESS_KEY_ID":"keyId","JSPSYCH_S3_SECRET_ACCESS_KEY":"key","JSPSYCH_S3_BUCKET":"bucket","JSPSYCH_S3_SESSION_TOKEN":"token","JSPSYCH_S3_EXPIRATION":"datestring"}', + }; + }), }); }); @@ -113,3 +128,78 @@ test("Upload in progress", async () => { await s3.completeUpload(); expect(s3.uploadInProgress).toBe(false); }); + +test("Create upload throws expired credentials error", () => { + window.alert = jest.fn(); + const s3 = new LookitS3("key value"); + + // Implement the token expired error for s3.send + s3["s3"]["send"] = jest.fn(() => { + throw new MockExpiredTokenError(""); + }); + + expect(async () => await s3.createUpload()).rejects.toThrow( + ExpiredCredentials, + ); + expect(async () => await s3.createUpload()).rejects.toThrow( + "The video upload credentials have expired. Please re-start the experiment on the CHS website.", + ); + expect(window.alert).toHaveBeenCalledTimes(2); + expect(window.alert).toHaveBeenCalledWith( + "Your credentials have expired. Please re-start the experiment on the CHS website.", + ); +}); + +test("Upload part throws expired credentials error", async () => { + window.alert = jest.fn(); + mockSendRtn = { UploadId: "upload id", ETag: "etag" }; + const s3 = new LookitS3("key value"); + + // Create the upload without errors + await s3.createUpload(); + s3.onDataAvailable(largeBlob); + + // Implement the token expired error for s3.send + s3["s3"]["send"] = jest.fn(() => { + throw new MockExpiredTokenError(""); + }); + + // completeUpload calls uploadPart + expect(async () => { + await s3.completeUpload(); + }).rejects.toThrow(ExpiredCredentials); + expect(async () => { + await s3.completeUpload(); + }).rejects.toThrow( + "The video upload credentials have expired. Please re-start the experiment on the CHS website.", + ); + expect(window.alert).toHaveBeenCalledTimes(2); + expect(window.alert).toHaveBeenCalledWith( + "Your credentials have expired. Please re-start the experiment on the CHS website.", + ); +}); + +test("Complete upload throws expired credentials", async () => { + mockSendRtn = { UploadId: "upload id", ETag: "etag" }; + const s3 = new LookitS3("key value"); + await s3.createUpload(); + + // Mock addUploadPartPromise because completeUpload will trigger uploadPart and s3.send for the part, before sending the complete upload command. If we just mock the s3.send error and call completeUpload, the error will always be thrown for uploadPart and the code will never reach the second s3.send command to complete the upload. + s3["uploadPart"] = jest.fn().mockImplementation(() => { + return Promise.resolve({ PartNumber: 1, ETag: "etag" }); + }); + + // Implement the token expired error for s3.send + s3["s3"]["send"] = jest.fn(() => { + throw new MockExpiredTokenError(""); + }); + + expect(async () => { + await s3.completeUpload(); + }).rejects.toThrow(ExpiredCredentials); + expect(async () => { + await s3.completeUpload(); + }).rejects.toThrow( + "The video upload credentials have expired. Please re-start the experiment on the CHS website.", + ); +}); From 6c60ded2c89f03273547a0456310d288ed25fba4 Mon Sep 17 00:00:00 2001 From: Becky Gilbert Date: Fri, 25 Oct 2024 13:42:21 -0700 Subject: [PATCH 5/6] add changeset --- .changeset/slimy-grapes-peel.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/slimy-grapes-peel.md diff --git a/.changeset/slimy-grapes-peel.md b/.changeset/slimy-grapes-peel.md new file mode 100644 index 00000000..2bd99d69 --- /dev/null +++ b/.changeset/slimy-grapes-peel.md @@ -0,0 +1,5 @@ +--- +"@lookit/data": minor +--- + +Move AWS secrets for video uploading from .env to lookit-api. From 7fc4b24696bfa98e91cb606cedc35f9d8fb249ce Mon Sep 17 00:00:00 2001 From: CJ Green <44074998+okaycj@users.noreply.github.com> Date: Tue, 29 Oct 2024 09:56:12 -0400 Subject: [PATCH 6/6] Removed unneeded environment type file --- packages/data/src/environment.d.ts | 12 ------------ 1 file changed, 12 deletions(-) delete mode 100644 packages/data/src/environment.d.ts diff --git a/packages/data/src/environment.d.ts b/packages/data/src/environment.d.ts deleted file mode 100644 index ced33c27..00000000 --- a/packages/data/src/environment.d.ts +++ /dev/null @@ -1,12 +0,0 @@ -declare global { - namespace NodeJS { - interface ProcessEnv { - JSPSYCH_S3_REGION: string; - JSPSYCH_S3_ACCESS_KEY_ID: string; - JSPSYCH_S3_SECRET_ACCESS_KEY: string; - JSPSYCH_S3_BUCKET: string; - } - } -} - -export {};