Skip to content

Commit

Permalink
Add s3 class
Browse files Browse the repository at this point in the history
  • Loading branch information
okaycj committed Apr 24, 2024
1 parent cf31d8d commit e2eff44
Show file tree
Hide file tree
Showing 10 changed files with 77 additions and 63 deletions.
1 change: 1 addition & 0 deletions .prettierignore
Original file line number Diff line number Diff line change
@@ -0,0 +1 @@
packages/data/src/s3
15 changes: 14 additions & 1 deletion packages/data/rollup.config.mjs
Original file line number Diff line number Diff line change
@@ -1,3 +1,16 @@
import { makeRollupConfig } from "@jspsych/config/rollup";
import { nodeResolve } from "@rollup/plugin-node-resolve";

export default makeRollupConfig("chsData");
const config = makeRollupConfig("chsData");

config.map((c) => {
c.plugins = [
nodeResolve({
browser: true,
preferBuiltins: false,
}),
...c.plugins,
];
});

export default config;
6 changes: 3 additions & 3 deletions packages/data/src/index.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
import deepFreeze from "deep-freeze-es6";
import {
finish,
retrieveChild,
Expand All @@ -7,6 +6,8 @@ import {
retrieveStudy,
updateResponse,
} from "./api";

import s3 from "./s3";
import { Child, PastSession, Response, Study } from "./types";

declare global {
Expand All @@ -30,8 +31,7 @@ async function load(response_uuid: string) {
response: await retrieveResponse(response_uuid),
},
});
deepFreeze(window.chs);
}
}

export default { load, retrieveResponse, updateResponse, finish };
export default { load, retrieveResponse, updateResponse, finish, s3 };
83 changes: 44 additions & 39 deletions packages/data/src/s3.ts
Original file line number Diff line number Diff line change
@@ -1,18 +1,17 @@
import { S3 } from "@aws-sdk/client-s3";

type Env = { accessKeyId: string; secretAccessKey: string; bucket: string };
import { Env } from "./types";

export default class {
env: Env;
key: string;
blobParts: Blob[];
promises: Promise<{ PartNumber: number; ETag?: string }>[];
partNumber: number;
partsUploaded: number;
s3: S3;
uploadId?: string;

constructor(key: string, s3vars: Env) {
private blobParts: Blob[];
private promises: Promise<{ PartNumber: number; ETag?: string }>[];
private partNumber: number;
private partsUploaded: number;
private s3: S3;
private uploadId?: string;
private env: Env;
private key: string;

public constructor(key: string, s3vars: Env) {
this.env = s3vars;
this.key = key;
this.blobParts = [];
Expand All @@ -28,17 +27,24 @@ export default class {
});
}

get blobPartsSize() {
private get blobPartsSize() {
return this.blobParts.reduce((p, c) => {
return p + c.size;
}, 0);
}

get percentUploadComplete() {
public get percentUploadComplete() {
return Math.floor((this.partsUploaded / this.partNumber) * 100);
}

async createUpload() {
private addUploadPartPromise() {
this.promises.push(
this.uploadPart(new Blob(this.blobParts), this.partNumber++),
);
this.blobParts = [];
}

public async createUpload() {
this.logRecordingEvent(`Creating video upload connection.`);
const createResponse = await this.s3.createMultipartUpload({
Bucket: this.env.bucket,
Expand All @@ -49,9 +55,14 @@ export default class {
this.logRecordingEvent(`Connection established.`);
}

async uploadPart(blob: Blob, partNumber: number) {
public async uploadPart(blob: Blob, partNumber: number) {
let retry = 0;
let err;

if (!this.uploadId) {
throw Error("no upload id.");
}

while (retry < 3) {
try {
const uploadPartResponse = await this.s3.uploadPart({
Expand Down Expand Up @@ -80,40 +91,34 @@ export default class {
throw Error(`Upload part failed after 3 attempts.\nError: ${err}`);
}

async completeUpload() {
public async completeUpload() {
this.addUploadPartPromise();
const parts = await Promise.all(this.promises);

return this.s3
.completeMultipartUpload({
Bucket: this.env.bucket,
Key: this.key,
MultipartUpload: {
Parts: parts,
},
UploadId: this.uploadId,
})
.then((resp) => {
this.logRecordingEvent(`Upload complete: ${resp.Location}`);
});
}

addUploadPartPromise() {
this.promises.push(
this.uploadPart(new Blob(this.blobParts), this.partNumber++),
);
this.blobParts = [];
if (!this.uploadId) {
throw Error("no upload id");
}

const resp = await this.s3.completeMultipartUpload({
Bucket: this.env.bucket,
Key: this.key,
MultipartUpload: {
Parts: await Promise.all(this.promises),
},
UploadId: this.uploadId,
});

this.logRecordingEvent(`Upload complete: ${resp.Location}`);
}

onDataAvailable(blob: Blob) {
public onDataAvailable(blob: Blob) {
this.blobParts.push(blob);

if (this.blobPartsSize > 5 * (1024 * 1024)) {
this.addUploadPartPromise();
}
}

logRecordingEvent(msg: string) {
public logRecordingEvent(msg: string) {
// right now this just prints to the console, but we could also send this info to permanent storage (similar to pipe logs)
const timestamp = new Date().toISOString();
console.log(`Recording log: ${timestamp}\nFile: ${this.key}\n${msg}\n`);
Expand Down
6 changes: 6 additions & 0 deletions packages/data/src/types.ts
Original file line number Diff line number Diff line change
Expand Up @@ -139,3 +139,9 @@ export interface ResponseAttrsUpdate {
survey_consent?: boolean;
completed_consent_frame?: boolean;
}

export type Env = {
accessKeyId: string;
secretAccessKey: string;
bucket: string;
};
7 changes: 4 additions & 3 deletions packages/data/src/utils.spec.ts
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,10 @@ function setLocationHref(href: string) {
* Helper function for this set of tests. This will update the current URL to the value
* provided in argument href.
*/
delete global.window.location;
global.window = Object.create(window);
global.window.location = { href };
// delete global.window.location;
// global.window = Object.create(window);
// global.window.location = { href };
return href;
}

test("Api get function", async () => {
Expand Down
2 changes: 1 addition & 1 deletion packages/lookit-initjspsych/src/utils.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import Api from "@lookit/data";
import { DataCollection } from "jspsych/dist/modules/data/DataCollection";
import Api from "../../data/dist";
import { UserFunc } from "./types";

export function on_data_update(responseUuid: string, userFunc?: UserFunc) {
Expand Down
4 changes: 2 additions & 2 deletions packages/surveys/src/consent.ts
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,8 @@ type Info = typeof info;
type Trial = TrialType<Info>;

export class ConsentSurveyPlugin extends SurveyPlugin {
static readonly info = info;
trial(display_element: HTMLElement, trial: Trial) {
public static readonly info = info;
public trial(display_element: HTMLElement, trial: Trial) {
super.trial(display_element, {
...trial,
survey_function: survey_function(trial.survey_function),
Expand Down
4 changes: 2 additions & 2 deletions packages/surveys/src/exit.ts
Original file line number Diff line number Diff line change
Expand Up @@ -100,8 +100,8 @@ function surveyParameters(trial: Trial) {
}

export class ExitSurveyPlugin extends SurveyPlugin {
static readonly info = info;
trial(display_element: HTMLElement, trial: Trial) {
public static readonly info = info;
public trial(display_element: HTMLElement, trial: Trial) {
super.trial(display_element, {
...trial,
survey_json: surveyParameters(trial),
Expand Down
12 changes: 0 additions & 12 deletions packages/surveys/src/index.ts
Original file line number Diff line number Diff line change
@@ -1,16 +1,4 @@
import { Child, PastSession, Response, Study } from "@lookit/data/dist/types";
import { ConsentSurveyPlugin } from "./consent";
import { ExitSurveyPlugin } from "./exit";

declare global {
interface Window {
chs: {
study: Study;
child: Child;
pastSessions: PastSession[];
response: Response;
};
}
}

export default { exit: ExitSurveyPlugin, consent: ConsentSurveyPlugin };

0 comments on commit e2eff44

Please sign in to comment.