From 590d538f0ffa512b7c8ad87668d145a8245d68bf Mon Sep 17 00:00:00 2001 From: Arthur de Moulins Date: Mon, 6 Nov 2023 15:32:17 +0100 Subject: [PATCH] PS-586 fix upload resume --- uploader/client/src/multiPartUpload.js | 147 ++++++++++------------ uploader/client/src/uploadStateStorage.js | 4 +- 2 files changed, 71 insertions(+), 80 deletions(-) diff --git a/uploader/client/src/multiPartUpload.js b/uploader/client/src/multiPartUpload.js index bd4dd7d65..80ca12507 100644 --- a/uploader/client/src/multiPartUpload.js +++ b/uploader/client/src/multiPartUpload.js @@ -6,100 +6,91 @@ const fileChunkSize = 5242880 // 5242880 is the minimum allowed by AWS S3; export async function uploadMultipartFile(targetId, userId, file, onProgress) { const fileUID = getUniqueFileId(file.file, fileChunkSize); - try { - const resumableUpload = uploadStateStorage.getUpload(userId, fileUID); - const uploadParts = []; + const resumableUpload = uploadStateStorage.getUpload(userId, fileUID); + const uploadParts = []; - let resumeChunkIndex = 1, - uploadId, - path; - - if (resumableUpload) { - uploadId = resumableUpload.u; - resumeChunkIndex = resumableUpload.c.length + 1; - for (let i = 0; i < resumableUpload.c.length; i++) { - uploadParts.push({ - ETag: resumableUpload.c[i], - PartNumber: i + 1, - }); - } - } else { - file.abortController = new AbortController(); - - const res = await apiClient.post(`/uploads`, { - filename: file.file.name, - type: file.file.type, - size: file.file.size, - }, { - signal: file.abortController.signal, - }); - console.debug('res', res); - uploadId = res.data.id; - path = res.data.path; - uploadStateStorage.initUpload(userId, fileUID, uploadId, path); - } + let resumeChunkIndex = 1, + uploadId; - const fileSize = file.file.size; - const numChunks = Math.floor(fileSize / fileChunkSize) + 1; - - for (let index = resumeChunkIndex; index < numChunks + 1; index++) { - const start = (index - 1) * fileChunkSize; - const end = (index) * fileChunkSize; - - file.abortController = new AbortController(); - - const getUploadUrlResp = await apiClient.post(`/uploads/${uploadId}/part`, { - part: index, - }, { - signal: file.abortController.signal, + if (resumableUpload) { + uploadId = resumableUpload.u; + resumeChunkIndex = resumableUpload.c.length + 1; + for (let i = 0; i < resumableUpload.c.length; i++) { + uploadParts.push({ + ETag: resumableUpload.c[i], + PartNumber: i + 1, }); - console.debug('getUploadUrlResp', getUploadUrlResp); + } + } else { + file.abortController = new AbortController(); - const {url} = getUploadUrlResp.data; + const res = await apiClient.post(`/uploads`, { + filename: file.file.name, + type: file.file.type, + size: file.file.size, + }, { + signal: file.abortController.signal, + }); + uploadId = res.data.id; + uploadStateStorage.initUpload(userId, fileUID, uploadId); + } - const blob = (index < numChunks) ? file.file.slice(start, end) : file.file.slice(start); + const fileSize = file.file.size; + const numChunks = Math.floor(fileSize / fileChunkSize) + 1; - file.abortController = new AbortController(); + for (let index = resumeChunkIndex; index < numChunks + 1; index++) { + const start = (index - 1) * fileChunkSize; + const end = (index) * fileChunkSize; - const uploadResp = await apiClient.put(url, blob, { - signal: file.abortController.signal, - anonymous: true, - onUploadProgress: (e) => { - const multiPartEvent = { - ...e, - loaded: e.loaded + start, - }; + file.abortController = new AbortController(); - onProgress(multiPartEvent); - } - }); + const getUploadUrlResp = await apiClient.post(`/uploads/${uploadId}/part`, { + part: index, + }, { + signal: file.abortController.signal, + }); - const eTag = uploadResp.headers.etag; - uploadParts.push({ - ETag: eTag, - PartNumber: index, - }); + const {url} = getUploadUrlResp.data; - uploadStateStorage.updateUpload(userId, fileUID, eTag); - } + const blob = (index < numChunks) ? file.file.slice(start, end) : file.file.slice(start); file.abortController = new AbortController(); - const finalRes = await apiClient.post(`/assets`, { - targetId, - multipart: { - uploadId, - parts: uploadParts, - } - }, { + const uploadResp = await apiClient.put(url, blob, { signal: file.abortController.signal, + anonymous: true, + onUploadProgress: (e) => { + const multiPartEvent = { + ...e, + loaded: e.loaded + start, + }; + + onProgress(multiPartEvent); + } }); - uploadStateStorage.removeUpload(userId, fileUID); + const eTag = uploadResp.headers.etag; + uploadParts.push({ + ETag: eTag, + PartNumber: index, + }); - return finalRes; - } catch (e) { - uploadStateStorage.removeUpload(userId, fileUID); - throw e; + uploadStateStorage.updateUpload(userId, fileUID, eTag); } + + file.abortController = new AbortController(); + + const finalRes = await apiClient.post(`/assets`, { + targetId, + multipart: { + uploadId, + parts: uploadParts, + } + }, { + signal: file.abortController.signal, + }); + + uploadStateStorage.removeUpload(userId, fileUID); + + return finalRes; } diff --git a/uploader/client/src/uploadStateStorage.js b/uploader/client/src/uploadStateStorage.js index 1bc8306ee..9aecab974 100644 --- a/uploader/client/src/uploadStateStorage.js +++ b/uploader/client/src/uploadStateStorage.js @@ -15,13 +15,12 @@ class UploadStateStorage { return d[userId][fileUID]; } - initUpload(userId, fileUID, uploadId, path) { + initUpload(userId, fileUID, uploadId) { const d = this.getData(); d[userId] = d[userId] || {}; d[userId][fileUID] = { u: uploadId, - p: path, c: [], }; @@ -29,6 +28,7 @@ class UploadStateStorage { } updateUpload(userId, fileUID, chunkETag) { + console.debug('updateUpload', userId, fileUID, chunkETag); const d = this.getData(); d[userId][fileUID].c.push(chunkETag); this.setData(d);