From b0d122de0dc47d152f100d60936aef11732a8dad Mon Sep 17 00:00:00 2001 From: Ammar Dodin <8563179+ammardodin@users.noreply.github.com> Date: Wed, 17 Jul 2019 11:32:38 -0400 Subject: [PATCH] parallelize file uploads (#7) * parallelize file uploads --- src/__specs__/cli.spec.js | 149 ++++++++++++++++++++++++-------------- src/commands/upload.js | 5 +- 2 files changed, 99 insertions(+), 55 deletions(-) diff --git a/src/__specs__/cli.spec.js b/src/__specs__/cli.spec.js index f391642..dd35a31 100644 --- a/src/__specs__/cli.spec.js +++ b/src/__specs__/cli.spec.js @@ -47,6 +47,16 @@ describe('CLI dispatch tests', function cliTests() { }; before(() => { + const simplifyRequest = req => { + const simplifiedRequest = { + method: req.method, + body: req.body, + }; + if (req.headers && req.headers.authorization) { + simplifiedRequest.headers = { authorization: req.headers.authorization }; + } + return simplifiedRequest; + }; server = createServer(async (req, res) => { const parts = parse(req.url); @@ -55,7 +65,7 @@ describe('CLI dispatch tests', function cliTests() { const req2 = req; req2.body = body.toString(); - matchedRequests.push(req2); + matchedRequests.push(simplifyRequest(req2)); const request = expectRequests[parts.pathname]; @@ -269,29 +279,43 @@ describe('CLI dispatch tests', function cliTests() { expect(matchedRequests).to.have.length(6 + 1); expect(unmatchedRequests).to.have.length(0); - const [s, r1, u1, r2, u2, r3, u3] = matchedRequests; - expect(s.method).to.equal('GET'); - - expect(r1.method).to.equal('POST'); - expect(r1.headers).to.have.property('authorization', 'Token org:app:secret'); - expect(r1.body).to.equal('{"filepath":"~/subdir/one.js"}'); - - expect(u1.method).to.equal('PUT'); - expect(u1.body).to.equal('\'one js contents\';\n'); - - expect(r2.method).to.equal('POST'); - expect(r2.headers).to.have.property('authorization', 'Token org:app:secret'); - expect(r2.body).to.equal('{"filepath":"~/two.js.map"}'); - - expect(u2.method).to.equal('PUT'); - expect(u2.body).to.equal('two map contents\n'); - - expect(r3.method).to.equal('POST'); - expect(r3.headers).to.have.property('authorization', 'Token org:app:secret'); - expect(r3.body).to.equal('{"filepath":"~/two.jsx"}'); - - expect(u3.method).to.equal('PUT'); - expect(u3.body).to.equal('\'two jsx contents\';\n'); + expect(matchedRequests[0].method).to.equal('GET'); + + expect(matchedRequests).to.deep.include.members([ + { + method: 'POST', + headers: { + authorization: 'Token org:app:secret', + }, + body: '{"filepath":"~/subdir/one.js"}', + }, + { + method: 'POST', + headers: { + authorization: 'Token org:app:secret', + }, + body: '{"filepath":"~/two.js.map"}', + }, + { + method: 'POST', + headers: { + authorization: 'Token org:app:secret', + }, + body: '{"filepath":"~/two.jsx"}', + }, + { + method: 'PUT', + body: '\'one js contents\';\n', + }, + { + method: 'PUT', + body: 'two map contents\n', + }, + { + method: 'PUT', + body: '\'two jsx contents\';\n', + }] + ); })); it('should support a custom url prefix', mochaAsync(async () => { @@ -310,19 +334,28 @@ describe('CLI dispatch tests', function cliTests() { expect(matchedRequests).to.have.length(6 + 1); expect(unmatchedRequests).to.have.length(0); - const [s, r1, u1, r2] = matchedRequests; - expect(s.method).to.equal('GET'); - - expect(r1.method).to.equal('POST'); - expect(r1.headers).to.have.property('authorization', 'Token org:app:secret'); - expect(r1.body).to.equal('{"filepath":"~/public/subdir/one.js"}'); - - expect(u1.method).to.equal('PUT'); - expect(u1.body).to.equal('\'one js contents\';\n'); - - expect(r2.method).to.equal('POST'); - expect(r2.headers).to.have.property('authorization', 'Token org:app:secret'); - expect(r2.body).to.equal('{"filepath":"~/public/two.js.map"}'); + expect(matchedRequests[0].method).to.equal('GET'); + + expect(matchedRequests).to.deep.include.members([ + { + method: 'POST', + headers: { + authorization: 'Token org:app:secret', + }, + body: '{"filepath":"~/public/subdir/one.js"}', + }, + { + method: 'PUT', + body: '\'one js contents\';\n', + }, + { + method: 'POST', + headers: { + authorization: 'Token org:app:secret', + }, + body: '{"filepath":"~/public/two.js.map"}', + }, + ]); })); it('should upload the passed file', mochaAsync(async () => { @@ -369,22 +402,32 @@ describe('CLI dispatch tests', function cliTests() { expect(matchedRequests).to.have.length(5); expect(unmatchedRequests).to.have.length(0); - const [s, r1, u1, r2, u2] = matchedRequests; - expect(s.method).to.equal('GET'); - - expect(r1.method).to.equal('POST'); - expect(r1.headers).to.have.property('authorization', 'Token org:app:secret'); - expect(r1.body).to.equal('{"filepath":"~/one.js"}'); - - expect(u1.method).to.equal('PUT'); - expect(u1.body).to.equal('\'one js contents\';\n'); - - expect(r2.method).to.equal('POST'); - expect(r2.headers).to.have.property('authorization', 'Token org:app:secret'); - expect(r2.body).to.equal('{"filepath":"~/two.jsx"}'); - - expect(u2.method).to.equal('PUT'); - expect(u2.body).to.equal('\'two jsx contents\';\n'); + expect(matchedRequests[0].method).to.equal('GET'); + + expect(matchedRequests).to.deep.include.members([ + { + method: 'POST', + headers: { + authorization: 'Token org:app:secret', + }, + body: '{"filepath":"~/one.js"}', + }, + { + method: 'PUT', + body: '\'one js contents\';\n', + }, + { + method: 'POST', + headers: { + authorization: 'Token org:app:secret', + }, + body: '{"filepath":"~/two.jsx"}', + }, + { + method: 'PUT', + body: '\'two jsx contents\';\n', + }, + ]); })); it('should error if the server ping fails', mochaAsync(async () => { diff --git a/src/commands/upload.js b/src/commands/upload.js index 1f507d1..b2410d7 100644 --- a/src/commands/upload.js +++ b/src/commands/upload.js @@ -114,8 +114,9 @@ export const handler = async (args) => { console.info(`Found ${fileList.length} file${fileList.length === 1 ? '' : 's'} ...`); - for (const fileInfo of fileList) { - await uploadFile(fileInfo); + const CHUNK_SIZE = 10; + for (let i = 0; i < fileList.length; i += CHUNK_SIZE) { + await Promise.all(fileList.slice(i, i + CHUNK_SIZE).map(uploadFile)); } console.info('Success!');