Copy in chunks of 100 with multiple cp threads

J053Fabi0 · Sep 8, 2023 · b7c08ee · b7c08ee
1 parent 4b75ff6
commit b7c08ee
Show file tree

Hide file tree

Showing 2 changed files with 6 additions and 1 deletion.
diff --git a/deno.json b/deno.json
@@ -2,5 +2,8 @@
   "tasks": {
     "run": "deno run --allow-net --allow-write --allow-run --allow-env --allow-read index.ts",
     "check": "deno check mod.ts"
+  },
+  "imports": {
+    "lodash": "https://deno.land/x/[email protected]/mod.ts"
   }
 }
diff --git a/src/copyData.ts b/src/copyData.ts
@@ -1,3 +1,4 @@
+import { lodash as _ } from "lodash";
 import { cp } from "../utils/commands.ts";
 import getFiles from "../utils/getFiles.ts";
 import { join, MultiProgressBar } from "../deps.ts";
@@ -61,7 +62,8 @@ export default async function copyData(
       })();
 
     console.log("Copying the rest of the files with copy, including directories");
-    await cp([...otherFiles.map((f) => `${join(fromShardPath, f.name)}`), toShardPath]);
+    const filesWithFullPath = otherFiles.map((f) => join(fromShardPath, f.name));
+    await Promise.all(_.chunk(filesWithFullPath, 100).map((files) => cp([...files, toShardPath])));
 
     i = Infinity;
     if (logProgressBar) {