Add nicer debug message. Remove experiment form duplicate script.

danmindru · Sep 15, 2019 · b22704d · b22704d
1 parent 4ff3c07
commit b22704d
Show file tree

Hide file tree

Showing 2 changed files with 42 additions and 33 deletions.
diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "cncnet-map-downloader.js",
-  "version": "2.0.0",
+  "version": "2.0.1",
   "description": "Downloads all maps from cncnet",
   "main": "index.js",
   "scripts": {

diff --git a/scripts/remove-duplicates.js b/scripts/remove-duplicates.js
@@ -11,14 +11,17 @@ const { debug } = require('./constants');
  *
  * @param { string } targetDir
  */
-const removeFile = (targetDir) => (filePath) => new Promise((resolve, reject) => fs.unlink(path.resolve(targetDir, filePath), (error) => {
-  if (error) {
-    console.error(`Failed to remove ${filePath}`, error);
-    reject(null);
-  }
+const removeFile = (targetDir) => (filePath) =>
+  new Promise((resolve, reject) =>
+    fs.unlink(path.resolve(targetDir, filePath), (error) => {
+      if (error) {
+        console.error(`Failed to remove ${filePath}`, error);
+        reject(null);
+      }
 
-  resolve(path.resolve(targetDir, filePath));
-}));
+      resolve(path.resolve(targetDir, filePath));
+    })
+  );
 
 /**
  * Get the size of a file.
@@ -27,14 +30,17 @@ const removeFile = (targetDir) => (filePath) => new Promise((resolve, reject) =>
  *
  * @return { number }
  */
-const getFileSize = (targetDir) => (filePath) => new Promise((resolve, reject) => fs.stat(path.resolve(targetDir, filePath), (error, stats) => {
-  if (error) {
-    console.error(`Failed to get file size for ${filePath}`, error);
-    reject(null);
-  }
+const getFileSize = (targetDir) => (filePath) =>
+  new Promise((resolve, reject) =>
+    fs.stat(path.resolve(targetDir, filePath), (error, stats) => {
+      if (error) {
+        console.error(`Failed to get file size for ${filePath}`, error);
+        reject(null);
+      }
 
-  resolve({ size: stats.size, filePath });
-}));
+      resolve({ size: stats.size, filePath });
+    })
+  );
 
 /**
  * Given a path, removes duplicate files by first checking size, then hash.
@@ -44,37 +50,42 @@ const getFileSize = (targetDir) => (filePath) => new Promise((resolve, reject) =
 const removeDuplicates = async (targetDir) => {
   const targetDirFilelist = fs.readdirSync(targetDir);
 
+  console.log(`\nComparing file sizes...`);
+
   // Get file sizes, filtering out failed fs.stats
   const fileSizes = await Promise.all(targetDirFilelist.map(getFileSize(targetDir)));
   const filesBySize = fileSizes
     .filter((size) => size)
-    .reduce((acc, { size, filePath }, ) => {
-      return { ...acc, [size]: acc[size] ? [...acc[size], filePath] : [filePath] }
+    .reduce((acc, { size, filePath }) => {
+      return { ...acc, [size]: acc[size] ? [...acc[size], filePath] : [filePath] };
     }, {});
 
+  console.log(`Checking for duplicates...`);
+
   // Filter by items with duplicates
-  const fileIndexesWithSameSize = Object.keys(filesBySize)
-    .filter((key) => {
-      const values = filesBySize[key];
-      return values.length > 1
-    });
+  const fileIndexesWithSameSize = Object.keys(filesBySize).filter((key) => {
+    const values = filesBySize[key];
+    return values.length > 1;
+  });
 
   // Filter by items with same checksum
-  const fileIndexesWithSameChecksum = fileIndexesWithSameSize
-    .filter(async (key) => {
-      const values = filesBySize[key];
+  const fileIndexesWithSameChecksum = fileIndexesWithSameSize.filter(async (key) => {
+    const values = filesBySize[key];
 
-      const hashArray = await Promise.all(values.map((filePath) => hasha.fromFile(path.resolve(targetDir, filePath), { algorithm: 'md5' })))
+    const hashArray = await Promise.all(
+      values.map((filePath) => hasha.fromFile(path.resolve(targetDir, filePath), { algorithm: 'md5' }))
+    );
 
-      return hashArray.every(v => v === hashArray[0])
-    });
+    return hashArray.every((v) => v === hashArray[0]);
+  });
 
+  console.log(`Removing duplicates...`);
   // Remove duplicates, taking the first item of the duplicate array
   const fileRemovalPromises = flatten(
     fileIndexesWithSameChecksum.map((key) => {
       const filesToRemove = filesBySize[key].slice(1);
 
-      if(!debug) {
+      if (debug) {
         console.debug(chalk.green(`\nFound map duplicates for file: ${filesBySize[key][0]}. Will keep only 1 map.`));
         console.debug(chalk.yellow(`Removing duplicates: ${filesToRemove.map((f) => `\n - ${chalk.bold(f)}`)}`));
       }
@@ -85,10 +96,8 @@ const removeDuplicates = async (targetDir) => {
 
   const removalResult = await Promise.all(fileRemovalPromises);
   return removalResult.filter((resolved) => resolved).length;
-}
+};
 
 module.exports = {
   removeDuplicates
-}
-
-removeDuplicates('./experiment');
+};