From 7e11388099ce8a092894f0e256d8e40e803e9b88 Mon Sep 17 00:00:00 2001 From: Teeh Amaral Date: Wed, 30 Nov 2016 19:59:44 -0300 Subject: [PATCH 1/5] Adapter (in progress) --- lib/index.js | 113 ++++++++++++++++++++++++++++++++++++++---------- lib/transfer.js | 30 ++++++++++++- 2 files changed, 117 insertions(+), 26 deletions(-) diff --git a/lib/index.js b/lib/index.js index 0d1d5f1..4966c1f 100644 --- a/lib/index.js +++ b/lib/index.js @@ -26,7 +26,7 @@ function initialize(config) { Parse.serverURL = config.serverURL; return transfer.init(config); }).then(function() { - return getAllFileObjects(); + return getAllFileObjects(config); }).then(function(objects) { return transfer.run(objects); }).then(function() { @@ -38,11 +38,15 @@ function initialize(config) { }); } -function getAllFileObjects() { +function getAllFileObjects(config) { console.log("Fetching schema..."); return schemas.get().then(function(res){ console.log("Fetching all objects with files..."); - var schemasWithFiles = onlyFiles(res); + if (config.onlyFiles){ + var schemasWithFiles = onlyFiles(res); + } else { + var schemasWithFiles = filesAndFilesArray(res, config.extraFields); + } return Promise.all(schemasWithFiles.map(getObjectsWithFilesFromSchema)); }).then(function(results) { var files = results.reduce(function(c, r) { @@ -57,16 +61,47 @@ function getAllFileObjects() { function onlyFiles(schemas) { return schemas.map(function(schema) { - var fileFields = Object.keys(schema.fields).filter(function(key){ - var value = schema.fields[key]; - return value.type == "File"; - }); - if (fileFields.length > 0) { - return { - className: schema.className, - fields: fileFields - } - } + var fileFields = Object.keys(schema.fields).filter(function(key){ + var value = schema.fields[key]; + return value.type == "File"; + }); + if (fileFields.length > 0) { + return { + className: schema.className, + fields: fileFields + } + } + }).filter(function(s){ return s != undefined }) +} + +function filesAndFilesArray(schemas, extra) { + return schemas.map(function(schema) { + var fileFields = Object.keys(schema.fields).filter(function(key){ + var value = schema.fields[key]; + + var is_valid = false; + + if (extra[schema.className] != undefined){ + if (extra[schema.className].indexOf(key) != -1){ + is_valid = true; + } + } + + // console.log((value.type == "File") || (is_valid)); + + return (value.type == "File") || (is_valid); + // return value.type == "File"; + }); + if (fileFields.length > 0) { + // console.log({ + // className: schema.className, + // fields: fileFields + // }); + return { + className: schema.className, + fields: fileFields + } + } }).filter(function(s){ return s != undefined }) } @@ -101,19 +136,49 @@ function getObjectsWithFilesFromSchema(schema) { return getAllObjects(query).then(function(results) { return results.reduce(function(current, result){ - return current.concat( + // console.log(current); + // return current.concat( + var fileResults = []; schema.fields.map(function(field){ - var fName = result.get(field) ? result.get(field).name() : 'DELETE'; - var fUrl = result.get(field) ? result.get(field).url() : 'DELETE'; - return { - className: schema.className, - objectId: result.id, - fieldName: field, - fileName: fName, - url: fUrl + + if (Array.isArray(result.get(field))){ + for (var i = 0; i < result.get(field).length; i++) { + var fName = result.get(field)[i] ? result.get(field)[i].name() : 'DELETE'; + var fUrl = result.get(field)[i] ? result.get(field)[i].url() : 'DELETE'; + + fileResults.push({ + className: schema.className, + objectId: result.id, + fieldName: field, + fileName: fName, + url: fUrl + }) + } + } else { + var fName = result.get(field) ? result.get(field).name() : 'DELETE'; + var fUrl = result.get(field) ? result.get(field).url() : 'DELETE'; + fileResults.push({ + className: schema.className, + objectId: result.id, + fieldName: field, + fileName: fName, + url: fUrl + }) } - }) - ); + // var fName = result.get(field) ? result.get(field).name() : 'DELETE'; + // var fUrl = result.get(field) ? result.get(field).url() : 'DELETE'; + + // return { + // className: schema.className, + // objectId: result.id, + // fieldName: field, + // fileName: fName, + // url: fUrl + // } + + }); + return fileResults; + // ); }, []); }); } diff --git a/lib/transfer.js b/lib/transfer.js index e21cb9b..d7e4603 100644 --- a/lib/transfer.js +++ b/lib/transfer.js @@ -154,10 +154,36 @@ function _processFiles(files, handler) { function _changeDBFileField(file) { return new Promise(function(resolve, reject) { if (file.fileName == file.newFileName || !config.renameInDatabase) { - return resolve(); + // return resolve(); } + + if (config.filesAdapter._filesDir){ + var _path = config.filesAdapter._filesDir + "/" + file.newFileName; + } else if (config.filesAdapter._s3Client){ + var _path = "https://" + config.filesAdapter._bucket + "." + config.filesAdapter._s3Client.config.endpoint + "/" + config.filesAdapter._bucketPrefix + file.newFileName; + } + var update = {$set:{}}; - update.$set[file.fieldName] = file.newFileName; + + if (config.extraFields[file.className].indexOf(file.fieldName) != -1){ + db.collection(file.className).findOne({_id: file.objectId}).then(function(result){ + var images = result.images; + images.push({ + __type: 'File', + name: file.newFileName, + url: _path + }); + update.$set[file.fieldName] = images; + }); + } else { + update.$set[file.fieldName] = { + __type: 'File', + name: file.newFileName, + url: _path + }; + } + // https://ilhacloud-dev.s3.amazonaws.com/knowbook-dev/files/tfss-589a85c1-1fba-4307-b901-45c09b766a2d-file1446020888455.pdf + db.collection(file.className).update( { _id : file.objectId }, update, From c4b8c8119d02efce289fbcd96b2c20efbc7e884e Mon Sep 17 00:00:00 2001 From: Teeh Amaral Date: Thu, 1 Dec 2016 17:41:29 -0300 Subject: [PATCH 2/5] Add feature to migrate files array as well when renameFiles and renameInDatabase are true --- config.example.js | 11 ++++++++ lib/index.js | 68 +++++++++++++++++------------------------------ lib/transfer.js | 52 +++++++++++++++++++++++------------- 3 files changed, 68 insertions(+), 63 deletions(-) diff --git a/config.example.js b/config.example.js index aee49f7..cef4db9 100644 --- a/config.example.js +++ b/config.example.js @@ -8,9 +8,20 @@ module.exports = { mongoURL: "mongodb://:@mongourl.com:27017/database_name", serverURL: "https://api.customparseserver.com/parse", filesToTransfer: 'parseOnly', + renameFiles: false, renameInDatabase: false, transferTo: 'filesystem', + // If false, will migrate files array as well + // Note: The parameters renameFiles and renameInDatabase must be true + onlyFiles: false, + + // Extra collection and fields that use files array, use to migrate them as well + // Note: Use if onlyFiles equals false + extraFields: { + collectionName: ['fieldNameOne', 'fieldNameTwo'] + }, + // For filesystem configuration filesystemPath: './downloaded_files', diff --git a/lib/index.js b/lib/index.js index 4966c1f..dee0213 100644 --- a/lib/index.js +++ b/lib/index.js @@ -86,17 +86,9 @@ function filesAndFilesArray(schemas, extra) { is_valid = true; } } - - // console.log((value.type == "File") || (is_valid)); - return (value.type == "File") || (is_valid); - // return value.type == "File"; }); if (fileFields.length > 0) { - // console.log({ - // className: schema.className, - // fields: fileFields - // }); return { className: schema.className, fields: fileFields @@ -130,55 +122,43 @@ function getObjectsWithFilesFromSchema(schema) { query.limit(1000); var checks = schema.fields.map(function(field) { - return new Parse.Query(schema.className).exists(field); + return new Parse.Query(schema.className).exists(field); }); query._orQuery(checks); return getAllObjects(query).then(function(results) { return results.reduce(function(current, result){ - // console.log(current); - // return current.concat( - var fileResults = []; - schema.fields.map(function(field){ - - if (Array.isArray(result.get(field))){ - for (var i = 0; i < result.get(field).length; i++) { - var fName = result.get(field)[i] ? result.get(field)[i].name() : 'DELETE'; - var fUrl = result.get(field)[i] ? result.get(field)[i].url() : 'DELETE'; + var fileResults = []; + schema.fields.map(function(field){ + if (Array.isArray(result.get(field))){ + for (var i = 0; i < result.get(field).length; i++) { + var fName = result.get(field)[i] ? result.get(field)[i].name() : 'DELETE'; + var fUrl = result.get(field)[i] ? result.get(field)[i].url() : 'DELETE'; - fileResults.push({ - className: schema.className, - objectId: result.id, - fieldName: field, - fileName: fName, - url: fUrl - }) - } - } else { - var fName = result.get(field) ? result.get(field).name() : 'DELETE'; - var fUrl = result.get(field) ? result.get(field).url() : 'DELETE'; fileResults.push({ className: schema.className, objectId: result.id, fieldName: field, fileName: fName, - url: fUrl + url: fUrl, + i: i }) } - // var fName = result.get(field) ? result.get(field).name() : 'DELETE'; - // var fUrl = result.get(field) ? result.get(field).url() : 'DELETE'; - - // return { - // className: schema.className, - // objectId: result.id, - // fieldName: field, - // fileName: fName, - // url: fUrl - // } - - }); - return fileResults; - // ); + } else { + var fName = result.get(field) ? result.get(field).name() : 'DELETE'; + var fUrl = result.get(field) ? result.get(field).url() : 'DELETE'; + fileResults.push({ + className: schema.className, + objectId: result.id, + fieldName: field, + fileName: fName, + url: fUrl, + i: 0 + }) + } + + }); + return current.concat(fileResults); }, []); }); } diff --git a/lib/transfer.js b/lib/transfer.js index d7e4603..c963183 100644 --- a/lib/transfer.js +++ b/lib/transfer.js @@ -131,7 +131,7 @@ function _processFiles(files, handler) { var asyncLimit = config.asyncLimit || 5; return new Promise(function(resolve, reject) { async.eachOfLimit(files, asyncLimit, function(file, index, callback) { - process.stdout.write('Processing '+(index+1)+'/'+files.length+'\r'); + process.stdout.write('Processing '+(index+1)+'/'+files.length+'\r'); file.newFileName = _createNewFileName(file.fileName); if (_shouldTransferFile(file)) { _transferFile(file).then(callback, callback); @@ -154,12 +154,12 @@ function _processFiles(files, handler) { function _changeDBFileField(file) { return new Promise(function(resolve, reject) { if (file.fileName == file.newFileName || !config.renameInDatabase) { - // return resolve(); + return resolve(); } - if (config.filesAdapter._filesDir){ + if (config.transferTo === 'filesystem'){ var _path = config.filesAdapter._filesDir + "/" + file.newFileName; - } else if (config.filesAdapter._s3Client){ + } else if (config.transferTo === 's3'){ var _path = "https://" + config.filesAdapter._bucket + "." + config.filesAdapter._s3Client.config.endpoint + "/" + config.filesAdapter._bucketPrefix + file.newFileName; } @@ -167,33 +167,47 @@ function _changeDBFileField(file) { if (config.extraFields[file.className].indexOf(file.fieldName) != -1){ db.collection(file.className).findOne({_id: file.objectId}).then(function(result){ - var images = result.images; - images.push({ + + var images = result[file.fieldName]; + + images[file.i] = { __type: 'File', name: file.newFileName, url: _path - }); + }; update.$set[file.fieldName] = images; + + db.collection(file.className).update( + { _id : file.objectId }, + update, + function(error, result) { + if (error) { + return reject(error); + } + resolve(); + } + ); + }); + } else { update.$set[file.fieldName] = { __type: 'File', name: file.newFileName, url: _path }; - } - // https://ilhacloud-dev.s3.amazonaws.com/knowbook-dev/files/tfss-589a85c1-1fba-4307-b901-45c09b766a2d-file1446020888455.pdf - - db.collection(file.className).update( - { _id : file.objectId }, - update, - function(error, result ) { - if (error) { - return reject(error); + + db.collection(file.className).update( + { _id : file.objectId }, + update, + function(error, result ) { + if (error) { + return reject(error); + } + resolve(); } - resolve(); - } - ); + ); + } }); } From 6bca5015210e487fd3c327eaf831599eaedcf73b Mon Sep 17 00:00:00 2001 From: Teeh Amaral Date: Fri, 2 Dec 2016 11:57:28 -0300 Subject: [PATCH 3/5] Add feature to migrate files string as well when renameFiles and renameInDatabase are true --- lib/index.js | 49 +++++++++++++++++++++++++++++++++++-------------- lib/transfer.js | 40 +++++++++++++++++++++++++++------------- 2 files changed, 62 insertions(+), 27 deletions(-) diff --git a/lib/index.js b/lib/index.js index dee0213..e51145f 100644 --- a/lib/index.js +++ b/lib/index.js @@ -5,6 +5,8 @@ var schemas = require('./schemas'); var transfer = require('./transfer'); var questions = require('./questions.js'); +var stringFileRegex = new RegExp(/^(?=.*\bhttp\b)(?=.*\bparsetfss\b)(?=.*\btfss-\b).*$/); + module.exports = initialize; function initialize(config) { @@ -86,6 +88,7 @@ function filesAndFilesArray(schemas, extra) { is_valid = true; } } + return (value.type == "File") || (is_valid); }); if (fileFields.length > 0) { @@ -130,21 +133,40 @@ function getObjectsWithFilesFromSchema(schema) { return results.reduce(function(current, result){ var fileResults = []; schema.fields.map(function(field){ - if (Array.isArray(result.get(field))){ + if (Array.isArray(result.get(field))) { for (var i = 0; i < result.get(field).length; i++) { - var fName = result.get(field)[i] ? result.get(field)[i].name() : 'DELETE'; - var fUrl = result.get(field)[i] ? result.get(field)[i].url() : 'DELETE'; - - fileResults.push({ - className: schema.className, - objectId: result.id, - fieldName: field, - fileName: fName, - url: fUrl, - i: i - }) + if (result.get(field)[i].name !== undefined){ + var fName = result.get(field)[i] ? result.get(field)[i].name() : 'DELETE'; + var fUrl = result.get(field)[i] ? result.get(field)[i].url() : 'DELETE'; + + fileResults.push({ + className: schema.className, + objectId: result.id, + fieldName: field, + fileName: fName, + url: fUrl, + i: i + }); + } } - } else { + } else if ((typeof result.get(field) === 'string') && (stringFileRegex.test(result.get(field)))){ + var fUrl = result.get(field) ? result.get(field) : 'DELETE'; + if (fUrl != 'DELETE') { + var splitString = fUrl.split("/"); + var fName = splitString[splitString.length-1]; + } else { + var fName = 'DELETE'; + } + fileResults.push({ + className: schema.className, + objectId: result.id, + fieldName: field, + fileName: fName, + url: fUrl, + i: 0, + type: 'string' + }) + } else if (typeof result.get(field) != 'string') { var fName = result.get(field) ? result.get(field).name() : 'DELETE'; var fUrl = result.get(field) ? result.get(field).url() : 'DELETE'; fileResults.push({ @@ -156,7 +178,6 @@ function getObjectsWithFilesFromSchema(schema) { i: 0 }) } - }); return current.concat(fileResults); }, []); diff --git a/lib/transfer.js b/lib/transfer.js index c963183..cd3ebb5 100644 --- a/lib/transfer.js +++ b/lib/transfer.js @@ -166,17 +166,9 @@ function _changeDBFileField(file) { var update = {$set:{}}; if (config.extraFields[file.className].indexOf(file.fieldName) != -1){ - db.collection(file.className).findOne({_id: file.objectId}).then(function(result){ - var images = result[file.fieldName]; - - images[file.i] = { - __type: 'File', - name: file.newFileName, - url: _path - }; - update.$set[file.fieldName] = images; - + if (file.type === 'string') { + update.$set[file.fieldName] = _path; db.collection(file.className).update( { _id : file.objectId }, update, @@ -186,11 +178,33 @@ function _changeDBFileField(file) { } resolve(); } - ); - - }); + ) + } else { + db.collection(file.className).findOne({_id: file.objectId}).then(function(result){ + var images = result[file.fieldName]; + + images[file.i] = { + __type: 'File', + name: file.newFileName, + url: _path + }; + + update.$set[file.fieldName] = images; + db.collection(file.className).update( + { _id : file.objectId }, + update, + function(error, result) { + if (error) { + return reject(error); + } + resolve(); + } + ); + }); + } } else { + update.$set[file.fieldName] = { __type: 'File', name: file.newFileName, From e59341b011018109d37051f35cee0d3f0cd9b189 Mon Sep 17 00:00:00 2001 From: Teeh Amaral Date: Mon, 5 Dec 2016 18:38:43 -0300 Subject: [PATCH 4/5] Improvements --- lib/transfer.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/transfer.js b/lib/transfer.js index cd3ebb5..2860749 100644 --- a/lib/transfer.js +++ b/lib/transfer.js @@ -165,7 +165,7 @@ function _changeDBFileField(file) { var update = {$set:{}}; - if (config.extraFields[file.className].indexOf(file.fieldName) != -1){ + if ((config.extraFields[file.className] != undefined) && (config.extraFields[file.className].indexOf(file.fieldName) != -1)) { if (file.type === 'string') { update.$set[file.fieldName] = _path; From 14b325d3e7b8127d973f7cdc7b180456671a3e74 Mon Sep 17 00:00:00 2001 From: Teeh Amaral Date: Wed, 7 Dec 2016 18:33:33 -0300 Subject: [PATCH 5/5] Change to support only files or only string/array files --- lib/index.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/lib/index.js b/lib/index.js index e51145f..45b9929 100644 --- a/lib/index.js +++ b/lib/index.js @@ -47,7 +47,7 @@ function getAllFileObjects(config) { if (config.onlyFiles){ var schemasWithFiles = onlyFiles(res); } else { - var schemasWithFiles = filesAndFilesArray(res, config.extraFields); + var schemasWithFiles = stringAndFilesArray(res, config.extraFields); } return Promise.all(schemasWithFiles.map(getObjectsWithFilesFromSchema)); }).then(function(results) { @@ -76,7 +76,7 @@ function onlyFiles(schemas) { }).filter(function(s){ return s != undefined }) } -function filesAndFilesArray(schemas, extra) { +function stringAndFilesArray(schemas, extra) { return schemas.map(function(schema) { var fileFields = Object.keys(schema.fields).filter(function(key){ var value = schema.fields[key]; @@ -89,7 +89,7 @@ function filesAndFilesArray(schemas, extra) { } } - return (value.type == "File") || (is_valid); + return is_valid; }); if (fileFields.length > 0) { return {