From cf5d1894ba980badc7890686184225bffc13c0ca Mon Sep 17 00:00:00 2001 From: VISHNUDAS-tunerlabse Date: Thu, 27 Apr 2023 22:29:24 +0530 Subject: [PATCH 1/3] projects delta data migration script added --- migrations/deltaDataMigrationForOCI/README.md | 5 + .../deltaDataMigrationScriptOfProjects.js | 136 ++++++++++++++++++ 2 files changed, 141 insertions(+) create mode 100644 migrations/deltaDataMigrationForOCI/README.md create mode 100644 migrations/deltaDataMigrationForOCI/deltaDataMigrationScriptOfProjects.js diff --git a/migrations/deltaDataMigrationForOCI/README.md b/migrations/deltaDataMigrationForOCI/README.md new file mode 100644 index 00000000..f6875302 --- /dev/null +++ b/migrations/deltaDataMigrationForOCI/README.md @@ -0,0 +1,5 @@ +## OCI Delta data migration of projects +Steps to run the migration files +- Navigate to migrations/deltaDataMigrationForOCI/ folder +- Run the script which will migrate projects delta data to OCI. + > node deltaDataMigrationScriptOfProjects.js \ No newline at end of file diff --git a/migrations/deltaDataMigrationForOCI/deltaDataMigrationScriptOfProjects.js b/migrations/deltaDataMigrationForOCI/deltaDataMigrationScriptOfProjects.js new file mode 100644 index 00000000..59d068a7 --- /dev/null +++ b/migrations/deltaDataMigrationForOCI/deltaDataMigrationScriptOfProjects.js @@ -0,0 +1,136 @@ +/** + * name : deltaDataMigrationScriptOfProjects.js + * author : Vishnu + * created-date : 27-04-2023 + * Description : Delta data migration script to OCI database + */ + +// Dependencies +const path = require("path"); +let rootPath = path.join(__dirname, '../../') +require('dotenv').config({ path: rootPath+'/.env' }) +let _ = require("lodash"); +var MongoClient = require('mongodb').MongoClient; +const kafkaProducersHelper = require(rootPath + "/generics/kafka/producers"); +var fs = require('fs'); + +// DB credentials +let mongoUrlOfOCIDb = process.env.MONGODB_URL; +let ociDbName = mongoUrlOfOCIDb.split("/").pop(); +let DbUrl = mongoUrlOfOCIDb.split(ociDbName)[0]; +let azureDbName = "sl-prod-old"; + +(async () => { + // Azure DB connection && OCI db connection + let connection = await MongoClient.connect(DbUrl, { useNewUrlParser: true }); + let db_Azure = connection.db(azureDbName); + let db_OCI = connection.db(ociDbName); + + try { + console.log("--------------------Execution Started------------------") + const fromDate = new Date('2023-04-18T00:00:00Z'); // Specify the date from which to fetch data + let addedProjects = []; + + let projectsDetailsFromAzureDb = await db_Azure.collection('projects').find({ + createdAt: {"$gte": fromDate}, + }).project({_id:1}).toArray(); + + // Get project Ids + projectIdsFromAzureDb = await getArrayOfMongoIds(projectsDetailsFromAzureDb); + + // Get data from OCI-mongo database + let projectsDetailsFromOCIDb = await db_OCI.collection('projects').find({ + }).project({_id:1}).toArray(); + + // Get project Ids + projectIdsFromOCIDb = await getArrayOfMongoIds(projectsDetailsFromOCIDb) + + let missingProjectIds = _.differenceWith(projectIdsFromAzureDb, projectIdsFromOCIDb,_.isEqual); + + if ( missingProjectIds.length > 0 ){ + await createDataInOCIdatabaseCollection(missingProjectIds,"projects"); + } + + /** + * getArrayOfMongoIds + * @param {Array} data + * @request - [{6025062519f84e54685df33f}] + * @returns {Array} - [6025062519f84e54685df33f] + */ + async function getArrayOfMongoIds( data ) { + // get mongoIds + let mongoIdsArray = []; + if ( data.length > 0 ) { + mongoIdsArray = data.map(function (obj) { + return obj._id; + }); + return mongoIdsArray; + } else { + return mongoIdsArray; + } + } + + /** + * createDataInOCIdatabaseCollection + * @param {Array} mongoIds - project _ids + * @param {String} collectionName + */ + async function createDataInOCIdatabaseCollection (mongoIds,collectionName) { + try{ + // Get entire project data from Db + let collectionDetailsFromAzureDb = await db_Azure.collection(collectionName).find({ + _id: {"$in": mongoIds}, + }).project().toArray(); + + let chunkOfDocument = _.chunk(collectionDetailsFromAzureDb, 10); + for ( let chunkOfDocumentIndex = 0; chunkOfDocumentIndex < chunkOfDocument.length; chunkOfDocumentIndex++ ) { + await insertIntoOCIDatabase(chunkOfDocument[chunkOfDocumentIndex],collectionName); + } + }catch(err){ + console.log("error occured") + } + } + + /** + * insertIntoOCIDatabase + * @param {Object} data - project data + * @param {String} collectionName + */ + async function insertIntoOCIDatabase (data,collectionName) { + try{ + let createDoc = await db_OCI.collection(collectionName).insertMany(data); + if(createDoc.ops && createDoc.ops.length > 0 ) { + for (let index=0; index < createDoc.ops.length; index++) { + addedProjects.push(createDoc.ops[index]._id) + await kafkaProducersHelper.pushProjectToKafka(createDoc.ops[index]); + } + } + }catch(err){ + console.log("error occured") + } + } + + // Write script exicution result into a file + if ( addedProjects.length > 0 ) { + //write updated project ids to file + fs.writeFile( + 'AddedProjectIds.json', + + JSON.stringify({projectIds:addedProjects}), + + function (err) { + if (err) { + console.error('Crap happens'); + } + } + ); + } + console.log("Added Project Count : ", addedProjects.length) + connection.close(); + console.log("--------------------Execution Finished------------------") + + } + catch (error) { + console.log(error) + } +})().catch(err => console.error(err)); \ No newline at end of file From d919b4bec59e04d8f0d6c874e797a24a2ed0f3ef Mon Sep 17 00:00:00 2001 From: VISHNUDAS-tunerlabse Date: Wed, 3 May 2023 09:19:32 +0530 Subject: [PATCH 2/3] change added to pass azure db name dynamically --- migrations/deltaDataMigrationForOCI/README.md | 5 +++-- .../deltaDataMigrationScriptOfProjects.js | 11 ++++++++--- 2 files changed, 11 insertions(+), 5 deletions(-) diff --git a/migrations/deltaDataMigrationForOCI/README.md b/migrations/deltaDataMigrationForOCI/README.md index f6875302..8749681e 100644 --- a/migrations/deltaDataMigrationForOCI/README.md +++ b/migrations/deltaDataMigrationForOCI/README.md @@ -1,5 +1,6 @@ ## OCI Delta data migration of projects Steps to run the migration files - Navigate to migrations/deltaDataMigrationForOCI/ folder -- Run the script which will migrate projects delta data to OCI. - > node deltaDataMigrationScriptOfProjects.js \ No newline at end of file +- Run the script which will migrate projects delta data to OCI. Please provide azure database name with the command +- if Azure database name is sl-prod-old then .. + > node deltaDataMigrationScriptOfProjects.js sl-prod-old \ No newline at end of file diff --git a/migrations/deltaDataMigrationForOCI/deltaDataMigrationScriptOfProjects.js b/migrations/deltaDataMigrationForOCI/deltaDataMigrationScriptOfProjects.js index 59d068a7..f0972afd 100644 --- a/migrations/deltaDataMigrationForOCI/deltaDataMigrationScriptOfProjects.js +++ b/migrations/deltaDataMigrationForOCI/deltaDataMigrationScriptOfProjects.js @@ -18,10 +18,15 @@ var fs = require('fs'); let mongoUrlOfOCIDb = process.env.MONGODB_URL; let ociDbName = mongoUrlOfOCIDb.split("/").pop(); let DbUrl = mongoUrlOfOCIDb.split(ociDbName)[0]; -let azureDbName = "sl-prod-old"; +let azureDbName = process.argv[2]; (async () => { - // Azure DB connection && OCI db connection + + if ( azureDbName === undefined ) { + console.log("Sorry! Please pass Azure Database name with node command.") + process.exit(); + } + let connection = await MongoClient.connect(DbUrl, { useNewUrlParser: true }); let db_Azure = connection.db(azureDbName); let db_OCI = connection.db(ociDbName); @@ -37,7 +42,7 @@ let azureDbName = "sl-prod-old"; // Get project Ids projectIdsFromAzureDb = await getArrayOfMongoIds(projectsDetailsFromAzureDb); - + // Get data from OCI-mongo database let projectsDetailsFromOCIDb = await db_OCI.collection('projects').find({ }).project({_id:1}).toArray(); From cbc705f4e42b837770724f43f15aeb1fcefbb6d7 Mon Sep 17 00:00:00 2001 From: VISHNUDAS-tunerlabse Date: Wed, 3 May 2023 09:22:17 +0530 Subject: [PATCH 3/3] comment added --- .../deltaDataMigrationScriptOfProjects.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/migrations/deltaDataMigrationForOCI/deltaDataMigrationScriptOfProjects.js b/migrations/deltaDataMigrationForOCI/deltaDataMigrationScriptOfProjects.js index f0972afd..278e5347 100644 --- a/migrations/deltaDataMigrationForOCI/deltaDataMigrationScriptOfProjects.js +++ b/migrations/deltaDataMigrationForOCI/deltaDataMigrationScriptOfProjects.js @@ -26,7 +26,7 @@ let azureDbName = process.argv[2]; console.log("Sorry! Please pass Azure Database name with node command.") process.exit(); } - + // Azure DB connection && OCI db connection let connection = await MongoClient.connect(DbUrl, { useNewUrlParser: true }); let db_Azure = connection.db(azureDbName); let db_OCI = connection.db(ociDbName);