From e1265e474e2637ec16f5e4d2e5ccf9199a23e97c Mon Sep 17 00:00:00 2001 From: Aman Kumar Date: Thu, 7 Sep 2023 13:10:22 +0530 Subject: [PATCH 01/45] feat: taxonomies & terms support in export-to-csv --- .../src/commands/cm/export-to-csv.js | 190 ++++++++++++------ .../src/util/config.js | 5 +- .../src/util/index.js | 133 +++++++++++- .../src/util/interactive.js | 20 ++ 4 files changed, 278 insertions(+), 70 deletions(-) create mode 100644 packages/contentstack-export-to-csv/src/util/interactive.js diff --git a/packages/contentstack-export-to-csv/src/commands/cm/export-to-csv.js b/packages/contentstack-export-to-csv/src/commands/cm/export-to-csv.js index 8fa579906c..51d047b810 100644 --- a/packages/contentstack-export-to-csv/src/commands/cm/export-to-csv.js +++ b/packages/contentstack-export-to-csv/src/commands/cm/export-to-csv.js @@ -9,13 +9,14 @@ const { } = require('@contentstack/cli-utilities'); const util = require('../../util'); const config = require('../../util/config'); +const interactive = require('../../util/interactive'); class ExportToCsvCommand extends Command { static flags = { action: flags.string({ required: false, multiple: false, - options: ['entries', 'users'], + options: ['entries', 'users', 'taxonomies'], description: `Option to export data (entries, users)`, }), alias: flags.string({ @@ -59,6 +60,9 @@ class ExportToCsvCommand extends Command { multiple: false, required: false, }), + 'taxonomy-uid': flags.string({ + description: 'Provide taxonomy uid fow which terms need to be exported', + }), }; async run() { @@ -75,6 +79,7 @@ class ExportToCsvCommand extends Command { 'content-type': contentTypesFlag, alias: managementTokenAlias, branch: branchUid, + 'taxonomy-uid': taxonomyUID, }, } = await this.parse(ExportToCsvCommand); @@ -96,69 +101,17 @@ class ExportToCsvCommand extends Command { let stackAPIClient; let language; let contentTypes = []; - let stackBranches; - const listOfTokens = configHandler.get('tokens'); - if (managementTokenAlias && listOfTokens[managementTokenAlias]) { - managementAPIClient = await managementSDKClient({ - host: this.cmaHost, - management_token: listOfTokens[managementTokenAlias].token, - }); - stack = { - name: stackName || managementTokenAlias, - apiKey: listOfTokens[managementTokenAlias].apiKey, - token: listOfTokens[managementTokenAlias].token, - }; - } else if (managementTokenAlias) { - this.error('Provided management token alias not found in your config.!'); + if (managementTokenAlias) { + const { stackDetails, apiClient } = await this.getAliasDetails(managementTokenAlias); + managementAPIClient = apiClient; + stack = stackDetails; } else { - let organization; - - if (!isAuthenticated()) { - this.error(config.CLI_EXPORT_CSV_ENTRIES_ERROR, { - exit: 2, - suggestions: ['https://www.contentstack.com/docs/developers/cli/authentication/'], - }); - } - - if (org) { - organization = { uid: org }; - } else { - organization = await util.chooseOrganization(managementAPIClient); // prompt for organization - } - if (!stackAPIKey) { - stack = await util.chooseStack(managementAPIClient, organization.uid); // prompt for stack - } else { - stack = await util.chooseStack(managementAPIClient, organization.uid, stackAPIKey); - } + stack = await this.getStackDetails(managementAPIClient, stackAPIKey, org); } stackAPIClient = this.getStackClient(managementAPIClient, stack); - - if (branchUid) { - try { - const branchExists = await doesBranchExist(stackAPIClient, branchUid); - if (branchExists?.errorCode) { - throw new Error(branchExists.errorMessage); - } - stack.branch_uid = branchUid; - stackAPIClient = this.getStackClient(managementAPIClient, stack); - } catch (error) { - if (error.message || error.errorMessage) { - cliux.error(util.formatError(error)); - this.exit(); - } - } - } else { - stackBranches = await this.getStackBranches(stackAPIClient); - if (stackBranches === undefined) { - stackAPIClient = this.getStackClient(managementAPIClient, stack); - } else { - const { branch } = await util.chooseBranch(stackBranches); - stack.branch_uid = branch; - stackAPIClient = this.getStackClient(managementAPIClient, stack); - } - } + await this.checkAndUpdateBranchDetail(branchUid, stack, stackAPIClient, managementAPIClient); const contentTypeCount = await util.getContentTypeCount(stackAPIClient); @@ -258,6 +211,30 @@ class ExportToCsvCommand extends Command { } break; } + case config.exportTaxonomies: + case 'taxonomies': { + let stack; + let stackAPIClient; + let taxUID; + if (managementTokenAlias) { + const { stackDetails, apiClient } = await this.getAliasDetails(managementTokenAlias); + managementAPIClient = apiClient; + stack = stackDetails; + } else { + stack = await this.getStackDetails(managementAPIClient, stackAPIKey, org); + } + if (taxonomyUID) { + taxUID = taxonomyUID; + } else { + taxUID = await interactive.askTaxonomyUID(); + //validate taxUID + } + + stackAPIClient = this.getStackClient(managementAPIClient, stack); + await this.checkAndUpdateBranchDetail(branchUid, stack, stackAPIClient, managementAPIClient); + await this.createTaxonomyAndTermCsvFile(stackName, stack, taxUID); + break; + } } } catch (error) { if (error.message || error.errorMessage) { @@ -292,6 +269,101 @@ class ExportToCsvCommand extends Command { .then(({ items }) => (items !== undefined ? items : [])) .catch((_err) => {}); } + + async checkAndUpdateBranchDetail(branchUid, stack, stackAPIClient, managementAPIClient) { + if (branchUid) { + try { + const branchExists = await doesBranchExist(stackAPIClient, branchUid); + if (branchExists?.errorCode) { + throw new Error(branchExists.errorMessage); + } + stack.branch_uid = branchUid; + stackAPIClient = getStackClient(managementAPIClient, stack); + } catch (error) { + if (error.message || error.errorMessage) { + cliux.error(util.formatError(error)); + this.exit(); + } + } + } else { + const stackBranches = await this.getStackBranches(stackAPIClient); + if (stackBranches === undefined) { + stackAPIClient = this.getStackClient(managementAPIClient, stack); + } else { + const { branch } = await util.chooseBranch(stackBranches); + stack.branch_uid = branch; + stackAPIClient = this.getStackClient(managementAPIClient, stack); + } + } + } + + async getAliasDetails(managementTokenAlias) { + let apiClient, stackDetails; + const listOfTokens = configHandler.get('tokens'); + if (managementTokenAlias && listOfTokens[managementTokenAlias]) { + apiClient = await managementSDKClient({ + host: this.cmaHost, + management_token: listOfTokens[managementTokenAlias].token, + }); + stackDetails = { + name: stackName || managementTokenAlias, + apiKey: listOfTokens[managementTokenAlias].apiKey, + token: listOfTokens[managementTokenAlias].token, + }; + } else if (managementTokenAlias) { + this.error('Provided management token alias not found in your config.!'); + } + return { + apiClient, + stackDetails, + }; + } + + async getStackDetails(managementAPIClient, stackAPIKey, org) { + let organization, stackDetails; + + if (!isAuthenticated()) { + this.error(config.CLI_EXPORT_CSV_ENTRIES_ERROR, { + exit: 2, + suggestions: ['https://www.contentstack.com/docs/developers/cli/authentication/'], + }); + } + + if (org) { + organization = { uid: org }; + } else { + organization = await util.chooseOrganization(managementAPIClient); // prompt for organization + } + if (!stackAPIKey) { + stackDetails = await util.chooseStack(managementAPIClient, organization.uid); // prompt for stack + } else { + stackDetails = await util.chooseStack(managementAPIClient, organization.uid, stackAPIKey); + } + return stackDetails; + } + + async createTaxonomyAndTermCsvFile(stackName, stack, taxUID){ + const { cma, name } = configHandler.get('region') || {}; + const payload = { + baseUrl: `${cma}/api/v3/taxonomies`, + apiKey: stack.apiKey, + mgToken: stack?.token + }; + //check whether the taxonomy is valid or not + const taxonomy = await util.getTaxonomy(payload, taxUID); + + payload['url'] = payload.baseUrl; + const taxonomies = await util.getAllTaxonomies(payload); + const filteredTaxonomies = util.formatTaxonomiesResp(taxonomies); + let fileName = `${stackName ? stackName : stack.name}_taxonomies.csv`; + util.write(this, filteredTaxonomies, fileName, 'taxonomies'); // write to file + + payload['url'] = `${payload.baseUrl}/${taxUID}/terms`; + const terms = await util.getAllTermsOfTaxonomy(payload); + const filteredTerms = util.formatTermsOfTaxonomyResp(terms, taxUID); + fileName = `${stackName ? stackName : stack.name}_${taxonomy ? taxonomy.name : ''}_${taxUID}_terms.csv`; + util.write(this, filteredTerms, fileName, 'terms'); + } } ExportToCsvCommand.description = `Export entries or organization users to csv using this command`; diff --git a/packages/contentstack-export-to-csv/src/util/config.js b/packages/contentstack-export-to-csv/src/util/config.js index 6a424cc3a5..1ce7942da4 100644 --- a/packages/contentstack-export-to-csv/src/util/config.js +++ b/packages/contentstack-export-to-csv/src/util/config.js @@ -2,9 +2,10 @@ module.exports = { cancelString: 'Cancel and Exit', exportEntries: 'Export entries to a .CSV file', exportUsers: "Export organization users' data to a .CSV file", + exportTaxonomies: 'Export taxonomies to a .CSV file', adminError: "Unable to export data. Make sure you're an admin or owner of this organization", organizationNameRegex: /\'/, CLI_EXPORT_CSV_LOGIN_FAILED: "You need to login to execute this command. See: auth:login --help", - CLI_EXPORT_CSV_ENTRIES_ERROR: "You need to either login or provide a management token to execute this command" - + CLI_EXPORT_CSV_ENTRIES_ERROR: "You need to either login or provide a management token to execute this command", + CLI_EXPORT_CSV_API_FAILED: 'Something went wrong. Please try again' }; diff --git a/packages/contentstack-export-to-csv/src/util/index.js b/packages/contentstack-export-to-csv/src/util/index.js index 73a232f0dc..1d9054c95a 100644 --- a/packages/contentstack-export-to-csv/src/util/index.js +++ b/packages/contentstack-export-to-csv/src/util/index.js @@ -2,13 +2,14 @@ const os = require('os'); const fs = require('fs'); const mkdirp = require('mkdirp'); const find = require('lodash/find'); +const flat = require('lodash/flatten'); const fastcsv = require('fast-csv'); const inquirer = require('inquirer'); const debug = require('debug')('export-to-csv'); const checkboxPlus = require('inquirer-checkbox-plus-prompt'); const config = require('./config.js'); -const { cliux, configHandler } = require('@contentstack/cli-utilities'); +const { cliux, configHandler, HttpClient, messageHandler } = require('@contentstack/cli-utilities'); const directory = './data'; const delimeter = os.platform() === 'win32' ? '\\' : '/'; @@ -105,7 +106,7 @@ async function getOrganizationsWhereUserIsAdmin(managementAPIClient) { organizations.forEach((org) => { result[org.name] = org.uid; }); - } + } return result; } catch (error) { @@ -321,7 +322,13 @@ function getEntries(stackAPIClient, contentType, language, skip, limit) { stackAPIClient .contentType(contentType) .entry() - .query({ include_publish_details: true, locale: language, skip: skip * 100, limit: limit, include_workflow: true }) + .query({ + include_publish_details: true, + locale: language, + skip: skip * 100, + limit: limit, + include_workflow: true, + }) .find() .then((entries) => resolve(entries)) .catch((error) => reject(error)); @@ -378,7 +385,7 @@ function cleanEntries(entries, language, environments, contentTypeUid) { return filteredEntries.map((entry) => { let workflow = ''; const envArr = []; - if(entry.publish_details.length) { + if (entry.publish_details.length) { entry.publish_details.forEach((env) => { envArr.push(JSON.stringify([environments[env['environment']], env['locale'], env['time']])); }); @@ -387,10 +394,10 @@ function cleanEntries(entries, language, environments, contentTypeUid) { delete entry.publish_details; delete entry.setWorkflowStage; if ('_workflow' in entry) { - if(entry._workflow?.name) { - workflow = entry['_workflow']['name']; - delete entry['_workflow']; - } + if (entry._workflow?.name) { + workflow = entry['_workflow']['name']; + delete entry['_workflow']; + } } entry = flatten(entry); entry['publish_details'] = envArr; @@ -443,7 +450,7 @@ function startupQuestions() { type: 'list', name: 'action', message: 'Choose Action', - choices: [config.exportEntries, config.exportUsers, 'Exit'], + choices: [config.exportEntries, config.exportUsers, config.exportTaxonomies, 'Exit'], }, ]; inquirer @@ -662,6 +669,109 @@ function wait(time) { }); } +async function getAllTaxonomies(payload, skip = 0, limit = 100, taxonomies = []) { + const response = await apiRequestHandler(payload, skip, limit); + skip += limit || 100; + taxonomies = [...taxonomies, ...response.taxonomies]; + if (skip >= response?.count) { + return taxonomies; + } else { + return getAllTaxonomies(payload, skip, limit, taxonomies); + } +} + +async function getAllTermsOfTaxonomy(payload, skip = 0, limit = 100, terms = []) { + const response = await apiRequestHandler(payload, skip, limit); + skip += limit || 100; + terms = [...terms, ...response.terms]; + if (skip >= response?.count) { + return terms; + } else { + return getAllTermsOfTaxonomy(payload, skip, limit, terms); + } +} + +async function getTaxonomy(payload, taxonomyUID) { + payload['url'] = `${payload.baseUrl}/${taxonomyUID}`; + return await apiRequestHandler(payload); +} + +async function apiRequestHandler(payload, skip, limit) { + const headers = { + api_key: payload.apiKey, + 'Content-Type': 'application/json', + }; + + if (payload?.mgToken) headers['authorization'] = payload.mgToken; + else headers['authToken'] = configHandler.get('authToken'); + + const params = { + include_count: true, + skip: 0, + limit: 30, + }; + + if (skip >= 0) params['skip'] = skip; + if (limit >= 0) params['limit'] = limit; + + return await new HttpClient() + .headers(headers) + .queryParams(params) + .get(payload.url) + .then((res) => { + //NOTE - temporary code for handling api errors response + const { status, data } = res; + if ([200, 201, 202].includes(status)) return data; + else { + let errorMsg; + if (status === 500) errorMsg = data.message; + else errorMsg = data.error_message; + if(errorMsg === undefined){ + errorMsg = Object.values(data.errors) && flat(Object.values(data.errors)); + } + cliux.print(`Error: ${errorMsg}`, { color: 'red' }); + process.exit(1); + } + }) + .catch((err) => handleErrorMsg(err)); +} + +function formatTaxonomiesResp(taxonomies) { + const filteredTaxonomies = taxonomies.map((taxonomy) => { + return { + 'Taxonomy UID': taxonomy.uid, + Name: taxonomy.name, + Description: taxonomy.description, + }; + }); + return filteredTaxonomies; +} + +function formatTermsOfTaxonomyResp(terms, taxonomyUID) { + const filteredTerms = terms.map((term) => { + return { + 'Taxonomy UID': taxonomyUID, + UID: term.uid, + Name: term.name, + Description: term.description, + 'Parent UID': term.parent_uid, + }; + }); + return filteredTerms; +} + +function handleErrorMsg(err) { + if (err?.errorMessage) { + cliux.print(`Error: ${err.errorMessage}`, { color: 'red' }); + } else if (err?.message) { + cliux.print(`Error: ${err.message}`, { color: 'red' }); + } else { + console.log(err); + cliux.print(`Error: ${messageHandler.parse('CLI_EXPORT_CSV_API_FAILED')}`, { color: 'red' }); + } + process.exit(1); +} + module.exports = { chooseOrganization: chooseOrganization, chooseStack: chooseStack, @@ -688,4 +798,9 @@ module.exports = { chooseInMemContentTypes: chooseInMemContentTypes, getEntriesCount: getEntriesCount, formatError: formatError, + getAllTaxonomies, + getAllTermsOfTaxonomy, + formatTaxonomiesResp, + formatTermsOfTaxonomyResp, + getTaxonomy, }; diff --git a/packages/contentstack-export-to-csv/src/util/interactive.js b/packages/contentstack-export-to-csv/src/util/interactive.js new file mode 100644 index 0000000000..2ff817d235 --- /dev/null +++ b/packages/contentstack-export-to-csv/src/util/interactive.js @@ -0,0 +1,20 @@ +const isEmpty = require('lodash/isEmpty'); +const { cliux } = require('@contentstack/cli-utilities'); + +function inquireRequireFieldValidation(input) { + if (isEmpty(input)) { + return "This field can't be empty"; + } + return true; +} + +async function askTaxonomyUID() { + return await cliux.inquire({ + type: 'input', + message: 'Enter taxonomy UID', + name: 'taxonomy_uid', + validate: inquireRequireFieldValidation, + }); +} + +module.exports = { askTaxonomyUID }; From a15c2e990922f9822cd90a9d3577538f7239b0b6 Mon Sep 17 00:00:00 2001 From: Aman Kumar Date: Mon, 11 Sep 2023 19:49:18 +0530 Subject: [PATCH 02/45] fix bugs --- .../src/commands/cm/export-to-csv.js | 23 +++++---- .../src/util/index.js | 49 +++++++++++-------- .../src/util/interactive.js | 2 +- 3 files changed, 40 insertions(+), 34 deletions(-) diff --git a/packages/contentstack-export-to-csv/src/commands/cm/export-to-csv.js b/packages/contentstack-export-to-csv/src/commands/cm/export-to-csv.js index 51d047b810..8ec439deea 100644 --- a/packages/contentstack-export-to-csv/src/commands/cm/export-to-csv.js +++ b/packages/contentstack-export-to-csv/src/commands/cm/export-to-csv.js @@ -103,7 +103,7 @@ class ExportToCsvCommand extends Command { let contentTypes = []; if (managementTokenAlias) { - const { stackDetails, apiClient } = await this.getAliasDetails(managementTokenAlias); + const { stackDetails, apiClient } = await this.getAliasDetails(managementTokenAlias, stackName); managementAPIClient = apiClient; stack = stackDetails; } else { @@ -217,7 +217,7 @@ class ExportToCsvCommand extends Command { let stackAPIClient; let taxUID; if (managementTokenAlias) { - const { stackDetails, apiClient } = await this.getAliasDetails(managementTokenAlias); + const { stackDetails, apiClient } = await this.getAliasDetails(managementTokenAlias, stackName); managementAPIClient = apiClient; stack = stackDetails; } else { @@ -227,7 +227,6 @@ class ExportToCsvCommand extends Command { taxUID = taxonomyUID; } else { taxUID = await interactive.askTaxonomyUID(); - //validate taxUID } stackAPIClient = this.getStackClient(managementAPIClient, stack); @@ -280,7 +279,7 @@ class ExportToCsvCommand extends Command { stack.branch_uid = branchUid; stackAPIClient = getStackClient(managementAPIClient, stack); } catch (error) { - if (error.message || error.errorMessage) { + if (error?.message || error?.errorMessage) { cliux.error(util.formatError(error)); this.exit(); } @@ -297,7 +296,7 @@ class ExportToCsvCommand extends Command { } } - async getAliasDetails(managementTokenAlias) { + async getAliasDetails(managementTokenAlias, stackName) { let apiClient, stackDetails; const listOfTokens = configHandler.get('tokens'); if (managementTokenAlias && listOfTokens[managementTokenAlias]) { @@ -343,9 +342,9 @@ class ExportToCsvCommand extends Command { } async createTaxonomyAndTermCsvFile(stackName, stack, taxUID){ - const { cma, name } = configHandler.get('region') || {}; + const { cma } = configHandler.get('region') || {}; const payload = { - baseUrl: `${cma}/api/v3/taxonomies`, + baseUrl: `${cma}/v3/taxonomies`, apiKey: stack.apiKey, mgToken: stack?.token }; @@ -354,15 +353,15 @@ class ExportToCsvCommand extends Command { payload['url'] = payload.baseUrl; const taxonomies = await util.getAllTaxonomies(payload); - const filteredTaxonomies = util.formatTaxonomiesResp(taxonomies); + const formattedTaxonomiesData = util.formatTaxonomiesData(taxonomies); let fileName = `${stackName ? stackName : stack.name}_taxonomies.csv`; - util.write(this, filteredTaxonomies, fileName, 'taxonomies'); // write to file + util.write(this, formattedTaxonomiesData, fileName, 'taxonomies'); payload['url'] = `${payload.baseUrl}/${taxUID}/terms`; const terms = await util.getAllTermsOfTaxonomy(payload); - const filteredTerms = util.formatTermsOfTaxonomyResp(terms, taxUID); - fileName = `${stackName ? stackName : stack.name}_${taxonomy ? taxonomy.name : ''}_${taxUID}_terms.csv`; - util.write(this, filteredTerms, fileName, 'terms'); + const formattedTermsData = util.formatTermsOfTaxonomyData(terms, taxUID); + fileName = `${stackName ? stackName : stack.name}_${taxonomy?.name ? taxonomy.name : ''}_${taxUID}_terms.csv`; + util.write(this, formattedTermsData, fileName, 'terms'); } } diff --git a/packages/contentstack-export-to-csv/src/util/index.js b/packages/contentstack-export-to-csv/src/util/index.js index 1d9054c95a..5d7f9e7296 100644 --- a/packages/contentstack-export-to-csv/src/util/index.js +++ b/packages/contentstack-export-to-csv/src/util/index.js @@ -671,29 +671,36 @@ function wait(time) { async function getAllTaxonomies(payload, skip = 0, limit = 100, taxonomies = []) { const response = await apiRequestHandler(payload, skip, limit); - skip += limit || 100; - taxonomies = [...taxonomies, ...response.taxonomies]; - if (skip >= response?.count) { - return taxonomies; - } else { - return getAllTaxonomies(payload, skip, limit, taxonomies); + if(response){ + skip += config.limit || 100; + taxonomies = [...taxonomies, ...response.taxonomies]; + if (skip >= response?.count) { + return taxonomies; + } else { + return getAllTaxonomies(payload, skip, limit, taxonomies); + } } + return taxonomies; } async function getAllTermsOfTaxonomy(payload, skip = 0, limit = 100, terms = []) { const response = await apiRequestHandler(payload, skip, limit); - skip += limit || 100; - terms = [...terms, ...response.terms]; - if (skip >= response?.count) { - return terms; - } else { - return getAllTermsOfTaxonomy(payload, skip, limit, terms); + if(response){ + skip += config.limit || 100; + terms = [...terms, ...response.terms]; + if (skip >= response?.count) { + return terms; + } else { + return getAllTermsOfTaxonomy(payload, skip, limit, terms); + } } + return terms; } async function getTaxonomy(payload, taxonomyUID) { payload['url'] = `${payload.baseUrl}/${taxonomyUID}`; - return await apiRequestHandler(payload); + const resp = await apiRequestHandler(payload); + return resp?.taxonomy || ''; } async function apiRequestHandler(payload, skip, limit) { @@ -703,7 +710,7 @@ async function apiRequestHandler(payload, skip, limit) { }; if (payload?.mgToken) headers['authorization'] = payload.mgToken; - else headers['authToken'] = configHandler.get('authToken'); + else headers['authToken'] = configHandler.get('authtoken'); const params = { include_count: true, @@ -724,10 +731,10 @@ async function apiRequestHandler(payload, skip, limit) { if ([200, 201, 202].includes(status)) return data; else { let errorMsg; - if (status === 500) errorMsg = data.message; - else errorMsg = data.error_message; + if ([500, 503, 502].includes(status)) errorMsg = data?.message || data; + else errorMsg = data?.error_message; if(errorMsg === undefined){ - errorMsg = Object.values(data.errors) && flat(Object.values(data.errors)); + errorMsg = Object.values(data?.errors) && flat(Object.values(data.errors)); } cliux.print(`Error: ${errorMsg}`, { color: 'red' }); process.exit(1); @@ -736,7 +743,7 @@ async function apiRequestHandler(payload, skip, limit) { .catch((err) => handleErrorMsg(err)); } -function formatTaxonomiesResp(taxonomies) { +function formatTaxonomiesData(taxonomies) { const filteredTaxonomies = taxonomies.map((taxonomy) => { return { 'Taxonomy UID': taxonomy.uid, @@ -747,7 +754,7 @@ function formatTaxonomiesResp(taxonomies) { return filteredTaxonomies; } -function formatTermsOfTaxonomyResp(terms, taxonomyUID) { +function formatTermsOfTaxonomyData(terms, taxonomyUID) { const filteredTerms = terms.map((term) => { return { 'Taxonomy UID': taxonomyUID, @@ -800,7 +807,7 @@ module.exports = { formatError: formatError, getAllTaxonomies, getAllTermsOfTaxonomy, - formatTaxonomiesResp, - formatTermsOfTaxonomyResp, + formatTaxonomiesData, + formatTermsOfTaxonomyData, getTaxonomy, }; diff --git a/packages/contentstack-export-to-csv/src/util/interactive.js b/packages/contentstack-export-to-csv/src/util/interactive.js index 2ff817d235..7d05733351 100644 --- a/packages/contentstack-export-to-csv/src/util/interactive.js +++ b/packages/contentstack-export-to-csv/src/util/interactive.js @@ -12,7 +12,7 @@ async function askTaxonomyUID() { return await cliux.inquire({ type: 'input', message: 'Enter taxonomy UID', - name: 'taxonomy_uid', + name: 'taxonomyUid', validate: inquireRequireFieldValidation, }); } From 2dabdbbfc905039b179494180bcbce42a81987a6 Mon Sep 17 00:00:00 2001 From: Aman Kumar Date: Mon, 11 Sep 2023 23:56:33 +0530 Subject: [PATCH 03/45] refactor: taxonomies --- .../src/commands/cm/export-to-csv.js | 27 ++++++- .../src/util/config.js | 1 + .../src/util/index.js | 73 ++++++++++++++----- 3 files changed, 82 insertions(+), 19 deletions(-) diff --git a/packages/contentstack-export-to-csv/src/commands/cm/export-to-csv.js b/packages/contentstack-export-to-csv/src/commands/cm/export-to-csv.js index 8ec439deea..bedf08fe14 100644 --- a/packages/contentstack-export-to-csv/src/commands/cm/export-to-csv.js +++ b/packages/contentstack-export-to-csv/src/commands/cm/export-to-csv.js @@ -230,7 +230,6 @@ class ExportToCsvCommand extends Command { } stackAPIClient = this.getStackClient(managementAPIClient, stack); - await this.checkAndUpdateBranchDetail(branchUid, stack, stackAPIClient, managementAPIClient); await this.createTaxonomyAndTermCsvFile(stackName, stack, taxUID); break; } @@ -269,6 +268,13 @@ class ExportToCsvCommand extends Command { .catch((_err) => {}); } + /** + * check whether branch enabled org or not and update branch details + * @param {string} branchUid + * @param {object} stack + * @param {*} stackAPIClient + * @param {*} managementAPIClient + */ async checkAndUpdateBranchDetail(branchUid, stack, stackAPIClient, managementAPIClient) { if (branchUid) { try { @@ -296,6 +302,12 @@ class ExportToCsvCommand extends Command { } } + /** + * fetch stack details from alias token + * @param {string} managementTokenAlias + * @param {string} stackName + * @returns + */ async getAliasDetails(managementTokenAlias, stackName) { let apiClient, stackDetails; const listOfTokens = configHandler.get('tokens'); @@ -318,6 +330,13 @@ class ExportToCsvCommand extends Command { }; } + /** + * fetch stack details on basis of the selected org and stack + * @param {*} managementAPIClient + * @param {string} stackAPIKey + * @param {string} org + * @returns + */ async getStackDetails(managementAPIClient, stackAPIKey, org) { let organization, stackDetails; @@ -341,6 +360,12 @@ class ExportToCsvCommand extends Command { return stackDetails; } + /** + * Create a taxonomies csv file for stack and a terms csv file for associated taxonomies + * @param {string} stackName + * @param {object} stack + * @param {string} taxUID + */ async createTaxonomyAndTermCsvFile(stackName, stack, taxUID){ const { cma } = configHandler.get('region') || {}; const payload = { diff --git a/packages/contentstack-export-to-csv/src/util/config.js b/packages/contentstack-export-to-csv/src/util/config.js index 1ce7942da4..64f986352c 100644 --- a/packages/contentstack-export-to-csv/src/util/config.js +++ b/packages/contentstack-export-to-csv/src/util/config.js @@ -1,4 +1,5 @@ module.exports = { + limit:100, cancelString: 'Cancel and Exit', exportEntries: 'Export entries to a .CSV file', exportUsers: "Export organization users' data to a .CSV file", diff --git a/packages/contentstack-export-to-csv/src/util/index.js b/packages/contentstack-export-to-csv/src/util/index.js index 5d7f9e7296..1d1e31c1ff 100644 --- a/packages/contentstack-export-to-csv/src/util/index.js +++ b/packages/contentstack-export-to-csv/src/util/index.js @@ -669,6 +669,14 @@ function wait(time) { }); } +/** + * fetch all taxonomies in the provided stack + * @param {object} payload + * @param {number} skip + * @param {number} limit + * @param {array} taxonomies + * @returns + */ async function getAllTaxonomies(payload, skip = 0, limit = 100, taxonomies = []) { const response = await apiRequestHandler(payload, skip, limit); if(response){ @@ -683,6 +691,14 @@ async function getAllTaxonomies(payload, skip = 0, limit = 100, taxonomies = []) return taxonomies; } +/** + * fetch terms of related taxonomy + * @param {object} payload + * @param {number} skip + * @param {number} limit + * @param {array} terms + * @returns + */ async function getAllTermsOfTaxonomy(payload, skip = 0, limit = 100, terms = []) { const response = await apiRequestHandler(payload, skip, limit); if(response){ @@ -697,6 +713,12 @@ async function getAllTermsOfTaxonomy(payload, skip = 0, limit = 100, terms = []) return terms; } +/** + * Verify the existence of a taxonomy. Obtain its details if it exists and return + * @param {object} payload + * @param {string} taxonomyUID + * @returns + */ async function getTaxonomy(payload, taxonomyUID) { payload['url'] = `${payload.baseUrl}/${taxonomyUID}`; const resp = await apiRequestHandler(payload); @@ -743,28 +765,43 @@ async function apiRequestHandler(payload, skip, limit) { .catch((err) => handleErrorMsg(err)); } +/** + * Change taxonomies data in required CSV headers format + * @param {array} taxonomies + * @returns + */ function formatTaxonomiesData(taxonomies) { - const filteredTaxonomies = taxonomies.map((taxonomy) => { - return { - 'Taxonomy UID': taxonomy.uid, - Name: taxonomy.name, - Description: taxonomy.description, - }; - }); - return filteredTaxonomies; + if(taxonomies?.length){ + const formattedTaxonomies = taxonomies.map((taxonomy) => { + return { + 'Taxonomy UID': taxonomy.uid, + Name: taxonomy.name, + Description: taxonomy.description, + }; + }); + return formattedTaxonomies; + } } +/** + * Modify the linked taxonomy data's terms in required CSV headers format + * @param {array} terms + * @param {string} taxonomyUID + * @returns + */ function formatTermsOfTaxonomyData(terms, taxonomyUID) { - const filteredTerms = terms.map((term) => { - return { - 'Taxonomy UID': taxonomyUID, - UID: term.uid, - Name: term.name, - Description: term.description, - 'Parent UID': term.parent_uid, - }; - }); - return filteredTerms; + if(terms?.length){ + const formattedTerms = terms.map((term) => { + return { + 'Taxonomy UID': taxonomyUID, + UID: term.uid, + Name: term.name, + Description: term.description, + 'Parent UID': term.parent_uid, + }; + }); + return formattedTerms; + } } function handleErrorMsg(err) { From f190ca416aedae0c67156bcf0ed28e89029978cf Mon Sep 17 00:00:00 2001 From: Aman Kumar Date: Tue, 12 Sep 2023 10:30:36 +0530 Subject: [PATCH 04/45] fix: correct prompt msg --- .../contentstack-export-to-csv/src/commands/cm/export-to-csv.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/contentstack-export-to-csv/src/commands/cm/export-to-csv.js b/packages/contentstack-export-to-csv/src/commands/cm/export-to-csv.js index bedf08fe14..c67223af41 100644 --- a/packages/contentstack-export-to-csv/src/commands/cm/export-to-csv.js +++ b/packages/contentstack-export-to-csv/src/commands/cm/export-to-csv.js @@ -61,7 +61,7 @@ class ExportToCsvCommand extends Command { required: false, }), 'taxonomy-uid': flags.string({ - description: 'Provide taxonomy uid fow which terms need to be exported', + description: 'Provide taxonomy UID for which terms need to be exported', }), }; From c3b84fe1c4beb9ff4e06cbf92d9f8e9b03d79bcb Mon Sep 17 00:00:00 2001 From: Aman Kumar Date: Tue, 12 Sep 2023 16:55:46 +0530 Subject: [PATCH 05/45] fix: handle empty file in case of taxonomy and terms --- .../src/commands/cm/export-to-csv.js | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/packages/contentstack-export-to-csv/src/commands/cm/export-to-csv.js b/packages/contentstack-export-to-csv/src/commands/cm/export-to-csv.js index c67223af41..ace45b38d7 100644 --- a/packages/contentstack-export-to-csv/src/commands/cm/export-to-csv.js +++ b/packages/contentstack-export-to-csv/src/commands/cm/export-to-csv.js @@ -380,13 +380,21 @@ class ExportToCsvCommand extends Command { const taxonomies = await util.getAllTaxonomies(payload); const formattedTaxonomiesData = util.formatTaxonomiesData(taxonomies); let fileName = `${stackName ? stackName : stack.name}_taxonomies.csv`; - util.write(this, formattedTaxonomiesData, fileName, 'taxonomies'); + if(formattedTaxonomiesData?.length){ + util.write(this, formattedTaxonomiesData, fileName, 'taxonomies'); + }else{ + cliux.print('No taxonomies are found in the provided stack. Please provide a valid stack!\n', {color: 'yellow'}); + } payload['url'] = `${payload.baseUrl}/${taxUID}/terms`; const terms = await util.getAllTermsOfTaxonomy(payload); const formattedTermsData = util.formatTermsOfTaxonomyData(terms, taxUID); fileName = `${stackName ? stackName : stack.name}_${taxonomy?.name ? taxonomy.name : ''}_${taxUID}_terms.csv`; - util.write(this, formattedTermsData, fileName, 'terms'); + if(formattedTermsData?.length){ + util.write(this, formattedTermsData, fileName, 'terms'); + }else{ + cliux.print(`No terms are found for the provided taxonomy UID. Please provide a valid taxonomy UID!`, {color: 'yellow'}); + } } } From 94dba59d46d9422d50d03a0281217b1e13b1a4aa Mon Sep 17 00:00:00 2001 From: Aman Kumar Date: Tue, 12 Sep 2023 20:00:08 +0530 Subject: [PATCH 06/45] refactor: taxonomy uid optional flag and related changes --- .../src/commands/cm/export-to-csv.js | 90 ++++++++++--------- .../src/util/interactive.js | 20 ----- 2 files changed, 50 insertions(+), 60 deletions(-) delete mode 100644 packages/contentstack-export-to-csv/src/util/interactive.js diff --git a/packages/contentstack-export-to-csv/src/commands/cm/export-to-csv.js b/packages/contentstack-export-to-csv/src/commands/cm/export-to-csv.js index ace45b38d7..26cbe7fe2e 100644 --- a/packages/contentstack-export-to-csv/src/commands/cm/export-to-csv.js +++ b/packages/contentstack-export-to-csv/src/commands/cm/export-to-csv.js @@ -9,7 +9,6 @@ const { } = require('@contentstack/cli-utilities'); const util = require('../../util'); const config = require('../../util/config'); -const interactive = require('../../util/interactive'); class ExportToCsvCommand extends Command { static flags = { @@ -223,14 +222,9 @@ class ExportToCsvCommand extends Command { } else { stack = await this.getStackDetails(managementAPIClient, stackAPIKey, org); } - if (taxonomyUID) { - taxUID = taxonomyUID; - } else { - taxUID = await interactive.askTaxonomyUID(); - } stackAPIClient = this.getStackClient(managementAPIClient, stack); - await this.createTaxonomyAndTermCsvFile(stackName, stack, taxUID); + await this.createTaxonomyAndTermCsvFile(stackName, stack, taxonomyUID); break; } } @@ -270,10 +264,10 @@ class ExportToCsvCommand extends Command { /** * check whether branch enabled org or not and update branch details - * @param {string} branchUid - * @param {object} stack - * @param {*} stackAPIClient - * @param {*} managementAPIClient + * @param {string} branchUid + * @param {object} stack + * @param {*} stackAPIClient + * @param {*} managementAPIClient */ async checkAndUpdateBranchDetail(branchUid, stack, stackAPIClient, managementAPIClient) { if (branchUid) { @@ -304,9 +298,9 @@ class ExportToCsvCommand extends Command { /** * fetch stack details from alias token - * @param {string} managementTokenAlias - * @param {string} stackName - * @returns + * @param {string} managementTokenAlias + * @param {string} stackName + * @returns */ async getAliasDetails(managementTokenAlias, stackName) { let apiClient, stackDetails; @@ -332,10 +326,10 @@ class ExportToCsvCommand extends Command { /** * fetch stack details on basis of the selected org and stack - * @param {*} managementAPIClient - * @param {string} stackAPIKey - * @param {string} org - * @returns + * @param {*} managementAPIClient + * @param {string} stackAPIKey + * @param {string} org + * @returns */ async getStackDetails(managementAPIClient, stackAPIKey, org) { let organization, stackDetails; @@ -362,43 +356,55 @@ class ExportToCsvCommand extends Command { /** * Create a taxonomies csv file for stack and a terms csv file for associated taxonomies - * @param {string} stackName - * @param {object} stack - * @param {string} taxUID + * @param {string} stackName + * @param {object} stack + * @param {string} taxUID */ - async createTaxonomyAndTermCsvFile(stackName, stack, taxUID){ + async createTaxonomyAndTermCsvFile(stackName, stack, taxUID) { const { cma } = configHandler.get('region') || {}; const payload = { baseUrl: `${cma}/v3/taxonomies`, apiKey: stack.apiKey, - mgToken: stack?.token + mgToken: stack?.token, }; //check whether the taxonomy is valid or not - const taxonomy = await util.getTaxonomy(payload, taxUID); + let taxonomies = []; + if (taxUID) { + const taxonomy = await util.getTaxonomy(payload, taxUID); + taxonomies.push(taxonomy); + } else { + payload['url'] = payload.baseUrl; + taxonomies = await util.getAllTaxonomies(payload); + } - payload['url'] = payload.baseUrl; - const taxonomies = await util.getAllTaxonomies(payload); const formattedTaxonomiesData = util.formatTaxonomiesData(taxonomies); - let fileName = `${stackName ? stackName : stack.name}_taxonomies.csv`; - if(formattedTaxonomiesData?.length){ - util.write(this, formattedTaxonomiesData, fileName, 'taxonomies'); - }else{ - cliux.print('No taxonomies are found in the provided stack. Please provide a valid stack!\n', {color: 'yellow'}); + if (formattedTaxonomiesData?.length) { + const fileName = `${stackName ? stackName : stack.name}_taxonomies.csv`; + util.write(this, formattedTaxonomiesData, fileName, 'taxonomies'); + } else { + cliux.print('info: No taxonomies found. Please provide a valid stack!', { color: 'blue' }); } - payload['url'] = `${payload.baseUrl}/${taxUID}/terms`; - const terms = await util.getAllTermsOfTaxonomy(payload); - const formattedTermsData = util.formatTermsOfTaxonomyData(terms, taxUID); - fileName = `${stackName ? stackName : stack.name}_${taxonomy?.name ? taxonomy.name : ''}_${taxUID}_terms.csv`; - if(formattedTermsData?.length){ - util.write(this, formattedTermsData, fileName, 'terms'); - }else{ - cliux.print(`No terms are found for the provided taxonomy UID. Please provide a valid taxonomy UID!`, {color: 'yellow'}); + for (let index = 0; index < taxonomies?.length; index++) { + const taxonomy = taxonomies[index]; + const taxonomyUID = taxonomy?.uid; + if (taxonomyUID) { + payload['url'] = `${payload.baseUrl}/${taxonomyUID}/terms`; + const terms = await util.getAllTermsOfTaxonomy(payload); + const formattedTermsData = util.formatTermsOfTaxonomyData(terms, taxonomyUID); + const taxonomyName = taxonomy?.name ? taxonomy.name : ''; + const termFileName = `${stackName ? stackName : stack.name}_${taxonomyName}_${taxonomyUID}_terms.csv`; + if (formattedTermsData?.length) { + util.write(this, formattedTermsData, termFileName, 'terms'); + } else { + cliux.print(`info: No terms found for the taxonomy UID - '${taxonomyUID}'`, { color: 'blue' }); + } + } } } } -ExportToCsvCommand.description = `Export entries or organization users to csv using this command`; +ExportToCsvCommand.description = `Export entries, taxonomies, terms or organization users to csv using this command`; ExportToCsvCommand.examples = [ 'csdx cm:export-to-csv', @@ -414,6 +420,10 @@ ExportToCsvCommand.examples = [ '', 'Exporting organization users to csv with organization name provided', 'csdx cm:export-to-csv --action --org --org-name ', + 'Exporting taxonomies and related terms to csv with taxonomy uid provided', + 'csdx cm:export-to-csv --action --alias --taxonomy-uid ', + 'Exporting taxonomies and respective terms to csv', + 'csdx cm:export-to-csv --action --alias ', ]; module.exports = ExportToCsvCommand; diff --git a/packages/contentstack-export-to-csv/src/util/interactive.js b/packages/contentstack-export-to-csv/src/util/interactive.js deleted file mode 100644 index 7d05733351..0000000000 --- a/packages/contentstack-export-to-csv/src/util/interactive.js +++ /dev/null @@ -1,20 +0,0 @@ -const isEmpty = require('lodash/isEmpty'); -const { cliux } = require('@contentstack/cli-utilities'); - -function inquireRequireFieldValidation(input) { - if (isEmpty(input)) { - return "This field can't be empty"; - } - return true; -} - -async function askTaxonomyUID() { - return await cliux.inquire({ - type: 'input', - message: 'Enter taxonomy UID', - name: 'taxonomyUid', - validate: inquireRequireFieldValidation, - }); -} - -module.exports = { askTaxonomyUID }; From 54ba7364f856bd4284a49fc7c4f66ce031dbaa94 Mon Sep 17 00:00:00 2001 From: Aman Kumar Date: Wed, 13 Sep 2023 14:35:52 +0530 Subject: [PATCH 07/45] fix: removed description from terms csv file --- packages/contentstack-export-to-csv/src/util/index.js | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/contentstack-export-to-csv/src/util/index.js b/packages/contentstack-export-to-csv/src/util/index.js index 1d1e31c1ff..a760ce6722 100644 --- a/packages/contentstack-export-to-csv/src/util/index.js +++ b/packages/contentstack-export-to-csv/src/util/index.js @@ -796,7 +796,6 @@ function formatTermsOfTaxonomyData(terms, taxonomyUID) { 'Taxonomy UID': taxonomyUID, UID: term.uid, Name: term.name, - Description: term.description, 'Parent UID': term.parent_uid, }; }); From 120dcc6f252bc6d44811707da1d830972e3e1927 Mon Sep 17 00:00:00 2001 From: Aman Kumar Date: Mon, 18 Sep 2023 09:22:25 +0530 Subject: [PATCH 08/45] feat: export-to-csv test cases using @oclif/test --- package-lock.json | 320 ++++++++- .../contentstack-export-to-csv/package.json | 12 +- .../test/mock-data/common.mock.json | 239 +++++++ .../test/unit/commands/export-to-csv.test.js | 672 +++++++++++++----- 4 files changed, 1041 insertions(+), 202 deletions(-) create mode 100644 packages/contentstack-export-to-csv/test/mock-data/common.mock.json diff --git a/package-lock.json b/package-lock.json index 2d8f67f74c..30eca23b7d 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2721,9 +2721,9 @@ } }, "node_modules/@types/chai": { - "version": "4.3.5", - "resolved": "https://registry.npmjs.org/@types/chai/-/chai-4.3.5.tgz", - "integrity": "sha512-mEo1sAde+UCE6b2hxn332f1g1E8WfYRu6p5SvTKr2ZKC1f7gFJXk4h5PyGP9Dt6gCaG8y8XhwnXWC6Iy2cmBng==" + "version": "4.3.6", + "resolved": "https://registry.npmjs.org/@types/chai/-/chai-4.3.6.tgz", + "integrity": "sha512-VOVRLM1mBxIRxydiViqPcKn6MIxZytrbMpd6RJLIWKxUNr3zux8no0Oc7kJx0WAPIitgZ0gkrDS+btlqQpubpw==" }, "node_modules/@types/cli-progress": { "version": "3.11.0", @@ -4647,9 +4647,9 @@ } }, "node_modules/chai": { - "version": "4.3.7", - "resolved": "https://registry.npmjs.org/chai/-/chai-4.3.7.tgz", - "integrity": "sha512-HLnAzZ2iupm25PlN0xFreAlBA5zaBSv3og0DdeGA4Ar6h6rJ3A0rolRUKJhSF2V10GZKDgWF/VmAEsNWjCRB+A==", + "version": "4.3.8", + "resolved": "https://registry.npmjs.org/chai/-/chai-4.3.8.tgz", + "integrity": "sha512-vX4YvVVtxlfSZ2VecZgFUTU5qPCYsobVI2O9FmwEXBhDigYGQA6jRXCycIs1yJnnWbZ6/+a2zNIF5DfVCcJBFQ==", "dev": true, "dependencies": { "assertion-error": "^1.1.0", @@ -21085,12 +21085,15 @@ }, "devDependencies": { "@oclif/test": "^2.2.10", - "chai": "^4.2.0", + "@types/chai": "^4.3.6", + "@types/mocha": "^10.0.1", + "chai": "^4.3.8", "debug": "^4.3.1", + "dotenv": "^16.3.1", "eslint": "^7.32.0", "eslint-config-oclif": "^4.0.0", "globby": "^10.0.2", - "mocha": "^10.0.0", + "mocha": "^10.2.0", "nyc": "^15.1.0", "oclif": "^3.8.1" }, @@ -21141,6 +21144,12 @@ "node": ">=10.10.0" } }, + "packages/contentstack-export-to-csv/node_modules/@types/mocha": { + "version": "10.0.1", + "resolved": "https://registry.npmjs.org/@types/mocha/-/mocha-10.0.1.tgz", + "integrity": "sha512-/fvYntiO1GeICvqbQ3doGDIP97vWmvFt83GKguJ6prmQM2iXZfFcq6YE8KteFyRtX2/h5Hf91BYvPodJKFYv5Q==", + "dev": true + }, "packages/contentstack-export-to-csv/node_modules/acorn": { "version": "7.4.1", "resolved": "https://registry.npmjs.org/acorn/-/acorn-7.4.1.tgz", @@ -21169,6 +21178,32 @@ "url": "https://github.com/sponsors/epoberezkin" } }, + "packages/contentstack-export-to-csv/node_modules/argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", + "dev": true + }, + "packages/contentstack-export-to-csv/node_modules/brace-expansion": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", + "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "dev": true, + "dependencies": { + "balanced-match": "^1.0.0" + } + }, + "packages/contentstack-export-to-csv/node_modules/cliui": { + "version": "7.0.4", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-7.0.4.tgz", + "integrity": "sha512-OcRE68cOsVMXp1Yvonl/fzkQOyjLSu/8bhPDfQt0e0/Eb283TKP20Fs2MqoPsr9SwA595rRCA+QMzYc9nBP+JQ==", + "dev": true, + "dependencies": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.0", + "wrap-ansi": "^7.0.0" + } + }, "packages/contentstack-export-to-csv/node_modules/eslint": { "version": "7.32.0", "resolved": "https://registry.npmjs.org/eslint/-/eslint-7.32.0.tgz", @@ -21282,6 +21317,26 @@ "node": ">=4" } }, + "packages/contentstack-export-to-csv/node_modules/glob": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.0.tgz", + "integrity": "sha512-lmLf6gtyrPq8tTjSmrO94wBeQbFR3HbLHbuyD69wuyQkImp2hWqMGB47OX65FBkPffO641IP9jWa1z4ivqG26Q==", + "dev": true, + "dependencies": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.0.4", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + }, + "engines": { + "node": "*" + }, + "funding": { + "url": "https://github.com/sponsors/isaacs" + } + }, "packages/contentstack-export-to-csv/node_modules/glob-parent": { "version": "5.1.2", "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", @@ -21309,6 +21364,22 @@ "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", "dev": true }, + "packages/contentstack-export-to-csv/node_modules/log-symbols": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-4.1.0.tgz", + "integrity": "sha512-8XPvpAA8uyhfteu8pIvQxpJZ7SYYdpUivZpGy6sFsBuKRY/7rQGavedeB8aK+Zkyq6upMFVL/9AW6vOYzfRyLg==", + "dev": true, + "dependencies": { + "chalk": "^4.1.0", + "is-unicode-supported": "^0.1.0" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "packages/contentstack-export-to-csv/node_modules/mkdirp": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-3.0.1.tgz", @@ -21323,6 +21394,94 @@ "url": "https://github.com/sponsors/isaacs" } }, + "packages/contentstack-export-to-csv/node_modules/mocha": { + "version": "10.2.0", + "resolved": "https://registry.npmjs.org/mocha/-/mocha-10.2.0.tgz", + "integrity": "sha512-IDY7fl/BecMwFHzoqF2sg/SHHANeBoMMXFlS9r0OXKDssYE1M5O43wUY/9BVPeIvfH2zmEbBfseqN9gBQZzXkg==", + "dev": true, + "dependencies": { + "ansi-colors": "4.1.1", + "browser-stdout": "1.3.1", + "chokidar": "3.5.3", + "debug": "4.3.4", + "diff": "5.0.0", + "escape-string-regexp": "4.0.0", + "find-up": "5.0.0", + "glob": "7.2.0", + "he": "1.2.0", + "js-yaml": "4.1.0", + "log-symbols": "4.1.0", + "minimatch": "5.0.1", + "ms": "2.1.3", + "nanoid": "3.3.3", + "serialize-javascript": "6.0.0", + "strip-json-comments": "3.1.1", + "supports-color": "8.1.1", + "workerpool": "6.2.1", + "yargs": "16.2.0", + "yargs-parser": "20.2.4", + "yargs-unparser": "2.0.0" + }, + "bin": { + "_mocha": "bin/_mocha", + "mocha": "bin/mocha.js" + }, + "engines": { + "node": ">= 14.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/mochajs" + } + }, + "packages/contentstack-export-to-csv/node_modules/mocha/node_modules/js-yaml": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", + "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", + "dev": true, + "dependencies": { + "argparse": "^2.0.1" + }, + "bin": { + "js-yaml": "bin/js-yaml.js" + } + }, + "packages/contentstack-export-to-csv/node_modules/mocha/node_modules/minimatch": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.0.1.tgz", + "integrity": "sha512-nLDxIFRyhDblz3qMuq+SoRZED4+miJ/G+tdDrjkkkRnjAsBexeGpgjLEQ0blJy7rHhR2b93rhQY4SvyWu9v03g==", + "dev": true, + "dependencies": { + "brace-expansion": "^2.0.1" + }, + "engines": { + "node": ">=10" + } + }, + "packages/contentstack-export-to-csv/node_modules/ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true + }, + "packages/contentstack-export-to-csv/node_modules/yargs": { + "version": "16.2.0", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-16.2.0.tgz", + "integrity": "sha512-D1mvvtDG0L5ft/jGWkLpG1+m0eQxOfaBvTNELraWj22wSVUMWxZUvYgJYcKh6jGGIkJFhH4IZPQhR4TKpc8mBw==", + "dev": true, + "dependencies": { + "cliui": "^7.0.2", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.0", + "y18n": "^5.0.5", + "yargs-parser": "^20.2.2" + }, + "engines": { + "node": ">=10" + } + }, "packages/contentstack-export/node_modules/@oclif/test": { "version": "1.2.9", "resolved": "https://registry.npmjs.org/@oclif/test/-/test-1.2.9.tgz", @@ -24067,9 +24226,12 @@ "@contentstack/cli-command": "~1.2.11", "@contentstack/cli-utilities": "~1.5.1", "@oclif/test": "^2.2.10", - "chai": "^4.2.0", + "@types/chai": "^4.3.6", + "@types/mocha": "^10.0.1", + "chai": "^4.3.8", "chalk": "^4.1.0", "debug": "^4.3.1", + "dotenv": "^16.3.1", "eslint": "^7.32.0", "eslint-config-oclif": "^4.0.0", "fast-csv": "^4.3.6", @@ -24077,7 +24239,7 @@ "inquirer": "8.2.4", "inquirer-checkbox-plus-prompt": "1.0.1", "mkdirp": "^3.0.1", - "mocha": "^10.0.0", + "mocha": "^10.2.0", "nyc": "^15.1.0", "oclif": "^3.8.1" }, @@ -24119,6 +24281,12 @@ "minimatch": "^3.0.4" } }, + "@types/mocha": { + "version": "10.0.1", + "resolved": "https://registry.npmjs.org/@types/mocha/-/mocha-10.0.1.tgz", + "integrity": "sha512-/fvYntiO1GeICvqbQ3doGDIP97vWmvFt83GKguJ6prmQM2iXZfFcq6YE8KteFyRtX2/h5Hf91BYvPodJKFYv5Q==", + "dev": true + }, "acorn": { "version": "7.4.1", "resolved": "https://registry.npmjs.org/acorn/-/acorn-7.4.1.tgz", @@ -24137,6 +24305,32 @@ "uri-js": "^4.2.2" } }, + "argparse": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/argparse/-/argparse-2.0.1.tgz", + "integrity": "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q==", + "dev": true + }, + "brace-expansion": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz", + "integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==", + "dev": true, + "requires": { + "balanced-match": "^1.0.0" + } + }, + "cliui": { + "version": "7.0.4", + "resolved": "https://registry.npmjs.org/cliui/-/cliui-7.0.4.tgz", + "integrity": "sha512-OcRE68cOsVMXp1Yvonl/fzkQOyjLSu/8bhPDfQt0e0/Eb283TKP20Fs2MqoPsr9SwA595rRCA+QMzYc9nBP+JQ==", + "dev": true, + "requires": { + "string-width": "^4.2.0", + "strip-ansi": "^6.0.0", + "wrap-ansi": "^7.0.0" + } + }, "eslint": { "version": "7.32.0", "resolved": "https://registry.npmjs.org/eslint/-/eslint-7.32.0.tgz", @@ -24227,6 +24421,20 @@ } } }, + "glob": { + "version": "7.2.0", + "resolved": "https://registry.npmjs.org/glob/-/glob-7.2.0.tgz", + "integrity": "sha512-lmLf6gtyrPq8tTjSmrO94wBeQbFR3HbLHbuyD69wuyQkImp2hWqMGB47OX65FBkPffO641IP9jWa1z4ivqG26Q==", + "dev": true, + "requires": { + "fs.realpath": "^1.0.0", + "inflight": "^1.0.4", + "inherits": "2", + "minimatch": "^3.0.4", + "once": "^1.3.0", + "path-is-absolute": "^1.0.0" + } + }, "glob-parent": { "version": "5.1.2", "resolved": "https://registry.npmjs.org/glob-parent/-/glob-parent-5.1.2.tgz", @@ -24248,10 +24456,90 @@ "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", "dev": true }, + "log-symbols": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-4.1.0.tgz", + "integrity": "sha512-8XPvpAA8uyhfteu8pIvQxpJZ7SYYdpUivZpGy6sFsBuKRY/7rQGavedeB8aK+Zkyq6upMFVL/9AW6vOYzfRyLg==", + "dev": true, + "requires": { + "chalk": "^4.1.0", + "is-unicode-supported": "^0.1.0" + } + }, "mkdirp": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/mkdirp/-/mkdirp-3.0.1.tgz", "integrity": "sha512-+NsyUUAZDmo6YVHzL/stxSu3t9YS1iljliy3BSDrXJ/dkn1KYdmtZODGGjLcc9XLgVVpH4KshHB8XmZgMhaBXg==" + }, + "mocha": { + "version": "10.2.0", + "resolved": "https://registry.npmjs.org/mocha/-/mocha-10.2.0.tgz", + "integrity": "sha512-IDY7fl/BecMwFHzoqF2sg/SHHANeBoMMXFlS9r0OXKDssYE1M5O43wUY/9BVPeIvfH2zmEbBfseqN9gBQZzXkg==", + "dev": true, + "requires": { + "ansi-colors": "4.1.1", + "browser-stdout": "1.3.1", + "chokidar": "3.5.3", + "debug": "4.3.4", + "diff": "5.0.0", + "escape-string-regexp": "4.0.0", + "find-up": "5.0.0", + "glob": "7.2.0", + "he": "1.2.0", + "js-yaml": "4.1.0", + "log-symbols": "4.1.0", + "minimatch": "5.0.1", + "ms": "2.1.3", + "nanoid": "3.3.3", + "serialize-javascript": "6.0.0", + "strip-json-comments": "3.1.1", + "supports-color": "8.1.1", + "workerpool": "6.2.1", + "yargs": "16.2.0", + "yargs-parser": "20.2.4", + "yargs-unparser": "2.0.0" + }, + "dependencies": { + "js-yaml": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/js-yaml/-/js-yaml-4.1.0.tgz", + "integrity": "sha512-wpxZs9NoxZaJESJGIZTyDEaYpl0FKSA+FB9aJiyemKhMwkxQg63h4T1KJgUGHpTqPDNRcmmYLugrRjJlBtWvRA==", + "dev": true, + "requires": { + "argparse": "^2.0.1" + } + }, + "minimatch": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.0.1.tgz", + "integrity": "sha512-nLDxIFRyhDblz3qMuq+SoRZED4+miJ/G+tdDrjkkkRnjAsBexeGpgjLEQ0blJy7rHhR2b93rhQY4SvyWu9v03g==", + "dev": true, + "requires": { + "brace-expansion": "^2.0.1" + } + } + } + }, + "ms": { + "version": "2.1.3", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", + "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==", + "dev": true + }, + "yargs": { + "version": "16.2.0", + "resolved": "https://registry.npmjs.org/yargs/-/yargs-16.2.0.tgz", + "integrity": "sha512-D1mvvtDG0L5ft/jGWkLpG1+m0eQxOfaBvTNELraWj22wSVUMWxZUvYgJYcKh6jGGIkJFhH4IZPQhR4TKpc8mBw==", + "dev": true, + "requires": { + "cliui": "^7.0.2", + "escalade": "^3.1.1", + "get-caller-file": "^2.0.5", + "require-directory": "^2.1.1", + "string-width": "^4.2.0", + "y18n": "^5.0.5", + "yargs-parser": "^20.2.2" + } } } }, @@ -27560,9 +27848,9 @@ } }, "@types/chai": { - "version": "4.3.5", - "resolved": "https://registry.npmjs.org/@types/chai/-/chai-4.3.5.tgz", - "integrity": "sha512-mEo1sAde+UCE6b2hxn332f1g1E8WfYRu6p5SvTKr2ZKC1f7gFJXk4h5PyGP9Dt6gCaG8y8XhwnXWC6Iy2cmBng==" + "version": "4.3.6", + "resolved": "https://registry.npmjs.org/@types/chai/-/chai-4.3.6.tgz", + "integrity": "sha512-VOVRLM1mBxIRxydiViqPcKn6MIxZytrbMpd6RJLIWKxUNr3zux8no0Oc7kJx0WAPIitgZ0gkrDS+btlqQpubpw==" }, "@types/cli-progress": { "version": "3.11.0", @@ -29043,9 +29331,9 @@ } }, "chai": { - "version": "4.3.7", - "resolved": "https://registry.npmjs.org/chai/-/chai-4.3.7.tgz", - "integrity": "sha512-HLnAzZ2iupm25PlN0xFreAlBA5zaBSv3og0DdeGA4Ar6h6rJ3A0rolRUKJhSF2V10GZKDgWF/VmAEsNWjCRB+A==", + "version": "4.3.8", + "resolved": "https://registry.npmjs.org/chai/-/chai-4.3.8.tgz", + "integrity": "sha512-vX4YvVVtxlfSZ2VecZgFUTU5qPCYsobVI2O9FmwEXBhDigYGQA6jRXCycIs1yJnnWbZ6/+a2zNIF5DfVCcJBFQ==", "dev": true, "requires": { "assertion-error": "^1.1.0", diff --git a/packages/contentstack-export-to-csv/package.json b/packages/contentstack-export-to-csv/package.json index 3ed9faa616..67a2a33636 100644 --- a/packages/contentstack-export-to-csv/package.json +++ b/packages/contentstack-export-to-csv/package.json @@ -15,12 +15,15 @@ }, "devDependencies": { "@oclif/test": "^2.2.10", - "chai": "^4.2.0", + "@types/chai": "^4.3.6", + "@types/mocha": "^10.0.1", + "chai": "^4.3.8", "debug": "^4.3.1", + "dotenv": "^16.3.1", "eslint": "^7.32.0", "eslint-config-oclif": "^4.0.0", "globby": "^10.0.2", - "mocha": "^10.0.0", + "mocha": "^10.2.0", "nyc": "^15.1.0", "oclif": "^3.8.1" }, @@ -44,7 +47,8 @@ "postpack": "rm -f oclif.manifest.json", "prepack": "oclif manifest && oclif readme", "test": "nyc mocha --forbid-only \"test/**/*.test.js\"", - "test:unit": "nyc mocha --timeout 10000 --forbid-only \"test/unit/**/*.test.js\"", + "test:unit": "mocha --timeout 10000 --forbid-only \"test/unit/**/*.test.js\"", + "test:unit:report": "nyc --extension .js mocha --forbid-only \"test/unit/**/*.test.js\"", "version": "oclif readme && git add README.md", "clean": "rm -rf ./node_modules tsconfig.build.tsbuildinfo" }, @@ -61,4 +65,4 @@ } }, "repository": "https://github.com/contentstack/cli" -} \ No newline at end of file +} diff --git a/packages/contentstack-export-to-csv/test/mock-data/common.mock.json b/packages/contentstack-export-to-csv/test/mock-data/common.mock.json new file mode 100644 index 0000000000..f16230eb1f --- /dev/null +++ b/packages/contentstack-export-to-csv/test/mock-data/common.mock.json @@ -0,0 +1,239 @@ +{ + "taxonomiesResp": { + "taxonomies": [ + { + "uid": "taxonomy_uid_1", + "name": "taxonomy uid 1", + "description": "", + "created_at": "2023-09-01T06:09:44.934Z", + "created_by": "user1", + "updated_at": "2023-09-01T06:44:16.604Z", + "updated_by": "user1" + }, + { + "uid": "taxonomy_uid_2", + "name": "taxonomy uid 2", + "description": "", + "created_at": "2023-09-01T06:09:44.934Z", + "created_by": "user1", + "updated_at": "2023-09-01T06:44:16.604Z", + "updated_by": "user1" + } + ], + "count": 2 + }, + "termsResp": { + "terms": [ + { + "uid": "wsq", + "name": "wsq", + "created_at": "2023-08-30T09:51:12.043Z", + "created_by": "user1", + "updated_at": "2023-08-30T09:51:12.043Z", + "updated_by": "user1", + "parent_uid": null, + "depth": 1 + }, + { + "uid": "term2", + "name": "term2", + "created_at": "2023-08-30T09:45:11.963Z", + "created_by": "user2", + "updated_at": "2023-08-30T09:45:11.963Z", + "updated_by": "user2", + "parent_uid": null, + "depth": 1 + } + ], + "count": 2 + }, + "organizations": [ + { + "uid": "test-uid-1", + "name": "test org 1" + }, + { + "uid": "test-uid-2", + "name": "test org 2" + } + ], + "stacks": [ + { + "name": "Stack 1", + "uid": "stack-uid-1", + "api_key": "stack_api_key_1" + }, + { + "name": "Stack 2", + "uid": "stack-uid-2", + "api_key": "stack_api_key_2" + } + ], + "users": { + "items": [ + { + "uid": "uid1", + "email": "test@gmail.abc", + "user_uid": "user1", + "org_uid": "test-uid-1", + "invited_by": "user2", + "invited_at": "2023-08-21T11:08:41.038Z", + "status": "accepted", + "acceptance_token": "dfghdfgd", + "created_at": "2023-08-21T11:08:41.036Z", + "updated_at": "2023-08-21T11:09:11.342Z", + "urlPath": "/user", + "organizations": [ + { + "uid": "test-uid-1", + "name": "test org 1", + "org_roles": [ + { + "uid": "role1", + "name": "Admin", + "description": "Admin Role", + "org_uid": "test-uid-1", + "admin": true, + "default": true + } + ] + } + ] + }, + { + "uid": "test-uid-2", + "name": "test org 2" + } + ] + }, + "roles": [ + { + "urlPath": "/roles/role1", + "uid": "role1", + "name": "admin", + "description": "The Admin role has rights to add/remove users from an organization, and has access to stacks created by self or shared by others.", + "org_uid": "test-uid-1", + "owner_uid": "user1", + "admin": true, + "default": true, + "users": ["user2", "user3"], + "created_at": "2023-08-08T10:09:43.445Z", + "updated_at": "2023-08-21T11:08:41.042Z" + } + ], + "contentTypes": { + "items": [ + { + "stackHeaders": { + "api_key": "stack_api_key_1" + }, + "urlPath": "/content_types/ct1", + "created_at": "2023-08-08T13:52:31.980Z", + "updated_at": "2023-08-08T13:52:34.265Z", + "title": "CT 1", + "uid": "ct_1", + "_version": 2, + "inbuilt_class": false, + "schema": [ + { + "data_type": "text", + "display_name": "Title", + "field_metadata": { + "_default": true, + "version": 3 + }, + "mandatory": true, + "uid": "title", + "unique": true, + "multiple": false, + "non_localizable": false + } + ] + }, + { + "stackHeaders": { + "api_key": "stack_api_key_1" + }, + "urlPath": "/content_types/ct2", + "created_at": "2023-08-08T13:52:31.980Z", + "updated_at": "2023-08-08T13:52:34.265Z", + "title": "CT 2", + "uid": "ct_2", + "_version": 2, + "inbuilt_class": false, + "schema": [ + { + "data_type": "text", + "display_name": "Title", + "field_metadata": { + "_default": true, + "version": 3 + }, + "mandatory": true, + "uid": "title", + "unique": true, + "multiple": false, + "non_localizable": false + } + ] + } + ] + }, + "branch": { + "stackHeaders": { + "api_key": "stack_api_key_1" + }, + "urlPath": "/stacks/branches/test_branch1", + "uid": "test_branch1", + "source": "", + "created_by": "user1", + "updated_by": "user1", + "created_at": "2023-08-08T13:51:43.217Z", + "updated_at": "2023-08-08T13:51:43.217Z", + "deleted_at": false + }, + "entry": { + "items": [ + { + "stackHeaders": { + "api_key": "stack_api_key_1" + }, + "content_type_uid": "home", + "urlPath": "/content_types/ct1/entries/test_entry1", + "title": "Test Entry1", + "url": "/", + "tags": [], + "locale": "en1", + "uid": "test_entry1", + "created_by": "user1", + "updated_by": "user1", + "created_at": "2023-08-08T13:52:46.592Z", + "updated_at": "2023-08-08T13:52:46.592Z", + "_version": 1 + } + ] + }, + "environments": { + "items": [ + { + "stackHeaders": { + "api_key": "stack_api_key_1" + }, + "urlPath": "/environments/development", + "urls": [ + { + "url": "http://localhost:3000/", + "locale": "en1" + } + ], + "name": "development", + "_version": 3, + "uid": "env1", + "created_by": "user1", + "updated_by": "user1", + "created_at": "2023-06-12T18:59:56.853Z", + "updated_at": "2023-06-12T18:59:56.853Z" + } + ] + } +} diff --git a/packages/contentstack-export-to-csv/test/unit/commands/export-to-csv.test.js b/packages/contentstack-export-to-csv/test/unit/commands/export-to-csv.test.js index d2415f7179..965a593cbb 100644 --- a/packages/contentstack-export-to-csv/test/unit/commands/export-to-csv.test.js +++ b/packages/contentstack-export-to-csv/test/unit/commands/export-to-csv.test.js @@ -1,3 +1,20 @@ +const fs = require('fs'); +const mkdirp = require('mkdirp'); +const { test, expect } = require('@oclif/test'); +const { join } = require('path'); +const { PassThrough } = require('stream'); +const inquirer = require('inquirer'); +const checkboxPlus = require('inquirer-checkbox-plus-prompt'); +const { cliux, configHandler } = require('@contentstack/cli-utilities'); + +const mockData = require('../../mock-data/common.mock.json'); + +const { cma } = configHandler.get('region'); +const directory = './data'; +const taxonomyFileName = join(process.cwd(), 'data', mockData.stacks[0].name); +const userFileName = join(process.cwd(), 'data', mockData.organizations[0].name); + + /* eslint-disable no-undef */ const { describe, it, beforeEach, afterEach } = require('mocha'); const ExportToCsvCommand = require('../../../src/commands/cm/export-to-csv'); @@ -6,214 +23,505 @@ const { config } = require('dotenv'); const inquirer = require('inquirer'); const { cliux } = require('@contentstack/cli-utilities'); -config(); +//!SECTION ------old test cases ------------- -describe('Export to csv command with action = entries', () => { - let inquireStub; - let errorStub; - let consoleLogStub; +// config(); - beforeEach(() => { - inquireStub = stub(inquirer, 'prompt'); - errorStub = stub(cliux, 'error'); - consoleLogStub = stub(cliux, 'print'); - }); +// describe('Export to csv command with action = entries', () => { +// let inquireStub; +// let errorStub; +// let consoleLogStub; - afterEach(() => { - inquireStub.restore(); - errorStub.restore(); - consoleLogStub.restore(); - }); +// beforeEach(() => { +// inquireStub = stub(inquirer, 'prompt'); +// errorStub = stub(cliux, 'error'); +// consoleLogStub = stub(cliux, 'print'); +// }); - it('Should ask for action when action is not passed (entries or users)', async () => { - await ExportToCsvCommand.run([]); - assert.calledOnce(inquireStub); - }); +// afterEach(() => { +// inquireStub.restore(); +// errorStub.restore(); +// consoleLogStub.restore(); +// }); - it('Should ask for org when org is not passed', async () => { - const args = ['--action', 'entries']; - await ExportToCsvCommand.run(args); - assert.calledOnce(inquireStub); - }); +// it('Should ask for action when action is not passed (entries or users)', async () => { +// await ExportToCsvCommand.run([]); +// assert.calledOnce(inquireStub); +// }); - it('Should ask for stack when stack api key flag is not passed', async (done) => { - const args = ['--action', 'entries', '--org', process.env.ORG]; - done(); - await ExportToCsvCommand.run(args); - assert.calledOnce(inquireStub); - }); +// it('Should ask for org when org is not passed', async () => { +// const args = ['--action', 'entries']; +// await ExportToCsvCommand.run(args); +// assert.calledOnce(inquireStub); +// }); - it('Should ask for branch when branch flag is not passed', async () => { - const args = ['--action', 'entries', '--org', process.env.ORG, '--stack-api-key', process.env.STACK]; - await ExportToCsvCommand.run(args); - assert.calledTwice(inquireStub); - }); +// it('Should ask for stack when stack api key flag is not passed', async (done) => { +// const args = ['--action', 'entries', '--org', process.env.ORG]; +// done(); +// await ExportToCsvCommand.run(args); +// assert.calledOnce(inquireStub); +// }); - it('Should throw an error if stack does not have branches enabled', async () => { - const args = [ - '--action', - 'entries', - '--org', - process.env.ORG_WITH_NO_BRANCHES, - '--stack-api-key', - process.env.STACK_WITH_ORG_WITH_NO_BRANCHES, - '--branch', - 'invalid', - ]; - await ExportToCsvCommand.run(args); - assert.calledWith( - errorStub, - 'Branches are not part of your plan. Please contact support@contentstack.com to upgrade your plan.', - ); - }); +// it('Should ask for branch when branch flag is not passed', async () => { +// const args = ['--action', 'entries', '--org', process.env.ORG, '--stack-api-key', process.env.STACK]; +// await ExportToCsvCommand.run(args); +// assert.calledTwice(inquireStub); +// }); - it('Should ask for content type when content type flag is not passed', async () => { - const args = [ - '--action', - 'entries', - '--org', - process.env.ORG, - '--stack-api-key', - process.env.STACK, - '--branch', - process.env.BRANCH, - ]; - await ExportToCsvCommand.run(args); - assert.calledOnce(inquireStub); - }); +// it('Should throw an error if stack does not have branches enabled', async () => { +// const args = [ +// '--action', +// 'entries', +// '--org', +// process.env.ORG_WITH_NO_BRANCHES, +// '--stack-api-key', +// process.env.STACK_WITH_ORG_WITH_NO_BRANCHES, +// '--branch', +// 'invalid', +// ]; +// await ExportToCsvCommand.run(args); +// assert.calledWith( +// errorStub, +// 'Branches are not part of your plan. Please contact support@contentstack.com to upgrade your plan.', +// ); +// }); - it('Should create a file starting with the name passed as stack-name flag', async () => { - const args = [ - '--action', - 'entries', - '--org', - process.env.ORG, - '--stack-api-key', - process.env.STACK, - '--branch', - process.env.BRANCH, - '--content-type', - 'page', - '--locale', - 'en-us', - '--stack-name', - 'okok', - ]; - await ExportToCsvCommand.run(args); - assert.calledWith(consoleLogStub, `Writing entries to file: ${process.cwd()}/okok_page_en-us_entries_export.csv`); - }); +// it('Should ask for content type when content type flag is not passed', async () => { +// const args = [ +// '--action', +// 'entries', +// '--org', +// process.env.ORG, +// '--stack-api-key', +// process.env.STACK, +// '--branch', +// process.env.BRANCH, +// ]; +// await ExportToCsvCommand.run(args); +// assert.calledOnce(inquireStub); +// }); - it('Should throw an error when invalid org is passed', async () => { - const args = ['--action', 'entries', '--org', 'invalid']; - await ExportToCsvCommand.run(args); - assert.calledWith(errorStub, `Couldn't find the organization. Please check input parameters.`); - }); +// it('Should create a file starting with the name passed as stack-name flag', async () => { +// const args = [ +// '--action', +// 'entries', +// '--org', +// process.env.ORG, +// '--stack-api-key', +// process.env.STACK, +// '--branch', +// process.env.BRANCH, +// '--content-type', +// 'page', +// '--locale', +// 'en-us', +// '--stack-name', +// 'okok', +// ]; +// await ExportToCsvCommand.run(args); +// assert.calledWith(consoleLogStub, `Writing entries to file: ${process.cwd()}/okok_page_en-us_entries_export.csv`); +// }); - it('Should throw an error when invalid stack is passed', async () => { - const args = ['--action', 'entries', '--org', process.env.ORG, '--stack-api-key', 'invalid']; - await ExportToCsvCommand.run(args); - assert.calledWith(errorStub, 'Could not find stack'); - }); +// it('Should throw an error when invalid org is passed', async () => { +// const args = ['--action', 'entries', '--org', 'invalid']; +// await ExportToCsvCommand.run(args); +// assert.calledWith(errorStub, `Couldn't find the organization. Please check input parameters.`); +// }); - it('Should throw an error when invalid branch is passed', async () => { - const args = [ - '--action', - 'entries', - '--org', - process.env.ORG, - '--stack-api-key', - process.env.STACK, - '--branch', - process.env.INVALID_BRANCH, - ]; - await ExportToCsvCommand.run(args); - assert.calledWith(errorStub, 'Failed to fetch Branch. Please try again with valid parameters.'); - }); +// it('Should throw an error when invalid stack is passed', async () => { +// const args = ['--action', 'entries', '--org', process.env.ORG, '--stack-api-key', 'invalid']; +// await ExportToCsvCommand.run(args); +// assert.calledWith(errorStub, 'Could not find stack'); +// }); - it('Should throw an error when invalid contenttype is passed', async () => { - const args = [ - '--action', - 'entries', - '--org', - process.env.ORG, - '--stack-api-key', - process.env.STACK, - '--branch', - process.env.BRANCH, - '--content-type', - 'invalid', - '--locale', - 'en-us', - ]; - await ExportToCsvCommand.run(args); - assert.calledWith( - errorStub, - `The Content Type invalid was not found. Please try again. Content Type is not valid.`, - ); - }); +// it('Should throw an error when invalid branch is passed', async () => { +// const args = [ +// '--action', +// 'entries', +// '--org', +// process.env.ORG, +// '--stack-api-key', +// process.env.STACK, +// '--branch', +// process.env.INVALID_BRANCH, +// ]; +// await ExportToCsvCommand.run(args); +// assert.calledWith(errorStub, 'Failed to fetch Branch. Please try again with valid parameters.'); +// }); - it('Should throw an error when invalid locale is passed', async () => { - const args = [ - '--action', - 'entries', - '--org', - process.env.ORG, - '--stack-api-key', - process.env.STACK, - '--branch', - process.env.BRANCH, - '--content-type', - 'header', - '--locale', - 'invalid', - ]; - await ExportToCsvCommand.run(args); - assert.calledWith(errorStub, 'Language was not found. Please try again.'); - }); -}); +// it('Should throw an error when invalid contenttype is passed', async () => { +// const args = [ +// '--action', +// 'entries', +// '--org', +// process.env.ORG, +// '--stack-api-key', +// process.env.STACK, +// '--branch', +// process.env.BRANCH, +// '--content-type', +// 'invalid', +// '--locale', +// 'en-us', +// ]; +// await ExportToCsvCommand.run(args); +// assert.calledWith( +// errorStub, +// `The Content Type invalid was not found. Please try again. Content Type is not valid.`, +// ); +// }); -describe('Export to csv command with action = users', () => { - let inquireStub; - let errorStub; - let consoleLogStub; +// it('Should throw an error when invalid locale is passed', async () => { +// const args = [ +// '--action', +// 'entries', +// '--org', +// process.env.ORG, +// '--stack-api-key', +// process.env.STACK, +// '--branch', +// process.env.BRANCH, +// '--content-type', +// 'header', +// '--locale', +// 'invalid', +// ]; +// await ExportToCsvCommand.run(args); +// assert.calledWith(errorStub, 'Language was not found. Please try again.'); +// }); +// }); - beforeEach(() => { - inquireStub = stub(inquirer, 'prompt'); - errorStub = stub(cliux, 'error'); - consoleLogStub = stub(cliux, 'print'); - }); +// describe('Export to csv command with action = users', () => { +// let inquireStub; +// let errorStub; +// let consoleLogStub; + +// beforeEach(() => { +// inquireStub = stub(inquirer, 'prompt'); +// errorStub = stub(cliux, 'error'); +// consoleLogStub = stub(cliux, 'print'); +// }); + +// afterEach(() => { +// inquireStub.restore(); +// errorStub.restore(); +// consoleLogStub.restore(); +// }); + +// it('Should ask for org when org is not passed', async () => { +// const args = ['--action', 'entries']; +// await ExportToCsvCommand.run(args); +// assert.calledOnce(inquireStub); +// }); + +// it('Should write users data to file if the user has permissions', async () => { +// const args = ['--action', 'users', '--org', process.env.ORG]; + +// await ExportToCsvCommand.run(args); +// assert.calledWith( +// consoleLogStub, +// `Writing organization details to file: ${process.cwd()}/${process.env.ORG}_users_export.csv`, +// ); +// }); + +// it('Should show an error that user does not have org permissions to perform the operation if user enters such org', async () => { +// const args = ['--action', 'users', '--org', process.env.ORG_WITH_NO_PERMISSION]; +// await ExportToCsvCommand.run(args); +// assert.calledWith(errorStub, `You don't have the permission to do this operation.`); +// }); - afterEach(() => { - inquireStub.restore(); - errorStub.restore(); - consoleLogStub.restore(); +// it('Should create a file starting with the name passed as org-name flag', async () => { +// const args = ['--action', 'users', '--org', process.env.ORG, '--org-name', 'okok']; +// await ExportToCsvCommand.run(args); +// assert.calledWith(consoleLogStub, `Writing organization details to file: ${process.cwd()}/okok_users_export.csv`); +// }); +// }); + + +//!SECTION ----- test cases using @oclif/test ----------- + +describe('export-to-csv with action taxonomies', () => { + describe('Create taxonomies & terms csv file with all flags including taxonomy uid', () => { + test + .stdout({ print: process.env.PRINT === 'true' || false }) + .stub(mkdirp, 'sync', () => directory) + .stub(fs, 'existsSync', () => new PassThrough()) + .stub(fs, 'createWriteStream', () => new PassThrough()) + .nock(cma, (api) => { + api + .get(`/v3/stacks?&query={"org_uid":"${mockData.organizations[0].uid}"}`) + .reply(200, { stacks: mockData.stacks }); + }) + .nock(cma, (api) => { + api + .get('/v3/taxonomies/taxonomy_uid_1?include_count=true&skip=0&limit=30') + .reply(200, { taxonomy: mockData.taxonomiesResp.taxonomies[0] }); + }) + .nock(cma, (api) => { + api + .get('/v3/taxonomies/taxonomy_uid_1/terms?include_count=true&skip=0&limit=100') + .reply(200, { terms: mockData.termsResp.terms, count: mockData.termsResp.count }); + }) + .command([ + 'cm:export-to-csv', + '--action', + 'taxonomies', + '--taxonomy-uid', + 'taxonomy_uid_1', + '--stack-api-key', + mockData.stacks[0].api_key, + '--org', + mockData.organizations[0].uid, + ]) + .do(({ stdout }) => { + expect(stdout).to.contain(`Writing taxonomies to file: ${taxonomyFileName}_taxonomies.csv`); + //expect(stdout).to.contain(`Writing taxonomies to file: ${taxonomyFileName}_taxonomies.csv`) + }) + .it('CSV file should be created'); }); - it('Should ask for org when org is not passed', async () => { - const args = ['--action', 'entries']; - await ExportToCsvCommand.run(args); - assert.calledOnce(inquireStub); + describe('Create taxonomies & terms csv file with all flags excluding taxonomy uid', () => { + test + .stdout({ print: process.env.PRINT === "true" || false }) + .stub(fs, 'existsSync', () => new PassThrough()) + .stub(fs, 'createWriteStream', () => new PassThrough()) + .nock(cma, (api) => { + api + .get(`/v3/stacks?&query={"org_uid":"${mockData.organizations[0].uid}"}`) + .reply(200, { stacks: mockData.stacks }); + }) + .nock(cma, (api) => { + api + .get('/v3/taxonomies?include_count=true&skip=0&limit=100') + .reply(200, { taxonomies: mockData.taxonomiesResp.taxonomies, count: mockData.taxonomiesResp.count }); + }) + .nock(cma, (api) => { + api + .get('/v3/taxonomies/taxonomy_uid_1/terms?include_count=true&skip=0&limit=100') + .reply(200, { terms: mockData.termsResp.terms, count: mockData.termsResp.count }); + }) + .nock(cma, (api) => { + api + .get('/v3/taxonomies/taxonomy_uid_2/terms?include_count=true&skip=0&limit=100') + .reply(200, { terms: [], count: 0 }); + }) + .command([ + 'cm:export-to-csv', + '--action', + 'taxonomies', + '--stack-api-key', + mockData.stacks[0].api_key, + '--org', + mockData.organizations[0].uid, + ]) + .do(({ stdout }) => { + expect(stdout).to.contain(`Writing taxonomies to file: ${taxonomyFileName}_taxonomies.csv`); + //expect(stdout).to.contain(`Writing taxonomies to file: ${taxonomyFileName}_taxonomies.csv`) + }) + .it('file should be created'); }); - it('Should write users data to file if the user has permissions', async () => { - const args = ['--action', 'users', '--org', process.env.ORG]; + describe('Create taxonomies & terms csv file with prompt', () => { + test + .stdout({ print: process.env.PRINT === "true" || false }) + .stub(mkdirp, 'sync', () => directory) + .stub(fs, 'existsSync', () => new PassThrough()) + .stub(fs, 'createWriteStream', () => new PassThrough()) + .stub(inquirer, 'registerPrompt', () => {}) + .stub(inquirer, 'prompt', () => { + return Promise.resolve({ + action: 'taxonomies', + chosenOrg: mockData.organizations[0].name, + chosenStack: mockData.stacks[0].name, + }); + }) + .nock(cma, (api) => { + api + .get(`/v3/stacks?&query={"org_uid":"${mockData.organizations[0].uid}"}`) + .reply(200, { stacks: mockData.stacks }); + }) + .nock(cma, (api) => { + api.get(`/v3/organizations?limit=100`).reply(200, { organizations: mockData.organizations }); + }) + .nock(cma, (api) => { + api + .get('/v3/taxonomies/taxonomy_uid_1?include_count=true&skip=0&limit=30') + .reply(200, { taxonomy: mockData.taxonomiesResp.taxonomies[0] }); + }) + .nock(cma, (api) => { + api + .get('/v3/taxonomies/taxonomy_uid_1/terms?include_count=true&skip=0&limit=100') + .reply(200, { terms: mockData.termsResp.terms, count: mockData.termsResp.count }); + }) + .command(['cm:export-to-csv', '--taxonomy-uid', 'taxonomy_uid_1']) + .it('CSV file should be created'); + }); +}); - await ExportToCsvCommand.run(args); - assert.calledWith( - consoleLogStub, - `Writing organization details to file: ${process.cwd()}/${process.env.ORG}_users_export.csv`, - ); +describe('export-to-csv with action entries', () => { + describe('Create entries csv file with flags', () => { + test + .stdout({ print: true }) + .stub(fs, 'existsSync', () => new PassThrough()) + .stub(fs, 'createWriteStream', () => new PassThrough()) + //nock api for content type + //nock api for list branches + //nock api for branch + .nock(cma, (api) => { + api + .get(`/v3/stacks?&query={"org_uid":"${mockData.organizations[0].uid}"}`) + .reply(200, { stacks: mockData.stacks }); + }) + .nock(cma, (api) => { + api + .get('/v3/environments') + .reply(200, mockData.environments); + }) + .nock(cma, (api) => { + api + .get('/v3/stack/branches') + .reply(200, { branch: mockData.branch }); + }) + .nock(cma, (api) => { + api + .get('/v3/content_types?count=true') + .reply(200, { count: 2 }); + }) + .nock(cma, (api) => { + api + .get('/v3/content_types') + .reply(200, { resp: mockData.contentTypes }); + }) + .nock(cma, (api) => { + api + .get(`/v3/content_types/${mockData.contentTypes.items[0].uid}/entries`) + .reply(200, { entry: mockData.entry }); + }) + .command([ + 'cm:export-to-csv', + '--action', + 'entries', + '--stack-api-key', + mockData.stacks[0].api_key, + '--org', + mockData.organizations[0].uid, + '--branch', + 'test_branch1', + '--locale', + 'en1', + '--content-type', + 'ct1' + ]) + .do(({ stdout }) => { + //expect(stdout).to.contain(`Writing taxonomies to file: ${taxonomyFileName}_taxonomies.csv`); + //expect(stdout).to.contain(`Writing taxonomies to file: ${taxonomyFileName}_taxonomies.csv`) + }) + .it('CSV file should be created'); }); - it('Should show an error that user does not have org permissions to perform the operation if user enters such org', async () => { - const args = ['--action', 'users', '--org', process.env.ORG_WITH_NO_PERMISSION]; - await ExportToCsvCommand.run(args); - assert.calledWith(errorStub, `You don't have the permission to do this operation.`); + // describe('Create entries csv file with prompt', () => { + // test + // .stdout({ print: process.env.PRINT === "true" || false }) + // .stub(fs, 'existsSync', () => new PassThrough()) + // .stub(fs, 'createWriteStream', () => new PassThrough()) + // .stub(inquirer, 'registerPrompt', () => {}) + // .stub(inquirer, 'prompt', () => { + // return Promise.resolve({ + // action: 'taxonomies', + // chosenOrg: mockData.organizations[0].name, + // chosenStack: mockData.stacks[0].name, + // }); + // }) + // .nock(cma, (api) => { + // api + // .get(`/v3/stacks?&query={"org_uid":"${mockData.organizations[0].uid}"}`) + // .reply(200, { stacks: mockData.stacks }); + // }) + // .nock(cma, (api) => { + // api.get(`/v3/organizations?limit=100`).reply(200, { organizations: mockData.organizations }); + // }) + // .nock(cma, (api) => { + // api + // .get('/v3/taxonomies/taxonomy_uid_1?include_count=true&skip=0&limit=30') + // .reply(200, { taxonomy: mockData.taxonomiesResp.taxonomies[0] }); + // }) + // .nock(cma, (api) => { + // api + // .get('/v3/taxonomies/taxonomy_uid_1/terms?include_count=true&skip=0&limit=100') + // .reply(200, { terms: mockData.termsResp.terms, count: mockData.termsResp.count }); + // }) + // .command(['cm:export-to-csv', '--taxonomy-uid', 'taxonomy_uid_1']) + // .it('CSV file should be created'); + // }); +}); + +describe('export-to-csv with action users', () => { + describe('Export users csv file with flags', () => { + test + .stdout({ print: process.env.PRINT === 'true' || false }) + .stub(cliux, 'print', (msg) => console.log(msg)) + .stub(mkdirp, 'sync', () => directory) + .stub(fs, 'existsSync', () => new PassThrough()) + .stub(fs, 'createWriteStream', () => new PassThrough()) + .nock(cma, (api) => { + api.get('/v3/user?include_orgs_roles=true').reply(200, { user: mockData.users.items[0] }); + }) + .nock(cma, (api) => { + api.get(`/v3/organizations/${mockData.organizations[0].uid}/roles`).reply(200, { roles: mockData.roles }); + }) + .nock(cma, (api) => { + api.get('/v3/user?include_orgs_roles=true').reply(200, { user: mockData.users.items[0] }); + }) + .nock(cma, (api) => { + api + .get(`/v3/organizations/${mockData.organizations[0].uid}/share?skip=0&page=1&limit=100`) + .reply(200, { users: mockData.users.items }); + }) + .command(['cm:export-to-csv', '--action', 'users', '--org', mockData.organizations[0].uid]) + // .do(({ stdout }) => + // expect(stdout).to.contain(`Writing organization details to file: ${userFileName}_users_export.csv`), + // ) + .it('Users csv file should be successfully created'); }); - it('Should create a file starting with the name passed as org-name flag', async () => { - const args = ['--action', 'users', '--org', process.env.ORG, '--org-name', 'okok']; - await ExportToCsvCommand.run(args); - assert.calledWith(consoleLogStub, `Writing organization details to file: ${process.cwd()}/okok_users_export.csv`); + describe('Export users csv file with prompt', () => { + test + .stdout({ print: process.env.PRINT === 'true' || false }) + .stub(cliux, 'print', (msg) => console.log(msg)) + .stub(mkdirp, 'sync', () => directory) + .stub(fs, 'existsSync', () => new PassThrough()) + .stub(fs, 'createWriteStream', () => new PassThrough()) + .stub(inquirer, 'registerPrompt', () => {}) + .stub(inquirer, 'prompt', () => { + return Promise.resolve({ action: 'users', chosenOrg: mockData.organizations[0].name }); + }) + .nock(cma, (api) => { + api.get(`/v3/organizations?limit=100`).reply(200, { organizations: mockData.organizations }); + }) + .nock(cma, (api) => { + api.get('/v3/user?include_orgs_roles=true').reply(200, { user: mockData.users.items[0] }); + }) + .nock(cma, (api) => { + api.get(`/v3/organizations/${mockData.organizations[0].uid}/roles`).reply(200, { roles: mockData.roles }); + }) + .nock(cma, (api) => { + api.get('/v3/user?include_orgs_roles=true').reply(200, { user: mockData.users.items[0] }); + }) + .nock(cma, (api) => { + api + .get(`/v3/organizations/${mockData.organizations[0].uid}/share?skip=0&page=1&limit=100`) + .reply(200, { users: mockData.users.items }); + }) + .command(['cm:export-to-csv']) + // .do(({ stdout }) => + // expect(stdout).to.contain(`Writing organization details to file: ${userFileName}_users_export.csv`), + // ) + .it('Users csv file should be successfully created'); }); }); + +.nock(cma, (api) => { + api + .get(`/v3/stacks?&query={"org_uid":"${mockData.organizations[0].uid}"}`) + .reply(200, { stacks: mockData.stacks }); +}) From acdb34379fbe4b480ab92e3c3c0abbdd64da58c9 Mon Sep 17 00:00:00 2001 From: Aman Kumar Date: Wed, 20 Sep 2023 20:38:00 +0530 Subject: [PATCH 09/45] feat: export taxonomies & terms --- .../contentstack-export/src/config/index.ts | 11 + .../src/export/modules/taxonomies.ts | 208 ++++++++++++++++++ .../src/types/default-config.ts | 12 + .../contentstack-export/src/types/index.ts | 19 +- 4 files changed, 249 insertions(+), 1 deletion(-) create mode 100644 packages/contentstack-export/src/export/modules/taxonomies.ts diff --git a/packages/contentstack-export/src/config/index.ts b/packages/contentstack-export/src/config/index.ts index 52a0bd214b..07656c3859 100644 --- a/packages/contentstack-export/src/config/index.ts +++ b/packages/contentstack-export/src/config/index.ts @@ -25,6 +25,7 @@ const config: DefaultConfig = { 'environments', 'extensions', 'webhooks', + 'taxonomies', 'global-fields', 'content-types', 'custom-roles', @@ -158,6 +159,16 @@ const config: DefaultConfig = { dirName: 'marketplace_apps', fileName: 'marketplace_apps.json', }, + taxonomies: { + dirName: 'taxonomies', + fileName: 'taxonomies.json', + invalidKeys: ['created_at', 'updated_at', 'created_by', 'updated_by'], + }, + terms: { + dirName: 'terms', + fileName: 'terms.json', + invalidKeys: ['created_at', 'updated_at', 'created_by', 'updated_by'], + }, }, languagesCode: [ 'af-za', diff --git a/packages/contentstack-export/src/export/modules/taxonomies.ts b/packages/contentstack-export/src/export/modules/taxonomies.ts new file mode 100644 index 0000000000..fbe2d52641 --- /dev/null +++ b/packages/contentstack-export/src/export/modules/taxonomies.ts @@ -0,0 +1,208 @@ +import omit from 'lodash/omit'; +import isEmpty from 'lodash/isEmpty'; +import flatten from 'lodash/flatten'; +import keys from 'lodash/keys'; +import { resolve as pResolve } from 'node:path'; +import { cliux, configHandler, HttpClient } from '@contentstack/cli-utilities'; + +import BaseClass from './base-class'; +import { log, fsUtil } from '../../utils'; +import { TaxonomiesConfig, TermsConfig, ModuleClassParams } from '../../types'; + +//NOTE: Temp types to request api need to remove once sdk ready +type TaxonomyPayload = { + baseUrl: string; + url: string; + mgToken: string; + apiKey: string; +}; + +export default class ExportTaxonomies extends BaseClass { + private taxonomies: Record>; + private terms: Record>; + private taxonomiesConfig: TaxonomiesConfig; + private termsConfig: TermsConfig; + public taxonomiesFolderPath: string; + public termsFolderPath: string; + private taxonomyPayload: TaxonomyPayload; + + constructor({ exportConfig, stackAPIClient }: ModuleClassParams) { + super({ exportConfig, stackAPIClient }); + this.taxonomies = {}; + this.terms = {}; + this.taxonomiesConfig = exportConfig.modules.taxonomies; + this.termsConfig = exportConfig.modules.terms; + this.taxonomyPayload = { + baseUrl: '', + url: '', + mgToken: exportConfig.management_token, + apiKey: exportConfig.source_stack, + }; + } + + async start(): Promise { + log(this.exportConfig, 'Starting taxonomies export', 'info'); + + //create taxonomies and terms folder in data directory path + this.taxonomiesFolderPath = pResolve( + this.exportConfig.data, + this.exportConfig.branchName || '', + this.taxonomiesConfig.dirName, + ); + await fsUtil.makeDirectory(this.taxonomiesFolderPath); + this.termsFolderPath = pResolve(this.taxonomiesFolderPath, this.termsConfig.dirName); + await fsUtil.makeDirectory(this.termsFolderPath); + + const { cma } = configHandler.get('region') || {}; + this.taxonomyPayload.baseUrl = `${cma}/v3/taxonomies`; + this.taxonomyPayload.url = this.taxonomyPayload.baseUrl; + + //fetch all taxonomies and write into taxonomies folder + await this.getAllTaxonomies(this.taxonomyPayload); + if (this.taxonomies === undefined || isEmpty(this.taxonomies)) { + log(this.exportConfig, 'No taxonomies found', 'info'); + return; + } else { + fsUtil.writeFile(pResolve(this.taxonomiesFolderPath, this.taxonomiesConfig.fileName), this.taxonomies); + log(this.exportConfig, 'All the taxonomies have been exported successfully!', 'success'); + } + + //fetch all terms of respective and write into taxonomies/terms folder + await this.getAllTerms(); + } + + /** + * fetch all taxonomies in the provided stack + * @param {TaxonomyPayload} payload + * @param {number} skip + * @returns + */ + async getAllTaxonomies(payload: TaxonomyPayload, skip = 0): Promise { + const response = await this.apiRequestHandler(payload, skip); + if (response) { + skip += this.taxonomiesConfig.limit || 100; + this.sanitizeTaxonomiesAttribs(response.taxonomies); + if (skip >= response?.count) { + return; + } else { + return await this.getAllTaxonomies(payload, skip); + } + } + return; + } + + /** + * remove invalid keys and write data into taxonomies + * @function sanitizeTaxonomiesAttribs + * @param taxonomies + */ + sanitizeTaxonomiesAttribs(taxonomies: Record[]) { + for (let index = 0; index < taxonomies?.length; index++) { + const taxonomyUID = taxonomies[index].uid; + const taxonomyName = taxonomies[index]?.name; + this.taxonomies[taxonomyUID] = omit(taxonomies[index], this.taxonomiesConfig.invalidKeys); + log(this.exportConfig, `'${taxonomyName}' taxonomy was exported successfully`, 'success'); + } + } + + /** + * fetch all terms of respective taxonomy and write it into -terms file + */ + async getAllTerms() { + const taxonomiesUID = keys(this.taxonomies) || []; + for (let index = 0; index < taxonomiesUID?.length; index++) { + const taxonomyUID = taxonomiesUID[index]; + this.taxonomyPayload.url = `${this.taxonomyPayload.baseUrl}/${taxonomyUID}/terms`; + this.terms = {}; + await this.fetchTermsOfTaxonomy(this.taxonomyPayload); + + if (this.terms === undefined || isEmpty(this.terms)) { + log(this.exportConfig, `No terms found for taxonomy - '${taxonomyUID}'`, 'info'); + } else { + fsUtil.writeFile(pResolve(this.termsFolderPath, `${taxonomyUID}-${this.termsConfig.fileName}`), this.terms); + log(this.exportConfig, `Taxonomy - '${taxonomyUID}' terms was exported successfully`, 'success'); + } + } + log(this.exportConfig, `All the terms have been exported successfully!`, 'success'); + } + + /** + * fetch all terms of the provided taxonomy uid + * @param {TaxonomyPayload} payload + * @param {number} skip + * @returns + */ + async fetchTermsOfTaxonomy(payload: TaxonomyPayload, skip = 0): Promise { + const response = await this.apiRequestHandler(payload, skip); + if (response) { + skip += this.termsConfig.limit || 100; + this.sanitizeTermsAttribs(response.terms); + if (skip >= response?.count) { + return; + } else { + return await this.fetchTermsOfTaxonomy(payload, skip); + } + } + return; + } + + /** + * remove invalid keys and write data into taxonomies + * @function sanitizeTaxonomiesAttribs + * @param terms + */ + sanitizeTermsAttribs(terms: Record[]) { + for (let index = 0; index < terms?.length; index++) { + const termUID = terms[index]?.uid; + this.terms[termUID] = omit(terms[index], this.termsConfig.invalidKeys); + } + } + + async apiRequestHandler(payload: TaxonomyPayload, skip: number) { + const headers: any = { + api_key: payload.apiKey, + 'Content-Type': 'application/json', + }; + + if (payload?.mgToken) headers['authorization'] = payload.mgToken; + else headers['authToken'] = configHandler.get('authtoken'); + + const params = { + include_count: true, + skip: 0, + limit: this.taxonomiesConfig.limit || 100, + }; + + if (skip >= 0) params['skip'] = skip; + + return await new HttpClient() + .headers(headers) + .queryParams(params) + .get(payload.url) + .then((res: any) => { + //NOTE - temporary code for handling api errors response + const { status, data } = res; + if ([200, 201, 202].includes(status)) return data; + else { + let errorMsg; + if ([500, 503, 502].includes(status)) errorMsg = data?.message || data; + else errorMsg = data?.error_message; + if (errorMsg === undefined) { + errorMsg = Object.values(data?.errors) && flatten(Object.values(data.errors)); + } + cliux.print(`Error: ${errorMsg}`, { color: 'red' }); + } + }) + .catch((err: any) => this.handleErrorMsg(err)); + } + + handleErrorMsg(err: any) { + if (err?.errorMessage) { + cliux.print(`Error: ${err.errorMessage}`, { color: 'red' }); + } else if (err?.message) { + cliux.print(`Error: ${err.message}`, { color: 'red' }); + } else { + cliux.print(`Error: Something went wrong. Please try again.`, { color: 'red' }); + } + } +} diff --git a/packages/contentstack-export/src/types/default-config.ts b/packages/contentstack-export/src/types/default-config.ts index 918786bdac..d7671a5eb6 100644 --- a/packages/contentstack-export/src/types/default-config.ts +++ b/packages/contentstack-export/src/types/default-config.ts @@ -143,6 +143,18 @@ export default interface DefaultConfig { fileName: string; requiredKeys: string[]; }; + taxonomies: { + dirName: string; + fileName: string; + invalidKeys: string[]; + dependencies?: Modules[]; + }; + terms: { + dirName: string; + fileName: string; + invalidKeys: string[]; + dependencies?: Modules[]; + }; }; languagesCode: string[]; updatedModules: string[]; diff --git a/packages/contentstack-export/src/types/index.ts b/packages/contentstack-export/src/types/index.ts index ccf95c0918..ee5121491b 100644 --- a/packages/contentstack-export/src/types/index.ts +++ b/packages/contentstack-export/src/types/index.ts @@ -40,7 +40,8 @@ export type Modules = | 'custom-roles' | 'workflows' | 'labels' - | 'marketplace-apps'; + | 'marketplace-apps' + | 'taxonomies'; export type ModuleClassParams = { stackAPIClient: ReturnType; @@ -121,5 +122,21 @@ export interface StackConfig{ limit?: number; } +export interface TaxonomiesConfig{ + dirName: string; + fileName: string; + invalidKeys: string[]; + dependencies?: Modules[]; + limit?: number; +} + +export interface TermsConfig{ + dirName: string; + fileName: string; + invalidKeys: string[]; + dependencies?: Modules[]; + limit?: number; +} + export { default as DefaultConfig } from './default-config'; export { default as ExportConfig } from './export-config'; From 582adddcac47c25104dcd16818c72451d00d8c41 Mon Sep 17 00:00:00 2001 From: Aman Kumar Date: Thu, 21 Sep 2023 15:04:58 +0530 Subject: [PATCH 10/45] fix: remove unused package --- .../test/unit/commands/export-to-csv.test.js | 9 --------- 1 file changed, 9 deletions(-) diff --git a/packages/contentstack-export-to-csv/test/unit/commands/export-to-csv.test.js b/packages/contentstack-export-to-csv/test/unit/commands/export-to-csv.test.js index 965a593cbb..ba036d306d 100644 --- a/packages/contentstack-export-to-csv/test/unit/commands/export-to-csv.test.js +++ b/packages/contentstack-export-to-csv/test/unit/commands/export-to-csv.test.js @@ -14,15 +14,6 @@ const directory = './data'; const taxonomyFileName = join(process.cwd(), 'data', mockData.stacks[0].name); const userFileName = join(process.cwd(), 'data', mockData.organizations[0].name); - -/* eslint-disable no-undef */ -const { describe, it, beforeEach, afterEach } = require('mocha'); -const ExportToCsvCommand = require('../../../src/commands/cm/export-to-csv'); -const { stub, assert } = require('sinon'); -const { config } = require('dotenv'); -const inquirer = require('inquirer'); -const { cliux } = require('@contentstack/cli-utilities'); - //!SECTION ------old test cases ------------- // config(); From e44cd139c0260c8db06c9f4fd887d3d41c54a769 Mon Sep 17 00:00:00 2001 From: Aman Kumar Date: Thu, 21 Sep 2023 19:36:41 +0530 Subject: [PATCH 11/45] refactor: message --- .../src/export/modules/taxonomies.ts | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/packages/contentstack-export/src/export/modules/taxonomies.ts b/packages/contentstack-export/src/export/modules/taxonomies.ts index fbe2d52641..fa463a880f 100644 --- a/packages/contentstack-export/src/export/modules/taxonomies.ts +++ b/packages/contentstack-export/src/export/modules/taxonomies.ts @@ -9,7 +9,7 @@ import BaseClass from './base-class'; import { log, fsUtil } from '../../utils'; import { TaxonomiesConfig, TermsConfig, ModuleClassParams } from '../../types'; -//NOTE: Temp types to request api need to remove once sdk ready +//NOTE: Temp types need to remove once sdk available type TaxonomyPayload = { baseUrl: string; url: string; @@ -21,10 +21,10 @@ export default class ExportTaxonomies extends BaseClass { private taxonomies: Record>; private terms: Record>; private taxonomiesConfig: TaxonomiesConfig; + private taxonomyPayload: TaxonomyPayload; private termsConfig: TermsConfig; public taxonomiesFolderPath: string; public termsFolderPath: string; - private taxonomyPayload: TaxonomyPayload; constructor({ exportConfig, stackAPIClient }: ModuleClassParams) { super({ exportConfig, stackAPIClient }); @@ -79,7 +79,7 @@ export default class ExportTaxonomies extends BaseClass { */ async getAllTaxonomies(payload: TaxonomyPayload, skip = 0): Promise { const response = await this.apiRequestHandler(payload, skip); - if (response) { + if (response?.taxonomies) { skip += this.taxonomiesConfig.limit || 100; this.sanitizeTaxonomiesAttribs(response.taxonomies); if (skip >= response?.count) { @@ -120,7 +120,7 @@ export default class ExportTaxonomies extends BaseClass { log(this.exportConfig, `No terms found for taxonomy - '${taxonomyUID}'`, 'info'); } else { fsUtil.writeFile(pResolve(this.termsFolderPath, `${taxonomyUID}-${this.termsConfig.fileName}`), this.terms); - log(this.exportConfig, `Taxonomy - '${taxonomyUID}' terms was exported successfully`, 'success'); + log(this.exportConfig, `Terms from taxonomy '${taxonomyUID}' were successfully exported.`, 'success'); } } log(this.exportConfig, `All the terms have been exported successfully!`, 'success'); @@ -134,7 +134,7 @@ export default class ExportTaxonomies extends BaseClass { */ async fetchTermsOfTaxonomy(payload: TaxonomyPayload, skip = 0): Promise { const response = await this.apiRequestHandler(payload, skip); - if (response) { + if (response?.terms) { skip += this.termsConfig.limit || 100; this.sanitizeTermsAttribs(response.terms); if (skip >= response?.count) { @@ -158,6 +158,7 @@ export default class ExportTaxonomies extends BaseClass { } } + //NOTE: Temp code need to remove once sdk available async apiRequestHandler(payload: TaxonomyPayload, skip: number) { const headers: any = { api_key: payload.apiKey, @@ -196,6 +197,7 @@ export default class ExportTaxonomies extends BaseClass { .catch((err: any) => this.handleErrorMsg(err)); } + //NOTE: Temp code need to remove once sdk available handleErrorMsg(err: any) { if (err?.errorMessage) { cliux.print(`Error: ${err.errorMessage}`, { color: 'red' }); From b28f8a8ae87591ca0b53bbf92c087580f716bfab Mon Sep 17 00:00:00 2001 From: Aman Kumar Date: Mon, 25 Sep 2023 15:28:38 +0530 Subject: [PATCH 12/45] fix test cases --- .../test/unit/commands/export-to-csv.test.js | 352 ++++-------------- 1 file changed, 64 insertions(+), 288 deletions(-) diff --git a/packages/contentstack-export-to-csv/test/unit/commands/export-to-csv.test.js b/packages/contentstack-export-to-csv/test/unit/commands/export-to-csv.test.js index ba036d306d..06e6ad61eb 100644 --- a/packages/contentstack-export-to-csv/test/unit/commands/export-to-csv.test.js +++ b/packages/contentstack-export-to-csv/test/unit/commands/export-to-csv.test.js @@ -1,5 +1,4 @@ const fs = require('fs'); -const mkdirp = require('mkdirp'); const { test, expect } = require('@oclif/test'); const { join } = require('path'); const { PassThrough } = require('stream'); @@ -10,232 +9,14 @@ const { cliux, configHandler } = require('@contentstack/cli-utilities'); const mockData = require('../../mock-data/common.mock.json'); const { cma } = configHandler.get('region'); -const directory = './data'; -const taxonomyFileName = join(process.cwd(), 'data', mockData.stacks[0].name); -const userFileName = join(process.cwd(), 'data', mockData.organizations[0].name); - -//!SECTION ------old test cases ------------- - -// config(); - -// describe('Export to csv command with action = entries', () => { -// let inquireStub; -// let errorStub; -// let consoleLogStub; - -// beforeEach(() => { -// inquireStub = stub(inquirer, 'prompt'); -// errorStub = stub(cliux, 'error'); -// consoleLogStub = stub(cliux, 'print'); -// }); - -// afterEach(() => { -// inquireStub.restore(); -// errorStub.restore(); -// consoleLogStub.restore(); -// }); - -// it('Should ask for action when action is not passed (entries or users)', async () => { -// await ExportToCsvCommand.run([]); -// assert.calledOnce(inquireStub); -// }); - -// it('Should ask for org when org is not passed', async () => { -// const args = ['--action', 'entries']; -// await ExportToCsvCommand.run(args); -// assert.calledOnce(inquireStub); -// }); - -// it('Should ask for stack when stack api key flag is not passed', async (done) => { -// const args = ['--action', 'entries', '--org', process.env.ORG]; -// done(); -// await ExportToCsvCommand.run(args); -// assert.calledOnce(inquireStub); -// }); - -// it('Should ask for branch when branch flag is not passed', async () => { -// const args = ['--action', 'entries', '--org', process.env.ORG, '--stack-api-key', process.env.STACK]; -// await ExportToCsvCommand.run(args); -// assert.calledTwice(inquireStub); -// }); - -// it('Should throw an error if stack does not have branches enabled', async () => { -// const args = [ -// '--action', -// 'entries', -// '--org', -// process.env.ORG_WITH_NO_BRANCHES, -// '--stack-api-key', -// process.env.STACK_WITH_ORG_WITH_NO_BRANCHES, -// '--branch', -// 'invalid', -// ]; -// await ExportToCsvCommand.run(args); -// assert.calledWith( -// errorStub, -// 'Branches are not part of your plan. Please contact support@contentstack.com to upgrade your plan.', -// ); -// }); - -// it('Should ask for content type when content type flag is not passed', async () => { -// const args = [ -// '--action', -// 'entries', -// '--org', -// process.env.ORG, -// '--stack-api-key', -// process.env.STACK, -// '--branch', -// process.env.BRANCH, -// ]; -// await ExportToCsvCommand.run(args); -// assert.calledOnce(inquireStub); -// }); - -// it('Should create a file starting with the name passed as stack-name flag', async () => { -// const args = [ -// '--action', -// 'entries', -// '--org', -// process.env.ORG, -// '--stack-api-key', -// process.env.STACK, -// '--branch', -// process.env.BRANCH, -// '--content-type', -// 'page', -// '--locale', -// 'en-us', -// '--stack-name', -// 'okok', -// ]; -// await ExportToCsvCommand.run(args); -// assert.calledWith(consoleLogStub, `Writing entries to file: ${process.cwd()}/okok_page_en-us_entries_export.csv`); -// }); - -// it('Should throw an error when invalid org is passed', async () => { -// const args = ['--action', 'entries', '--org', 'invalid']; -// await ExportToCsvCommand.run(args); -// assert.calledWith(errorStub, `Couldn't find the organization. Please check input parameters.`); -// }); - -// it('Should throw an error when invalid stack is passed', async () => { -// const args = ['--action', 'entries', '--org', process.env.ORG, '--stack-api-key', 'invalid']; -// await ExportToCsvCommand.run(args); -// assert.calledWith(errorStub, 'Could not find stack'); -// }); - -// it('Should throw an error when invalid branch is passed', async () => { -// const args = [ -// '--action', -// 'entries', -// '--org', -// process.env.ORG, -// '--stack-api-key', -// process.env.STACK, -// '--branch', -// process.env.INVALID_BRANCH, -// ]; -// await ExportToCsvCommand.run(args); -// assert.calledWith(errorStub, 'Failed to fetch Branch. Please try again with valid parameters.'); -// }); - -// it('Should throw an error when invalid contenttype is passed', async () => { -// const args = [ -// '--action', -// 'entries', -// '--org', -// process.env.ORG, -// '--stack-api-key', -// process.env.STACK, -// '--branch', -// process.env.BRANCH, -// '--content-type', -// 'invalid', -// '--locale', -// 'en-us', -// ]; -// await ExportToCsvCommand.run(args); -// assert.calledWith( -// errorStub, -// `The Content Type invalid was not found. Please try again. Content Type is not valid.`, -// ); -// }); - -// it('Should throw an error when invalid locale is passed', async () => { -// const args = [ -// '--action', -// 'entries', -// '--org', -// process.env.ORG, -// '--stack-api-key', -// process.env.STACK, -// '--branch', -// process.env.BRANCH, -// '--content-type', -// 'header', -// '--locale', -// 'invalid', -// ]; -// await ExportToCsvCommand.run(args); -// assert.calledWith(errorStub, 'Language was not found. Please try again.'); -// }); -// }); - -// describe('Export to csv command with action = users', () => { -// let inquireStub; -// let errorStub; -// let consoleLogStub; - -// beforeEach(() => { -// inquireStub = stub(inquirer, 'prompt'); -// errorStub = stub(cliux, 'error'); -// consoleLogStub = stub(cliux, 'print'); -// }); - -// afterEach(() => { -// inquireStub.restore(); -// errorStub.restore(); -// consoleLogStub.restore(); -// }); - -// it('Should ask for org when org is not passed', async () => { -// const args = ['--action', 'entries']; -// await ExportToCsvCommand.run(args); -// assert.calledOnce(inquireStub); -// }); - -// it('Should write users data to file if the user has permissions', async () => { -// const args = ['--action', 'users', '--org', process.env.ORG]; - -// await ExportToCsvCommand.run(args); -// assert.calledWith( -// consoleLogStub, -// `Writing organization details to file: ${process.cwd()}/${process.env.ORG}_users_export.csv`, -// ); -// }); - -// it('Should show an error that user does not have org permissions to perform the operation if user enters such org', async () => { -// const args = ['--action', 'users', '--org', process.env.ORG_WITH_NO_PERMISSION]; -// await ExportToCsvCommand.run(args); -// assert.calledWith(errorStub, `You don't have the permission to do this operation.`); -// }); - -// it('Should create a file starting with the name passed as org-name flag', async () => { -// const args = ['--action', 'users', '--org', process.env.ORG, '--org-name', 'okok']; -// await ExportToCsvCommand.run(args); -// assert.calledWith(consoleLogStub, `Writing organization details to file: ${process.cwd()}/okok_users_export.csv`); -// }); -// }); - - -//!SECTION ----- test cases using @oclif/test ----------- +// const directory = './data'; +// const taxonomyFileName = join(process.cwd(), 'data', mockData.stacks[0].name); +// const userFileName = join(process.cwd(), 'data', mockData.organizations[0].name); describe('export-to-csv with action taxonomies', () => { describe('Create taxonomies & terms csv file with all flags including taxonomy uid', () => { test - .stdout({ print: process.env.PRINT === 'true' || false }) - .stub(mkdirp, 'sync', () => directory) + .stdout({ print: true }) .stub(fs, 'existsSync', () => new PassThrough()) .stub(fs, 'createWriteStream', () => new PassThrough()) .nock(cma, (api) => { @@ -270,7 +51,6 @@ describe('export-to-csv with action taxonomies', () => { }) .it('CSV file should be created'); }); - describe('Create taxonomies & terms csv file with all flags excluding taxonomy uid', () => { test .stdout({ print: process.env.PRINT === "true" || false }) @@ -311,7 +91,6 @@ describe('export-to-csv with action taxonomies', () => { }) .it('file should be created'); }); - describe('Create taxonomies & terms csv file with prompt', () => { test .stdout({ print: process.env.PRINT === "true" || false }) @@ -364,24 +143,16 @@ describe('export-to-csv with action entries', () => { .reply(200, { stacks: mockData.stacks }); }) .nock(cma, (api) => { - api - .get('/v3/environments') - .reply(200, mockData.environments); + api.get('/v3/environments').reply(200, mockData.environments); }) .nock(cma, (api) => { - api - .get('/v3/stack/branches') - .reply(200, { branch: mockData.branch }); + api.get('/v3/stack/branches').reply(200, { branch: mockData.branch }); }) .nock(cma, (api) => { - api - .get('/v3/content_types?count=true') - .reply(200, { count: 2 }); + api.get('/v3/content_types?count=true').reply(200, { count: 2 }); }) .nock(cma, (api) => { - api - .get('/v3/content_types') - .reply(200, { resp: mockData.contentTypes }); + api.get('/v3/content_types').reply(200, { resp: mockData.contentTypes }); }) .nock(cma, (api) => { api @@ -397,61 +168,74 @@ describe('export-to-csv with action entries', () => { '--org', mockData.organizations[0].uid, '--branch', - 'test_branch1', + mockData.branch.uid, '--locale', 'en1', '--content-type', - 'ct1' + mockData.contentTypes.items[0].uid, ]) .do(({ stdout }) => { //expect(stdout).to.contain(`Writing taxonomies to file: ${taxonomyFileName}_taxonomies.csv`); //expect(stdout).to.contain(`Writing taxonomies to file: ${taxonomyFileName}_taxonomies.csv`) }) - .it('CSV file should be created'); + .it('Entries CSV file should be created'); }); - // describe('Create entries csv file with prompt', () => { - // test - // .stdout({ print: process.env.PRINT === "true" || false }) - // .stub(fs, 'existsSync', () => new PassThrough()) - // .stub(fs, 'createWriteStream', () => new PassThrough()) - // .stub(inquirer, 'registerPrompt', () => {}) - // .stub(inquirer, 'prompt', () => { - // return Promise.resolve({ - // action: 'taxonomies', - // chosenOrg: mockData.organizations[0].name, - // chosenStack: mockData.stacks[0].name, - // }); - // }) - // .nock(cma, (api) => { - // api - // .get(`/v3/stacks?&query={"org_uid":"${mockData.organizations[0].uid}"}`) - // .reply(200, { stacks: mockData.stacks }); - // }) - // .nock(cma, (api) => { - // api.get(`/v3/organizations?limit=100`).reply(200, { organizations: mockData.organizations }); - // }) - // .nock(cma, (api) => { - // api - // .get('/v3/taxonomies/taxonomy_uid_1?include_count=true&skip=0&limit=30') - // .reply(200, { taxonomy: mockData.taxonomiesResp.taxonomies[0] }); - // }) - // .nock(cma, (api) => { - // api - // .get('/v3/taxonomies/taxonomy_uid_1/terms?include_count=true&skip=0&limit=100') - // .reply(200, { terms: mockData.termsResp.terms, count: mockData.termsResp.count }); - // }) - // .command(['cm:export-to-csv', '--taxonomy-uid', 'taxonomy_uid_1']) - // .it('CSV file should be created'); - // }); + describe('Create entries csv file with prompt', () => { + test + .stdout({ print: true }) + .stub(fs, 'existsSync', () => new PassThrough()) + .stub(fs, 'createWriteStream', () => new PassThrough()) + .stub(inquirer, 'registerPrompt', () => {}) + .stub(inquirer, 'prompt', () => { + return Promise.resolve({ + action: 'entries', + chosenOrg: mockData.organizations[0].name, + chosenStack: mockData.stacks[0].name, + chosenLanguage: 'en1', + chosenContentTypes: mockData.contentTypes.items[0].uid, + branch: mockData.branch.uid, + }); + }) + .nock(cma, (api) => { + api + .get(`/v3/stacks?&query={"org_uid":"${mockData.organizations[0].uid}"}`) + .reply(200, { stacks: mockData.stacks }); + }) + .nock(cma, (api) => { + api.get(`/v3/organizations?limit=100`).reply(200, { organizations: mockData.organizations }); + }) + .nock(cma, (api) => { + api + .get(`/v3/stacks?&query={"org_uid":"${mockData.organizations[0].uid}"}`) + .reply(200, { stacks: mockData.stacks }); + }) + .nock(cma, (api) => { + api.get('/v3/environments').reply(200, mockData.environments); + }) + .nock(cma, (api) => { + api.get('/v3/stack/branches').reply(200, { branch: mockData.branch }); + }) + .nock(cma, (api) => { + api.get('/v3/content_types?count=true').reply(200, { count: 2 }); + }) + .nock(cma, (api) => { + api.get('/v3/content_types').reply(200, { resp: mockData.contentTypes }); + }) + .nock(cma, (api) => { + api + .get(`/v3/content_types/${mockData.contentTypes.items[0].uid}/entries`) + .reply(200, { entry: mockData.entry }); + }) + .command(['cm:export-to-csv']) + .it('Entries CSV file should be created with prompt'); + }); }); describe('export-to-csv with action users', () => { describe('Export users csv file with flags', () => { test - .stdout({ print: process.env.PRINT === 'true' || false }) - .stub(cliux, 'print', (msg) => console.log(msg)) - .stub(mkdirp, 'sync', () => directory) + .stdout({ print: true }) .stub(fs, 'existsSync', () => new PassThrough()) .stub(fs, 'createWriteStream', () => new PassThrough()) .nock(cma, (api) => { @@ -466,7 +250,7 @@ describe('export-to-csv with action users', () => { .nock(cma, (api) => { api .get(`/v3/organizations/${mockData.organizations[0].uid}/share?skip=0&page=1&limit=100`) - .reply(200, { users: mockData.users.items }); + .reply(200, { users: mockData.users }); }) .command(['cm:export-to-csv', '--action', 'users', '--org', mockData.organizations[0].uid]) // .do(({ stdout }) => @@ -474,12 +258,10 @@ describe('export-to-csv with action users', () => { // ) .it('Users csv file should be successfully created'); }); - + describe('Export users csv file with prompt', () => { test - .stdout({ print: process.env.PRINT === 'true' || false }) - .stub(cliux, 'print', (msg) => console.log(msg)) - .stub(mkdirp, 'sync', () => directory) + .stdout({ print: true }) .stub(fs, 'existsSync', () => new PassThrough()) .stub(fs, 'createWriteStream', () => new PassThrough()) .stub(inquirer, 'registerPrompt', () => {}) @@ -510,9 +292,3 @@ describe('export-to-csv with action users', () => { .it('Users csv file should be successfully created'); }); }); - -.nock(cma, (api) => { - api - .get(`/v3/stacks?&query={"org_uid":"${mockData.organizations[0].uid}"}`) - .reply(200, { stacks: mockData.stacks }); -}) From 65432ac45c9c50b70a199a973803a5f1bb6c887d Mon Sep 17 00:00:00 2001 From: Aman Kumar Date: Mon, 25 Sep 2023 16:17:25 +0530 Subject: [PATCH 13/45] fix: mkdirp issue --- .../test/unit/commands/export-to-csv.test.js | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/packages/contentstack-export-to-csv/test/unit/commands/export-to-csv.test.js b/packages/contentstack-export-to-csv/test/unit/commands/export-to-csv.test.js index 06e6ad61eb..ad4891a503 100644 --- a/packages/contentstack-export-to-csv/test/unit/commands/export-to-csv.test.js +++ b/packages/contentstack-export-to-csv/test/unit/commands/export-to-csv.test.js @@ -10,8 +10,8 @@ const mockData = require('../../mock-data/common.mock.json'); const { cma } = configHandler.get('region'); // const directory = './data'; -// const taxonomyFileName = join(process.cwd(), 'data', mockData.stacks[0].name); -// const userFileName = join(process.cwd(), 'data', mockData.organizations[0].name); +const taxonomyFileName = join(process.cwd(), 'data', mockData.stacks[0].name); +//const userFileName = join(process.cwd(), 'data', mockData.organizations[0].name); describe('export-to-csv with action taxonomies', () => { describe('Create taxonomies & terms csv file with all flags including taxonomy uid', () => { @@ -94,7 +94,6 @@ describe('export-to-csv with action taxonomies', () => { describe('Create taxonomies & terms csv file with prompt', () => { test .stdout({ print: process.env.PRINT === "true" || false }) - .stub(mkdirp, 'sync', () => directory) .stub(fs, 'existsSync', () => new PassThrough()) .stub(fs, 'createWriteStream', () => new PassThrough()) .stub(inquirer, 'registerPrompt', () => {}) From 9f02cb5a8fc7addfc4dd8ac0e15edf76728727b2 Mon Sep 17 00:00:00 2001 From: Aman Kumar Date: Mon, 25 Sep 2023 16:44:27 +0530 Subject: [PATCH 14/45] feat: import taxonomies --- .../contentstack-import/src/config/index.ts | 9 + .../src/import/modules/base-class.ts | 18 +- .../src/import/modules/taxonomies.ts | 247 ++++++++++++++++++ .../src/types/default-config.ts | 10 + .../contentstack-import/src/types/index.ts | 17 +- 5 files changed, 299 insertions(+), 2 deletions(-) create mode 100644 packages/contentstack-import/src/import/modules/taxonomies.ts diff --git a/packages/contentstack-import/src/config/index.ts b/packages/contentstack-import/src/config/index.ts index c9003f3622..3a53ae0baf 100644 --- a/packages/contentstack-import/src/config/index.ts +++ b/packages/contentstack-import/src/config/index.ts @@ -28,6 +28,7 @@ const config: DefaultConfig = { 'assets', 'extensions', 'marketplace-apps', + 'taxonomies', 'global-fields', 'content-types', 'custom-roles', @@ -142,6 +143,14 @@ const config: DefaultConfig = { dirName: 'marketplace_apps', fileName: 'marketplace_apps.json', }, + taxonomies: { + dirName: 'taxonomies', + fileName: 'taxonomies.json' + }, + terms: { + dirName: 'terms', + fileName: 'terms.json' + }, }, languagesCode: [ 'af-za', diff --git a/packages/contentstack-import/src/import/modules/base-class.ts b/packages/contentstack-import/src/import/modules/base-class.ts index fa99462ead..e47f8790ab 100644 --- a/packages/contentstack-import/src/import/modules/base-class.ts +++ b/packages/contentstack-import/src/import/modules/base-class.ts @@ -18,6 +18,7 @@ import { LabelData } from '@contentstack/management/types/stack/label'; import { WebhookData } from '@contentstack/management/types/stack/webhook'; import { WorkflowData } from '@contentstack/management/types/stack/workflow'; import { RoleData } from '@contentstack/management/types/stack/role'; +import { HttpClient } from '@contentstack/cli-utilities'; import { log } from '../../utils'; import { ImportConfig, ModuleClassParams } from '../../types'; @@ -47,7 +48,9 @@ export type ApiModuleType = | 'create-entries' | 'update-entries' | 'publish-entries' - | 'delete-entries'; + | 'delete-entries' + | 'create-taxonomies' + | 'create-terms'; export type ApiOptions = { uid?: string; @@ -381,6 +384,19 @@ export default abstract class BaseClass { .delete({ locale: additionalInfo.locale }) .then(onSuccess) .catch(onReject); + case 'create-taxonomies': + return new HttpClient() + .headers(additionalInfo.headers) + .post(additionalInfo.url, { taxonomy: apiData }) + .then(onSuccess) + .catch(onReject); + case 'create-terms': + const url = `${additionalInfo.baseUrl}/${apiData.taxonomy_uid}/terms`; + return new HttpClient() + .headers(additionalInfo.headers) + .post(url, { term: apiData }) + .then(onSuccess) + .catch(onReject); default: return Promise.resolve(); } diff --git a/packages/contentstack-import/src/import/modules/taxonomies.ts b/packages/contentstack-import/src/import/modules/taxonomies.ts new file mode 100644 index 0000000000..d0b5b9e820 --- /dev/null +++ b/packages/contentstack-import/src/import/modules/taxonomies.ts @@ -0,0 +1,247 @@ +import isEmpty from 'lodash/isEmpty'; +import values from 'lodash/values'; +import keys from 'lodash/keys'; +import flatten from 'lodash/flatten'; +import { join, resolve as pResolve } from 'node:path'; +import { cliux, configHandler, HttpClient } from '@contentstack/cli-utilities'; + +import config from '../../config'; +import { log, formatError, fsUtil, fileHelper } from '../../utils'; +import BaseClass, { ApiOptions } from './base-class'; +import { ModuleClassParams, TaxonomiesConfig, TermsConfig } from '../../types'; + +//NOTE: Temp types need to remove once sdk available +type TaxonomyPayload = { + baseUrl: string; + url: string; + mgToken: string; + reqPayload: Record; + headers: Record; +}; + +export default class ImportTaxonomies extends BaseClass { + private taxonomiesMapperDirPath: string; + private termsMapperDirPath: string; + private taxonomiesFolderPath: string; + private termsFolderPath: string; + private taxSuccessPath: string; + private taxFailsPath: string; + private taxonomiesConfig: TaxonomiesConfig; + private termsConfig: TermsConfig; + private taxonomies: Record; + private taxonomiesSuccess: Record = {}; + private taxonomiesFailed: Record = {}; + private taxonomyPayload: TaxonomyPayload; + + constructor({ importConfig, stackAPIClient }: ModuleClassParams) { + super({ importConfig, stackAPIClient }); + this.taxonomiesConfig = importConfig.modules.taxonomies; + this.termsConfig = importConfig.modules.terms; + this.taxonomiesMapperDirPath = join(importConfig.backupDir, 'mapper', 'taxonomies'); + this.termsMapperDirPath = join(this.taxonomiesMapperDirPath, 'terms'); + this.taxonomiesFolderPath = join(importConfig.backupDir, this.taxonomiesConfig.dirName); + this.termsFolderPath = join(this.taxonomiesFolderPath, this.termsConfig.dirName); + this.taxSuccessPath = join(this.taxonomiesMapperDirPath, 'success.json'); + this.taxFailsPath = join(this.taxonomiesMapperDirPath, 'fails.json'); + this.taxonomyPayload = { + baseUrl: '', + url: '', + mgToken: importConfig.management_token, + reqPayload: {}, + headers: { + 'Content-Type': 'application/json', + api_key: importConfig.target_stack, + }, + }; + } + + /** + * @method start + * @returns {Promise} Promise + */ + async start(): Promise { + log(this.importConfig, 'Migrating taxonomies', 'info'); + + //Step1 check folder exists or not + if (fileHelper.fileExistsSync(this.taxonomiesFolderPath)) { + this.taxonomies = fsUtil.readFile(join(this.taxonomiesFolderPath, 'taxonomies.json'), true) as Record< + string, + unknown + >; + } else { + log(this.importConfig, `No such file or directory - '${this.taxonomiesFolderPath}'`, 'error'); + return; + } + + //NOTE - Temp code for api request + const { cma } = configHandler.get('region') || {}; + this.taxonomyPayload.baseUrl = `${cma}/v3/taxonomies`; + this.taxonomyPayload.url = this.taxonomyPayload.baseUrl; + if (this.taxonomyPayload?.mgToken) this.taxonomyPayload.headers['authorization'] = this.taxonomyPayload.mgToken; + else this.taxonomyPayload.headers['authtoken'] = configHandler.get('authtoken'); + + //Step 2 create taxonomies & terms mapper directory + await fsUtil.makeDirectory(this.taxonomiesMapperDirPath); + await fsUtil.makeDirectory(this.termsMapperDirPath); + + await this.importTaxonomies(); + // create terms related to respective taxonomy + if (!fileHelper.fileExistsSync(this.termsFolderPath)) { + log(this.importConfig, `No such file or directory - '${this.taxonomiesFolderPath}'`, 'error'); + return; + } + await this.importTerms(); + + if (this.taxonomiesSuccess !== undefined && !isEmpty(this.taxonomiesSuccess)) { + fsUtil.writeFile(this.taxSuccessPath, this.taxonomiesSuccess); + } + + if (this.taxonomiesFailed !== undefined &&!isEmpty(this.taxonomiesFailed)) { + fsUtil.writeFile(this.taxFailsPath, this.taxonomiesFailed); + } + + log(this.importConfig, 'Taxonomies have been imported successfully!', 'success'); + } + + async importTaxonomies(): Promise { + if (this.taxonomies === undefined || isEmpty(this.taxonomies)) { + log(this.importConfig, 'No Taxonomies Found', 'info'); + return; + } + + const apiContent = values(this.taxonomies); + + const onSuccess = ({ + response: { data, status } = { data: null, status: null }, + apiData: { uid, name } = { uid: null, name: '' }, + }: any) => { + //NOTE - Temp code to handle error thru API. Will remove this once sdk is ready + if ([200, 201, 202].includes(status)) { + this.taxonomiesSuccess[uid] = data; + log(this.importConfig, `Taxonomy '${name}' imported successfully`, 'success'); + } else { + let errorMsg; + if ([500, 503, 502].includes(status)) errorMsg = data?.message || data; + else errorMsg = data?.error_message; + this.taxonomiesFailed[uid] = `Taxonomy '${name}' failed to be import. ${JSON.stringify(errorMsg)}`; + log(this.importConfig, `Taxonomy '${name}' failed to be import. ${JSON.stringify(errorMsg)}`, 'error'); + } + }; + + const onReject = ({ error, apiData }: any) => { + const err = error?.message ? JSON.parse(error.message) : error; + const { name } = apiData; + if (err?.errors?.name) { + log(this.importConfig, `Taxonomy '${name}' already exists`, 'info'); + } else { + this.taxonomiesFailed[apiData.uid] = apiData; + log(this.importConfig, `Taxonomy '${name}' failed to be import ${formatError(error)}`, 'error'); + log(this.importConfig, error, 'error'); + } + }; + + await this.makeConcurrentCall( + { + apiContent, + processName: 'import taxonomies', + apiParams: { + serializeData: this.serializeTaxonomy.bind(this), + reject: onReject, + resolve: onSuccess, + entity: 'create-taxonomies', + includeParamOnCompletion: true, + additionalInfo: this.taxonomyPayload, + }, + concurrencyLimit: config.concurrency || config.fetchConcurrency || 1, + }, + undefined, + false, + ); + } + + /** + * @method serializeTaxonomy + * @param {ApiOptions} apiOptions ApiOptions + * @returns {ApiOptions} ApiOptions + */ + serializeTaxonomy(apiOptions: ApiOptions): ApiOptions { + const { apiData: taxonomy } = apiOptions; + if (this.taxonomiesSuccess.hasOwnProperty(taxonomy.uid)) { + log(this.importConfig, `Taxonomy '${taxonomy.title}' already exists. Skipping it to avoid duplicates!`, 'info'); + apiOptions.entity = undefined; + } else { + apiOptions.apiData = taxonomy; + } + apiOptions.apiData = taxonomy; + return apiOptions; + } + + async importTerms(): Promise { + if (this.taxonomies === undefined || isEmpty(this.taxonomies)) { + return; + } + + const listOfTaxonomyUIDs = keys(this.taxonomies); + + const onSuccess = ({ + response, + apiData: { uid, name, taxonomy_uid } = { uid: null, name: '', taxonomy_uid: '' }, + }: any) => { + this.taxonomiesSuccess[uid] = response; + log(this.importConfig, `Term '${name}' imported successfully`, 'success'); + }; + + const onReject = ({ error, apiData }: any) => { + const err = error?.message ? JSON.parse(error.message) : error; + const { name } = apiData; + if (err?.errors?.name) { + log(this.importConfig, `Term '${name}' already exists`, 'info'); + } else { + this.taxonomiesSuccess[apiData.uid] = apiData; + log(this.importConfig, `Term '${name}' failed to be import ${formatError(error)}`, 'error'); + log(this.importConfig, error, 'error'); + } + }; + + for (const taxUID of listOfTaxonomyUIDs) { + const terms = fsUtil.readFile( + join(this.termsFolderPath, `${taxUID}-${this.termsConfig.fileName}`), + true, + ) as Record; + const dirPath = pResolve(this.termsMapperDirPath, `${taxUID}-terms`); + if (!fileHelper.fileExistsSync(dirPath)) { + await fsUtil.makeDirectory(dirPath); + } + //success and fail path for particular taxonomy uid + const apiContent = values(terms); + await this.makeConcurrentCall( + { + apiContent, + processName: 'import terms', + apiParams: { + serializeData: this.serializeTerms.bind(this), + reject: onReject, + resolve: onSuccess, + entity: 'create-terms', + includeParamOnCompletion: true, + additionalInfo: this.taxonomyPayload, + }, + concurrencyLimit: config.concurrency || config.fetchConcurrency || 1, + }, + undefined, + false, + ); + } + } + + /** + * @method serializeTerms + * @param {ApiOptions} apiOptions ApiOptions + * @returns {ApiOptions} ApiOptions + */ + serializeTerms(apiOptions: ApiOptions): ApiOptions { + const { apiData: term } = apiOptions; + apiOptions.apiData = term; + return apiOptions; + } +} diff --git a/packages/contentstack-import/src/types/default-config.ts b/packages/contentstack-import/src/types/default-config.ts index c41b949fdb..27b0af0859 100644 --- a/packages/contentstack-import/src/types/default-config.ts +++ b/packages/contentstack-import/src/types/default-config.ts @@ -113,6 +113,16 @@ export default interface DefaultConfig { fileName: string; requiredKeys: string[]; }; + taxonomies: { + dirName: string; + fileName: string; + dependencies?: Modules[]; + }; + terms: { + dirName: string; + fileName: string; + dependencies?: Modules[]; + }; }; languagesCode: string[]; apis: { diff --git a/packages/contentstack-import/src/types/index.ts b/packages/contentstack-import/src/types/index.ts index 104328215f..53a6b1f61e 100644 --- a/packages/contentstack-import/src/types/index.ts +++ b/packages/contentstack-import/src/types/index.ts @@ -40,7 +40,8 @@ export type Modules = | 'custom-roles' | 'workflows' | 'labels' - | 'marketplace-apps'; + | 'marketplace-apps' + | 'taxonomies'; export type ModuleClassParams = { stackAPIClient: ReturnType; @@ -88,5 +89,19 @@ export interface CustomRoleConfig{ customRolesLocalesFileName: string; } +export interface TaxonomiesConfig{ + dirName: string; + fileName: string; + dependencies?: Modules[]; + limit?: number; +} + +export interface TermsConfig{ + dirName: string; + fileName: string; + dependencies?: Modules[]; + limit?: number; +} + export { default as DefaultConfig } from './default-config'; export { default as ImportConfig } from './import-config'; From 9640571b2b851e5033bb1205a4643b5ced790a5b Mon Sep 17 00:00:00 2001 From: Aman Kumar Date: Tue, 26 Sep 2023 11:19:19 +0530 Subject: [PATCH 15/45] fix: test cases --- .../src/commands/cm/export-to-csv.js | 2 +- .../src/util/index.js | 5 +- .../test/mock-data/common.mock.json | 268 +++++++++--------- .../test/unit/commands/export-to-csv.test.js | 87 +++--- .../test/util/common-utils.test.js | 52 ++++ 5 files changed, 225 insertions(+), 189 deletions(-) create mode 100644 packages/contentstack-export-to-csv/test/util/common-utils.test.js diff --git a/packages/contentstack-export-to-csv/src/commands/cm/export-to-csv.js b/packages/contentstack-export-to-csv/src/commands/cm/export-to-csv.js index 26cbe7fe2e..1898bbeacc 100644 --- a/packages/contentstack-export-to-csv/src/commands/cm/export-to-csv.js +++ b/packages/contentstack-export-to-csv/src/commands/cm/export-to-csv.js @@ -277,7 +277,7 @@ class ExportToCsvCommand extends Command { throw new Error(branchExists.errorMessage); } stack.branch_uid = branchUid; - stackAPIClient = getStackClient(managementAPIClient, stack); + stackAPIClient = this.getStackClient(managementAPIClient, stack); } catch (error) { if (error?.message || error?.errorMessage) { cliux.error(util.formatError(error)); diff --git a/packages/contentstack-export-to-csv/src/util/index.js b/packages/contentstack-export-to-csv/src/util/index.js index a760ce6722..bf17eb3595 100644 --- a/packages/contentstack-export-to-csv/src/util/index.js +++ b/packages/contentstack-export-to-csv/src/util/index.js @@ -156,9 +156,7 @@ function chooseStack(managementAPIClient, orgUid, stackApiKey) { async function chooseBranch(branchList) { try { - const branches = await branchList; - - const branchesArray = branches.map((branch) => branch.uid); + const branchesArray = branchList.map((branch) => branch.uid); let _chooseBranch = [ { @@ -846,4 +844,5 @@ module.exports = { formatTaxonomiesData, formatTermsOfTaxonomyData, getTaxonomy, + getStacks }; diff --git a/packages/contentstack-export-to-csv/test/mock-data/common.mock.json b/packages/contentstack-export-to-csv/test/mock-data/common.mock.json index f16230eb1f..a0ce4c2b26 100644 --- a/packages/contentstack-export-to-csv/test/mock-data/common.mock.json +++ b/packages/contentstack-export-to-csv/test/mock-data/common.mock.json @@ -69,43 +69,41 @@ "api_key": "stack_api_key_2" } ], - "users": { - "items": [ - { - "uid": "uid1", - "email": "test@gmail.abc", - "user_uid": "user1", - "org_uid": "test-uid-1", - "invited_by": "user2", - "invited_at": "2023-08-21T11:08:41.038Z", - "status": "accepted", - "acceptance_token": "dfghdfgd", - "created_at": "2023-08-21T11:08:41.036Z", - "updated_at": "2023-08-21T11:09:11.342Z", - "urlPath": "/user", - "organizations": [ - { - "uid": "test-uid-1", - "name": "test org 1", - "org_roles": [ - { - "uid": "role1", - "name": "Admin", - "description": "Admin Role", - "org_uid": "test-uid-1", - "admin": true, - "default": true - } - ] - } - ] - }, - { - "uid": "test-uid-2", - "name": "test org 2" - } - ] - }, + "users": [ + { + "uid": "uid1", + "email": "test@gmail.abc", + "user_uid": "user1", + "org_uid": "test-uid-1", + "invited_by": "user2", + "invited_at": "2023-08-21T11:08:41.038Z", + "status": "accepted", + "acceptance_token": "dfghdfgd", + "created_at": "2023-08-21T11:08:41.036Z", + "updated_at": "2023-08-21T11:09:11.342Z", + "urlPath": "/user", + "organizations": [ + { + "uid": "test-uid-1", + "name": "test org 1", + "org_roles": [ + { + "uid": "role1", + "name": "Admin", + "description": "Admin Role", + "org_uid": "test-uid-1", + "admin": true, + "default": true + } + ] + } + ] + }, + { + "uid": "test-uid-2", + "name": "test org 2" + } + ], "roles": [ { "urlPath": "/roles/role1", @@ -121,64 +119,62 @@ "updated_at": "2023-08-21T11:08:41.042Z" } ], - "contentTypes": { - "items": [ - { - "stackHeaders": { - "api_key": "stack_api_key_1" - }, - "urlPath": "/content_types/ct1", - "created_at": "2023-08-08T13:52:31.980Z", - "updated_at": "2023-08-08T13:52:34.265Z", - "title": "CT 1", - "uid": "ct_1", - "_version": 2, - "inbuilt_class": false, - "schema": [ - { - "data_type": "text", - "display_name": "Title", - "field_metadata": { - "_default": true, - "version": 3 - }, - "mandatory": true, - "uid": "title", - "unique": true, - "multiple": false, - "non_localizable": false - } - ] + "contentTypes": [ + { + "stackHeaders": { + "api_key": "stack_api_key_1" }, - { - "stackHeaders": { - "api_key": "stack_api_key_1" - }, - "urlPath": "/content_types/ct2", - "created_at": "2023-08-08T13:52:31.980Z", - "updated_at": "2023-08-08T13:52:34.265Z", - "title": "CT 2", - "uid": "ct_2", - "_version": 2, - "inbuilt_class": false, - "schema": [ - { - "data_type": "text", - "display_name": "Title", - "field_metadata": { - "_default": true, - "version": 3 - }, - "mandatory": true, - "uid": "title", - "unique": true, - "multiple": false, - "non_localizable": false - } - ] - } - ] - }, + "urlPath": "/content_types/ct1", + "created_at": "2023-08-08T13:52:31.980Z", + "updated_at": "2023-08-08T13:52:34.265Z", + "title": "CT 1", + "uid": "ct_1", + "_version": 2, + "inbuilt_class": false, + "schema": [ + { + "data_type": "text", + "display_name": "Title", + "field_metadata": { + "_default": true, + "version": 3 + }, + "mandatory": true, + "uid": "title", + "unique": true, + "multiple": false, + "non_localizable": false + } + ] + }, + { + "stackHeaders": { + "api_key": "stack_api_key_1" + }, + "urlPath": "/content_types/ct2", + "created_at": "2023-08-08T13:52:31.980Z", + "updated_at": "2023-08-08T13:52:34.265Z", + "title": "CT 2", + "uid": "ct_2", + "_version": 2, + "inbuilt_class": false, + "schema": [ + { + "data_type": "text", + "display_name": "Title", + "field_metadata": { + "_default": true, + "version": 3 + }, + "mandatory": true, + "uid": "title", + "unique": true, + "multiple": false, + "non_localizable": false + } + ] + } + ], "branch": { "stackHeaders": { "api_key": "stack_api_key_1" @@ -192,48 +188,44 @@ "updated_at": "2023-08-08T13:51:43.217Z", "deleted_at": false }, - "entry": { - "items": [ - { - "stackHeaders": { - "api_key": "stack_api_key_1" - }, - "content_type_uid": "home", - "urlPath": "/content_types/ct1/entries/test_entry1", - "title": "Test Entry1", - "url": "/", - "tags": [], - "locale": "en1", - "uid": "test_entry1", - "created_by": "user1", - "updated_by": "user1", - "created_at": "2023-08-08T13:52:46.592Z", - "updated_at": "2023-08-08T13:52:46.592Z", - "_version": 1 - } - ] - }, - "environments": { - "items": [ - { - "stackHeaders": { - "api_key": "stack_api_key_1" - }, - "urlPath": "/environments/development", - "urls": [ - { - "url": "http://localhost:3000/", - "locale": "en1" - } - ], - "name": "development", - "_version": 3, - "uid": "env1", - "created_by": "user1", - "updated_by": "user1", - "created_at": "2023-06-12T18:59:56.853Z", - "updated_at": "2023-06-12T18:59:56.853Z" - } - ] - } + "entry": [ + { + "stackHeaders": { + "api_key": "stack_api_key_1" + }, + "content_type_uid": "home", + "urlPath": "/content_types/ct1/entries/test_entry1", + "title": "Test Entry1", + "url": "/", + "tags": [], + "locale": "en1", + "uid": "test_entry1", + "created_by": "user1", + "updated_by": "user1", + "created_at": "2023-08-08T13:52:46.592Z", + "updated_at": "2023-08-08T13:52:46.592Z", + "_version": 1 + } + ], + "environments": [ + { + "stackHeaders": { + "api_key": "stack_api_key_1" + }, + "urlPath": "/environments/development", + "urls": [ + { + "url": "http://localhost:3000/", + "locale": "en1" + } + ], + "name": "development", + "_version": 3, + "uid": "env1", + "created_by": "user1", + "updated_by": "user1", + "created_at": "2023-06-12T18:59:56.853Z", + "updated_at": "2023-06-12T18:59:56.853Z" + } + ] } diff --git a/packages/contentstack-export-to-csv/test/unit/commands/export-to-csv.test.js b/packages/contentstack-export-to-csv/test/unit/commands/export-to-csv.test.js index ad4891a503..bb43bad3c7 100644 --- a/packages/contentstack-export-to-csv/test/unit/commands/export-to-csv.test.js +++ b/packages/contentstack-export-to-csv/test/unit/commands/export-to-csv.test.js @@ -9,15 +9,15 @@ const { cliux, configHandler } = require('@contentstack/cli-utilities'); const mockData = require('../../mock-data/common.mock.json'); const { cma } = configHandler.get('region'); -// const directory = './data'; +const directory = join(process.cwd(), 'data'); const taxonomyFileName = join(process.cwd(), 'data', mockData.stacks[0].name); -//const userFileName = join(process.cwd(), 'data', mockData.organizations[0].name); describe('export-to-csv with action taxonomies', () => { + if(!fs.existsSync(directory)) fs.mkdirSync(directory); + describe('Create taxonomies & terms csv file with all flags including taxonomy uid', () => { test - .stdout({ print: true }) - .stub(fs, 'existsSync', () => new PassThrough()) + .stdout({ print: process.env.PRINT === "true" || false }) .stub(fs, 'createWriteStream', () => new PassThrough()) .nock(cma, (api) => { api @@ -51,10 +51,10 @@ describe('export-to-csv with action taxonomies', () => { }) .it('CSV file should be created'); }); + describe('Create taxonomies & terms csv file with all flags excluding taxonomy uid', () => { test - .stdout({ print: process.env.PRINT === "true" || false }) - .stub(fs, 'existsSync', () => new PassThrough()) + .stdout({ print: process.env.PRINT === "true" || false }) .stub(fs, 'createWriteStream', () => new PassThrough()) .nock(cma, (api) => { api @@ -87,14 +87,13 @@ describe('export-to-csv with action taxonomies', () => { ]) .do(({ stdout }) => { expect(stdout).to.contain(`Writing taxonomies to file: ${taxonomyFileName}_taxonomies.csv`); - //expect(stdout).to.contain(`Writing taxonomies to file: ${taxonomyFileName}_taxonomies.csv`) }) .it('file should be created'); }); + describe('Create taxonomies & terms csv file with prompt', () => { test .stdout({ print: process.env.PRINT === "true" || false }) - .stub(fs, 'existsSync', () => new PassThrough()) .stub(fs, 'createWriteStream', () => new PassThrough()) .stub(inquirer, 'registerPrompt', () => {}) .stub(inquirer, 'prompt', () => { @@ -128,35 +127,38 @@ describe('export-to-csv with action taxonomies', () => { }); describe('export-to-csv with action entries', () => { + if(!fs.existsSync(directory)) fs.mkdirSync(directory); + describe('Create entries csv file with flags', () => { test .stdout({ print: true }) - .stub(fs, 'existsSync', () => new PassThrough()) .stub(fs, 'createWriteStream', () => new PassThrough()) - //nock api for content type - //nock api for list branches - //nock api for branch .nock(cma, (api) => { api .get(`/v3/stacks?&query={"org_uid":"${mockData.organizations[0].uid}"}`) .reply(200, { stacks: mockData.stacks }); }) .nock(cma, (api) => { - api.get('/v3/environments').reply(200, mockData.environments); + api.get('/v3/environments').reply(200, {environments: mockData.environments}); }) + // .nock(cma, (api) => { + // api.get('/v3/stack/branches').reply(200, { branch: mockData.branch }); + // }) .nock(cma, (api) => { - api.get('/v3/stack/branches').reply(200, { branch: mockData.branch }); + api.get('/v3/content_types?count=true').reply(200, { content_types: 2 }); }) .nock(cma, (api) => { - api.get('/v3/content_types?count=true').reply(200, { count: 2 }); + api.get('/v3/content_types').reply(200, {res: mockData.contentTypes} ); }) .nock(cma, (api) => { - api.get('/v3/content_types').reply(200, { resp: mockData.contentTypes }); + api + .get(`/v3/content_types/${mockData.contentTypes[0].uid}/entries?include_publish_details=true&locale=en1&count=true`) + .reply(200, { entries: 2 }); }) .nock(cma, (api) => { api - .get(`/v3/content_types/${mockData.contentTypes.items[0].uid}/entries`) - .reply(200, { entry: mockData.entry }); + .get(`/v3/content_types/${mockData.contentTypes[0].uid}/entries?include_publish_details=true&locale=en1&skip=0&limit=100&include_workflow=true`) + .reply(200, {entries: mockData.entry}); }) .command([ 'cm:export-to-csv', @@ -171,19 +173,14 @@ describe('export-to-csv with action entries', () => { '--locale', 'en1', '--content-type', - mockData.contentTypes.items[0].uid, + mockData.contentTypes[0].uid, ]) - .do(({ stdout }) => { - //expect(stdout).to.contain(`Writing taxonomies to file: ${taxonomyFileName}_taxonomies.csv`); - //expect(stdout).to.contain(`Writing taxonomies to file: ${taxonomyFileName}_taxonomies.csv`) - }) .it('Entries CSV file should be created'); }); describe('Create entries csv file with prompt', () => { test .stdout({ print: true }) - .stub(fs, 'existsSync', () => new PassThrough()) .stub(fs, 'createWriteStream', () => new PassThrough()) .stub(inquirer, 'registerPrompt', () => {}) .stub(inquirer, 'prompt', () => { @@ -192,8 +189,7 @@ describe('export-to-csv with action entries', () => { chosenOrg: mockData.organizations[0].name, chosenStack: mockData.stacks[0].name, chosenLanguage: 'en1', - chosenContentTypes: mockData.contentTypes.items[0].uid, - branch: mockData.branch.uid, + chosenContentTypes: mockData.contentTypes[0].title }); }) .nock(cma, (api) => { @@ -210,21 +206,26 @@ describe('export-to-csv with action entries', () => { .reply(200, { stacks: mockData.stacks }); }) .nock(cma, (api) => { - api.get('/v3/environments').reply(200, mockData.environments); + api.get('/v3/environments').reply(200, {environments: mockData.environments}); }) + // .nock(cma, (api) => { + // api.get('/v3/stack/branches').reply(200, { branch: mockData.branch }); + // }) .nock(cma, (api) => { - api.get('/v3/stack/branches').reply(200, { branch: mockData.branch }); + api.get('/v3/content_types?count=true').reply(200, { content_types: 2 }); }) .nock(cma, (api) => { - api.get('/v3/content_types?count=true').reply(200, { count: 2 }); + api.get('/v3/content_types?skip=0&include_branch=true').reply(200, {contentTypes: mockData.contentTypes} ); }) .nock(cma, (api) => { - api.get('/v3/content_types').reply(200, { resp: mockData.contentTypes }); + api + .get(`/v3/content_types/${mockData.contentTypes[0].uid}/entries?include_publish_details=true&locale=en1&count=true`) + .reply(200, { entries: 2 }); }) .nock(cma, (api) => { api - .get(`/v3/content_types/${mockData.contentTypes.items[0].uid}/entries`) - .reply(200, { entry: mockData.entry }); + .get(`/v3/content_types/${mockData.contentTypes[0].uid}/entries?include_publish_details=true&locale=en1&skip=0&limit=100&include_workflow=true`) + .reply(200, {entry: mockData.entry}); }) .command(['cm:export-to-csv']) .it('Entries CSV file should be created with prompt'); @@ -234,17 +235,16 @@ describe('export-to-csv with action entries', () => { describe('export-to-csv with action users', () => { describe('Export users csv file with flags', () => { test - .stdout({ print: true }) - .stub(fs, 'existsSync', () => new PassThrough()) + .stdout({ print: process.env.PRINT === "true" || false }) .stub(fs, 'createWriteStream', () => new PassThrough()) .nock(cma, (api) => { - api.get('/v3/user?include_orgs_roles=true').reply(200, { user: mockData.users.items[0] }); + api.get('/v3/user?include_orgs_roles=true').reply(200, { user: mockData.users[0] }); }) .nock(cma, (api) => { api.get(`/v3/organizations/${mockData.organizations[0].uid}/roles`).reply(200, { roles: mockData.roles }); }) .nock(cma, (api) => { - api.get('/v3/user?include_orgs_roles=true').reply(200, { user: mockData.users.items[0] }); + api.get('/v3/user?include_orgs_roles=true').reply(200, { user: mockData.users[0] }); }) .nock(cma, (api) => { api @@ -252,16 +252,12 @@ describe('export-to-csv with action users', () => { .reply(200, { users: mockData.users }); }) .command(['cm:export-to-csv', '--action', 'users', '--org', mockData.organizations[0].uid]) - // .do(({ stdout }) => - // expect(stdout).to.contain(`Writing organization details to file: ${userFileName}_users_export.csv`), - // ) .it('Users csv file should be successfully created'); }); describe('Export users csv file with prompt', () => { test - .stdout({ print: true }) - .stub(fs, 'existsSync', () => new PassThrough()) + .stdout({ print: process.env.PRINT === "true" || false }) .stub(fs, 'createWriteStream', () => new PassThrough()) .stub(inquirer, 'registerPrompt', () => {}) .stub(inquirer, 'prompt', () => { @@ -271,23 +267,20 @@ describe('export-to-csv with action users', () => { api.get(`/v3/organizations?limit=100`).reply(200, { organizations: mockData.organizations }); }) .nock(cma, (api) => { - api.get('/v3/user?include_orgs_roles=true').reply(200, { user: mockData.users.items[0] }); + api.get('/v3/user?include_orgs_roles=true').reply(200, { user: mockData.users[0] }); }) .nock(cma, (api) => { api.get(`/v3/organizations/${mockData.organizations[0].uid}/roles`).reply(200, { roles: mockData.roles }); }) .nock(cma, (api) => { - api.get('/v3/user?include_orgs_roles=true').reply(200, { user: mockData.users.items[0] }); + api.get('/v3/user?include_orgs_roles=true').reply(200, { user: mockData.users[0] }); }) .nock(cma, (api) => { api .get(`/v3/organizations/${mockData.organizations[0].uid}/share?skip=0&page=1&limit=100`) - .reply(200, { users: mockData.users.items }); + .reply(200, { users: mockData.users }); }) .command(['cm:export-to-csv']) - // .do(({ stdout }) => - // expect(stdout).to.contain(`Writing organization details to file: ${userFileName}_users_export.csv`), - // ) .it('Users csv file should be successfully created'); }); }); diff --git a/packages/contentstack-export-to-csv/test/util/common-utils.test.js b/packages/contentstack-export-to-csv/test/util/common-utils.test.js new file mode 100644 index 0000000000..042ff4b84b --- /dev/null +++ b/packages/contentstack-export-to-csv/test/util/common-utils.test.js @@ -0,0 +1,52 @@ +const { fancy } = require('fancy-test'); +const { test, expect } = require('@oclif/test'); +const inquirer = require('inquirer'); +const { cliux, configHandler, ContentstackClient, managementSDKClient } = require('@contentstack/cli-utilities'); + +const mockData = require('../mock-data/common.mock.json'); +const { getStacks, chooseBranch } = require('../../src/util/index'); + +const { cma } = configHandler.get('region'); + +describe('common utils', () => { + let managementSdk; + before(async () => { + managementSdk = await managementSDKClient({ + host: cma.replace('https://', ''), + }); + }); + + describe('chooseStack', () => { + describe('choose stack from list of stacks', () => { + fancy + .nock(cma, (api) => + api + .get(`/v3/stacks?&query={"org_uid":"${mockData.organizations[0].uid}"}`) + .reply(200, { stacks: mockData.stacks }), + ) + .stub(inquirer, 'prompt', () => { + return Promise.resolve({ + chosenStack: mockData.stacks[0].name, + }); + }) + .it('Returns list of stacks', async () => { + await getStacks(managementSdk, mockData.organizations[0].uid); + }); + }); + }); + + describe('chooseBranch', () => { + describe('choose branch from list of branch', () => { + fancy + .stub(inquirer, 'prompt', () => { + return Promise.resolve({ + branch: mockData.branch.uid, + }); + }) + .it('Returns list of stacks', async () => { + const { branch } = await chooseBranch([mockData.branch]); + expect(branch).to.equal(mockData.branch.uid); + }); + }); + }); +}); From 1f1b7471d39e8f92b7a728694d0632696621fd2d Mon Sep 17 00:00:00 2001 From: Aman Kumar Date: Tue, 26 Sep 2023 16:51:06 +0530 Subject: [PATCH 16/45] feat: import terms --- .../src/commands/cm/export-to-csv.js | 2 +- .../src/export/modules/taxonomies.ts | 3 +- .../contentstack-import/src/config/index.ts | 2 +- .../src/import/modules/taxonomies.ts | 167 ++++++++++++------ .../src/utils/asset-helper.ts | 2 +- 5 files changed, 119 insertions(+), 57 deletions(-) diff --git a/packages/contentstack-export-to-csv/src/commands/cm/export-to-csv.js b/packages/contentstack-export-to-csv/src/commands/cm/export-to-csv.js index 26cbe7fe2e..1898bbeacc 100644 --- a/packages/contentstack-export-to-csv/src/commands/cm/export-to-csv.js +++ b/packages/contentstack-export-to-csv/src/commands/cm/export-to-csv.js @@ -277,7 +277,7 @@ class ExportToCsvCommand extends Command { throw new Error(branchExists.errorMessage); } stack.branch_uid = branchUid; - stackAPIClient = getStackClient(managementAPIClient, stack); + stackAPIClient = this.getStackClient(managementAPIClient, stack); } catch (error) { if (error?.message || error?.errorMessage) { cliux.error(util.formatError(error)); diff --git a/packages/contentstack-export/src/export/modules/taxonomies.ts b/packages/contentstack-export/src/export/modules/taxonomies.ts index fa463a880f..6e50298a50 100644 --- a/packages/contentstack-export/src/export/modules/taxonomies.ts +++ b/packages/contentstack-export/src/export/modules/taxonomies.ts @@ -120,7 +120,7 @@ export default class ExportTaxonomies extends BaseClass { log(this.exportConfig, `No terms found for taxonomy - '${taxonomyUID}'`, 'info'); } else { fsUtil.writeFile(pResolve(this.termsFolderPath, `${taxonomyUID}-${this.termsConfig.fileName}`), this.terms); - log(this.exportConfig, `Terms from taxonomy '${taxonomyUID}' were successfully exported.`, 'success'); + log(this.exportConfig, `Terms from taxonomy '${taxonomyUID}' were exported successfully.`, 'success'); } } log(this.exportConfig, `All the terms have been exported successfully!`, 'success'); @@ -172,6 +172,7 @@ export default class ExportTaxonomies extends BaseClass { include_count: true, skip: 0, limit: this.taxonomiesConfig.limit || 100, + depth: 0 // include all the terms if set to 0 }; if (skip >= 0) params['skip'] = skip; diff --git a/packages/contentstack-import/src/config/index.ts b/packages/contentstack-import/src/config/index.ts index 3a53ae0baf..315565ab28 100644 --- a/packages/contentstack-import/src/config/index.ts +++ b/packages/contentstack-import/src/config/index.ts @@ -26,9 +26,9 @@ const config: DefaultConfig = { 'locales', 'environments', 'assets', + 'taxonomies', 'extensions', 'marketplace-apps', - 'taxonomies', 'global-fields', 'content-types', 'custom-roles', diff --git a/packages/contentstack-import/src/import/modules/taxonomies.ts b/packages/contentstack-import/src/import/modules/taxonomies.ts index d0b5b9e820..551ff7822f 100644 --- a/packages/contentstack-import/src/import/modules/taxonomies.ts +++ b/packages/contentstack-import/src/import/modules/taxonomies.ts @@ -2,10 +2,9 @@ import isEmpty from 'lodash/isEmpty'; import values from 'lodash/values'; import keys from 'lodash/keys'; import flatten from 'lodash/flatten'; -import { join, resolve as pResolve } from 'node:path'; -import { cliux, configHandler, HttpClient } from '@contentstack/cli-utilities'; +import { join } from 'node:path'; +import { configHandler } from '@contentstack/cli-utilities'; -import config from '../../config'; import { log, formatError, fsUtil, fileHelper } from '../../utils'; import BaseClass, { ApiOptions } from './base-class'; import { ModuleClassParams, TaxonomiesConfig, TermsConfig } from '../../types'; @@ -21,17 +20,23 @@ type TaxonomyPayload = { export default class ImportTaxonomies extends BaseClass { private taxonomiesMapperDirPath: string; - private termsMapperDirPath: string; private taxonomiesFolderPath: string; - private termsFolderPath: string; private taxSuccessPath: string; private taxFailsPath: string; private taxonomiesConfig: TaxonomiesConfig; - private termsConfig: TermsConfig; private taxonomies: Record; - private taxonomiesSuccess: Record = {}; - private taxonomiesFailed: Record = {}; + private termsFolderPath: string; + private termsMapperDirPath: string; + private termsConfig: TermsConfig; + private termsSuccessPath: string; + private termsFailsPath: string; private taxonomyPayload: TaxonomyPayload; + public taxonomiesSuccess: Record = {}; + public taxonomiesFailed: Record = {}; + public termsSuccess: Record> = {}; + public termsFailed: Record> = {}; + public terms: Record = {}; + public taxonomyUIDs: string[] = []; constructor({ importConfig, stackAPIClient }: ModuleClassParams) { super({ importConfig, stackAPIClient }); @@ -43,6 +48,8 @@ export default class ImportTaxonomies extends BaseClass { this.termsFolderPath = join(this.taxonomiesFolderPath, this.termsConfig.dirName); this.taxSuccessPath = join(this.taxonomiesMapperDirPath, 'success.json'); this.taxFailsPath = join(this.taxonomiesMapperDirPath, 'fails.json'); + this.termsSuccessPath = join(this.termsMapperDirPath, 'success.json'); + this.termsFailsPath = join(this.termsMapperDirPath, 'fails.json'); this.taxonomyPayload = { baseUrl: '', url: '', @@ -84,25 +91,27 @@ export default class ImportTaxonomies extends BaseClass { await fsUtil.makeDirectory(this.taxonomiesMapperDirPath); await fsUtil.makeDirectory(this.termsMapperDirPath); + //Step 3 import taxonomy and create success & failure file await this.importTaxonomies(); - // create terms related to respective taxonomy + this.createTaxonomySuccessAndFailedFile(); + if (!fileHelper.fileExistsSync(this.termsFolderPath)) { - log(this.importConfig, `No such file or directory - '${this.taxonomiesFolderPath}'`, 'error'); + log(this.importConfig, `No such file or directory - '${this.termsFolderPath}'`, 'error'); return; } + //Step 4 import terms and create success & failure file await this.importTerms(); - - if (this.taxonomiesSuccess !== undefined && !isEmpty(this.taxonomiesSuccess)) { - fsUtil.writeFile(this.taxSuccessPath, this.taxonomiesSuccess); - } - - if (this.taxonomiesFailed !== undefined &&!isEmpty(this.taxonomiesFailed)) { - fsUtil.writeFile(this.taxFailsPath, this.taxonomiesFailed); - } + this.createTermSuccessAndFailedFile(); log(this.importConfig, 'Taxonomies have been imported successfully!', 'success'); } + /** + * create taxonomy and enter success & failure related data into taxonomies mapper file + * @method importTaxonomies + * @async + * @returns {Promise} Promise + */ async importTaxonomies(): Promise { if (this.taxonomies === undefined || isEmpty(this.taxonomies)) { log(this.importConfig, 'No Taxonomies Found', 'info'); @@ -110,6 +119,7 @@ export default class ImportTaxonomies extends BaseClass { } const apiContent = values(this.taxonomies); + this.taxonomyUIDs = keys(this.taxonomies); const onSuccess = ({ response: { data, status } = { data: null, status: null }, @@ -117,12 +127,16 @@ export default class ImportTaxonomies extends BaseClass { }: any) => { //NOTE - Temp code to handle error thru API. Will remove this once sdk is ready if ([200, 201, 202].includes(status)) { - this.taxonomiesSuccess[uid] = data; + const { taxonomy } = data; + this.taxonomiesSuccess[taxonomy.uid] = taxonomy; log(this.importConfig, `Taxonomy '${name}' imported successfully`, 'success'); } else { let errorMsg; if ([500, 503, 502].includes(status)) errorMsg = data?.message || data; else errorMsg = data?.error_message; + if (errorMsg === undefined) { + errorMsg = Object.values(data?.errors) && flatten(Object.values(data.errors)); + } this.taxonomiesFailed[uid] = `Taxonomy '${name}' failed to be import. ${JSON.stringify(errorMsg)}`; log(this.importConfig, `Taxonomy '${name}' failed to be import. ${JSON.stringify(errorMsg)}`, 'error'); } @@ -152,7 +166,7 @@ export default class ImportTaxonomies extends BaseClass { includeParamOnCompletion: true, additionalInfo: this.taxonomyPayload, }, - concurrencyLimit: config.concurrency || config.fetchConcurrency || 1, + concurrencyLimit: this.importConfig.concurrency || this.importConfig.fetchConcurrency || 1, }, undefined, false, @@ -176,61 +190,94 @@ export default class ImportTaxonomies extends BaseClass { return apiOptions; } + /** + * create taxonomies success and fail in (mapper/taxonomies) + * @method createTaxonomySuccessAndFailedFile + */ + createTaxonomySuccessAndFailedFile() { + if (this.taxonomiesSuccess !== undefined && !isEmpty(this.taxonomiesSuccess)) { + fsUtil.writeFile(this.taxSuccessPath, this.taxonomiesSuccess); + } + + if (this.taxonomiesFailed !== undefined && !isEmpty(this.taxonomiesFailed)) { + fsUtil.writeFile(this.taxFailsPath, this.taxonomiesFailed); + } + } + + /** + * create terms and enter success & failure related data into terms mapper file + * @method importTerms + * @async + * @returns {Promise} Promise + */ async importTerms(): Promise { - if (this.taxonomies === undefined || isEmpty(this.taxonomies)) { + if (!this.taxonomyUIDs?.length) { return; } - const listOfTaxonomyUIDs = keys(this.taxonomies); - const onSuccess = ({ - response, - apiData: { uid, name, taxonomy_uid } = { uid: null, name: '', taxonomy_uid: '' }, + response: { data, status } = { data: null, status: null }, + apiData: { uid, name, taxonomy_uid } = { uid: null, name: '', taxonomy_uid: null }, }: any) => { - this.taxonomiesSuccess[uid] = response; - log(this.importConfig, `Term '${name}' imported successfully`, 'success'); + //NOTE - Temp code to handle error thru API. Will remove this once sdk is ready + if ([200, 201, 202].includes(status)) { + if (!this.termsSuccess[taxonomy_uid]) this.termsSuccess[taxonomy_uid] = {}; + const { term } = data; + this.termsSuccess[taxonomy_uid][term.uid] = term; + log(this.importConfig, `Term '${name}' imported successfully`, 'success'); + } else { + if (!this.termsFailed[taxonomy_uid]) this.termsFailed[taxonomy_uid] = {}; + let errorMsg; + if ([500, 503, 502].includes(status)) errorMsg = data?.message || data; + else errorMsg = data?.error_message; + if (errorMsg === undefined) { + errorMsg = Object.values(data?.errors) && flatten(Object.values(data.errors)); + } + this.termsFailed[taxonomy_uid][uid] = `Terms '${name}' failed to be import. ${JSON.stringify(errorMsg)}`; + log(this.importConfig, `Terms '${name}' failed to be import. ${JSON.stringify(errorMsg)}`, 'error'); + } }; const onReject = ({ error, apiData }: any) => { + const { uid, taxonomy_uid, name } = apiData; + if (!this.termsFailed[taxonomy_uid]) this.termsFailed[taxonomy_uid] = {}; const err = error?.message ? JSON.parse(error.message) : error; - const { name } = apiData; if (err?.errors?.name) { log(this.importConfig, `Term '${name}' already exists`, 'info'); } else { - this.taxonomiesSuccess[apiData.uid] = apiData; + this.termsFailed[taxonomy_uid][apiData.uid] = apiData; log(this.importConfig, `Term '${name}' failed to be import ${formatError(error)}`, 'error'); log(this.importConfig, error, 'error'); } }; - for (const taxUID of listOfTaxonomyUIDs) { - const terms = fsUtil.readFile( + for (const taxUID of this.taxonomyUIDs) { + //read terms from respective taxonomy + this.terms = fsUtil.readFile( join(this.termsFolderPath, `${taxUID}-${this.termsConfig.fileName}`), true, ) as Record; - const dirPath = pResolve(this.termsMapperDirPath, `${taxUID}-terms`); - if (!fileHelper.fileExistsSync(dirPath)) { - await fsUtil.makeDirectory(dirPath); - } - //success and fail path for particular taxonomy uid - const apiContent = values(terms); - await this.makeConcurrentCall( - { - apiContent, - processName: 'import terms', - apiParams: { - serializeData: this.serializeTerms.bind(this), - reject: onReject, - resolve: onSuccess, - entity: 'create-terms', - includeParamOnCompletion: true, - additionalInfo: this.taxonomyPayload, + + if (this.terms !== undefined && !isEmpty(this.terms)) { + const apiContent = values(this.terms); + await this.makeConcurrentCall( + { + apiContent, + processName: 'import terms', + apiParams: { + serializeData: this.serializeTerms.bind(this), + reject: onReject, + resolve: onSuccess, + entity: 'create-terms', + includeParamOnCompletion: true, + additionalInfo: this.taxonomyPayload, + }, + concurrencyLimit: this.importConfig.concurrency || this.importConfig.fetchConcurrency || 1, }, - concurrencyLimit: config.concurrency || config.fetchConcurrency || 1, - }, - undefined, - false, - ); + undefined, + false, + ); + } } } @@ -244,4 +291,18 @@ export default class ImportTaxonomies extends BaseClass { apiOptions.apiData = term; return apiOptions; } + + /** + * create terms success and fail in (mapper/taxonomies/terms) + * @method createTermSuccessAndFailedFile + */ + createTermSuccessAndFailedFile() { + if (this.termsSuccess !== undefined && !isEmpty(this.termsSuccess)) { + fsUtil.writeFile(this.termsSuccessPath, this.termsSuccess); + } + + if (this.termsFailed !== undefined && !isEmpty(this.termsFailed)) { + fsUtil.writeFile(this.termsFailsPath, this.termsFailed); + } + } } diff --git a/packages/contentstack-import/src/utils/asset-helper.ts b/packages/contentstack-import/src/utils/asset-helper.ts index 7e7cc9e257..ceaead0bed 100644 --- a/packages/contentstack-import/src/utils/asset-helper.ts +++ b/packages/contentstack-import/src/utils/asset-helper.ts @@ -2,7 +2,7 @@ import Bluebird from 'bluebird'; import * as url from 'url'; import * as path from 'path'; import { ContentstackClient, managementSDKClient } from '@contentstack/cli-utilities'; -import { ImportConfig } from 'src/types'; +import { ImportConfig } from '../types'; const debug = require('debug')('util:requests'); let _ = require('lodash'); let { marked } = require('marked'); From 0fc7e724c034f296f951be16c559d2c041fb0e29 Mon Sep 17 00:00:00 2001 From: Aman Kumar Date: Tue, 26 Sep 2023 17:13:04 +0530 Subject: [PATCH 17/45] refactor: taxonomy export & import --- .../contentstack-export/src/export/modules/taxonomies.ts | 2 +- .../contentstack-import/src/import/modules/taxonomies.ts | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/contentstack-export/src/export/modules/taxonomies.ts b/packages/contentstack-export/src/export/modules/taxonomies.ts index 6e50298a50..09aa0db139 100644 --- a/packages/contentstack-export/src/export/modules/taxonomies.ts +++ b/packages/contentstack-export/src/export/modules/taxonomies.ts @@ -1,7 +1,7 @@ import omit from 'lodash/omit'; +import keys from 'lodash/keys'; import isEmpty from 'lodash/isEmpty'; import flatten from 'lodash/flatten'; -import keys from 'lodash/keys'; import { resolve as pResolve } from 'node:path'; import { cliux, configHandler, HttpClient } from '@contentstack/cli-utilities'; diff --git a/packages/contentstack-import/src/import/modules/taxonomies.ts b/packages/contentstack-import/src/import/modules/taxonomies.ts index 551ff7822f..940395cdc5 100644 --- a/packages/contentstack-import/src/import/modules/taxonomies.ts +++ b/packages/contentstack-import/src/import/modules/taxonomies.ts @@ -1,12 +1,12 @@ -import isEmpty from 'lodash/isEmpty'; -import values from 'lodash/values'; import keys from 'lodash/keys'; -import flatten from 'lodash/flatten'; import { join } from 'node:path'; +import values from 'lodash/values'; +import isEmpty from 'lodash/isEmpty'; +import flatten from 'lodash/flatten'; import { configHandler } from '@contentstack/cli-utilities'; -import { log, formatError, fsUtil, fileHelper } from '../../utils'; import BaseClass, { ApiOptions } from './base-class'; +import { log, formatError, fsUtil, fileHelper } from '../../utils'; import { ModuleClassParams, TaxonomiesConfig, TermsConfig } from '../../types'; //NOTE: Temp types need to remove once sdk available From 334b007ed4f3cf8cf3c950ed981206393d89887a Mon Sep 17 00:00:00 2001 From: Aman Kumar Date: Tue, 26 Sep 2023 17:26:26 +0530 Subject: [PATCH 18/45] refactor: removed limit from taxonomy and term types --- packages/contentstack-import/src/import/modules/taxonomies.ts | 2 +- packages/contentstack-import/src/types/index.ts | 2 -- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/contentstack-import/src/import/modules/taxonomies.ts b/packages/contentstack-import/src/import/modules/taxonomies.ts index 940395cdc5..37cade6c80 100644 --- a/packages/contentstack-import/src/import/modules/taxonomies.ts +++ b/packages/contentstack-import/src/import/modules/taxonomies.ts @@ -181,7 +181,7 @@ export default class ImportTaxonomies extends BaseClass { serializeTaxonomy(apiOptions: ApiOptions): ApiOptions { const { apiData: taxonomy } = apiOptions; if (this.taxonomiesSuccess.hasOwnProperty(taxonomy.uid)) { - log(this.importConfig, `Taxonomy '${taxonomy.title}' already exists. Skipping it to avoid duplicates!`, 'info'); + log(this.importConfig, `Taxonomy '${taxonomy.name}' already exists. Skipping it to avoid duplicates!`, 'info'); apiOptions.entity = undefined; } else { apiOptions.apiData = taxonomy; diff --git a/packages/contentstack-import/src/types/index.ts b/packages/contentstack-import/src/types/index.ts index 53a6b1f61e..b9d5d5111a 100644 --- a/packages/contentstack-import/src/types/index.ts +++ b/packages/contentstack-import/src/types/index.ts @@ -93,14 +93,12 @@ export interface TaxonomiesConfig{ dirName: string; fileName: string; dependencies?: Modules[]; - limit?: number; } export interface TermsConfig{ dirName: string; fileName: string; dependencies?: Modules[]; - limit?: number; } export { default as DefaultConfig } from './default-config'; From 3f56f2cd9775c76064f96c1310c7fc27d8fbd660 Mon Sep 17 00:00:00 2001 From: Aman Kumar Date: Wed, 27 Sep 2023 11:25:57 +0530 Subject: [PATCH 19/45] fix: entries test cases --- .../test/unit/commands/export-to-csv.test.js | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/packages/contentstack-export-to-csv/test/unit/commands/export-to-csv.test.js b/packages/contentstack-export-to-csv/test/unit/commands/export-to-csv.test.js index bb43bad3c7..e71ec43944 100644 --- a/packages/contentstack-export-to-csv/test/unit/commands/export-to-csv.test.js +++ b/packages/contentstack-export-to-csv/test/unit/commands/export-to-csv.test.js @@ -188,8 +188,7 @@ describe('export-to-csv with action entries', () => { action: 'entries', chosenOrg: mockData.organizations[0].name, chosenStack: mockData.stacks[0].name, - chosenLanguage: 'en1', - chosenContentTypes: mockData.contentTypes[0].title + chosenContentTypes: mockData.contentTypes[0] }); }) .nock(cma, (api) => { @@ -225,9 +224,9 @@ describe('export-to-csv with action entries', () => { .nock(cma, (api) => { api .get(`/v3/content_types/${mockData.contentTypes[0].uid}/entries?include_publish_details=true&locale=en1&skip=0&limit=100&include_workflow=true`) - .reply(200, {entry: mockData.entry}); + .reply(200, {entries: mockData.entry}); }) - .command(['cm:export-to-csv']) + .command(['cm:export-to-csv', '--locale', 'en1']) .it('Entries CSV file should be created with prompt'); }); }); From c87a19c3ef04c63c6f2323221dbcab7bb6d1801a Mon Sep 17 00:00:00 2001 From: Aman Kumar Date: Wed, 27 Sep 2023 15:14:20 +0530 Subject: [PATCH 20/45] fix: test cases --- .../src/util/index.js | 2 +- .../test/mock-data/common.mock.json | 12 ++++ .../test/unit/commands/export-to-csv.test.js | 48 +++++++------- pnpm-lock.yaml | 63 +++++++++++++++++-- 4 files changed, 96 insertions(+), 29 deletions(-) diff --git a/packages/contentstack-export-to-csv/src/util/index.js b/packages/contentstack-export-to-csv/src/util/index.js index 7cef3d6756..812ae97064 100644 --- a/packages/contentstack-export-to-csv/src/util/index.js +++ b/packages/contentstack-export-to-csv/src/util/index.js @@ -399,7 +399,7 @@ function cleanEntries(entries, language, environments, contentTypeUid) { return filteredEntries.map((entry) => { let workflow = ''; const envArr = []; - if (entry.publish_details.length) { + if (entry?.publish_details?.length) { entry.publish_details.forEach((env) => { envArr.push(JSON.stringify([environments[env['environment']], env['locale'], env['time']])); }); diff --git a/packages/contentstack-export-to-csv/test/mock-data/common.mock.json b/packages/contentstack-export-to-csv/test/mock-data/common.mock.json index a0ce4c2b26..d0fc9be610 100644 --- a/packages/contentstack-export-to-csv/test/mock-data/common.mock.json +++ b/packages/contentstack-export-to-csv/test/mock-data/common.mock.json @@ -227,5 +227,17 @@ "created_at": "2023-06-12T18:59:56.853Z", "updated_at": "2023-06-12T18:59:56.853Z" } + ], + "locales": [ + { + "code": "en1", + "name": "English - En", + "fallback_locale": "en-us", + "uid": "gsfdasgdf", + "created_at": "2023-09-11T10:44:40.213Z", + "updated_at": "2023-09-11T10:44:40.213Z", + "ACL": [], + "_version": 1 + } ] } diff --git a/packages/contentstack-export-to-csv/test/unit/commands/export-to-csv.test.js b/packages/contentstack-export-to-csv/test/unit/commands/export-to-csv.test.js index e71ec43944..96ffca2c46 100644 --- a/packages/contentstack-export-to-csv/test/unit/commands/export-to-csv.test.js +++ b/packages/contentstack-export-to-csv/test/unit/commands/export-to-csv.test.js @@ -47,7 +47,6 @@ describe('export-to-csv with action taxonomies', () => { ]) .do(({ stdout }) => { expect(stdout).to.contain(`Writing taxonomies to file: ${taxonomyFileName}_taxonomies.csv`); - //expect(stdout).to.contain(`Writing taxonomies to file: ${taxonomyFileName}_taxonomies.csv`) }) .it('CSV file should be created'); }); @@ -127,11 +126,11 @@ describe('export-to-csv with action taxonomies', () => { }); describe('export-to-csv with action entries', () => { - if(!fs.existsSync(directory)) fs.mkdirSync(directory); + if (!fs.existsSync(directory)) fs.mkdirSync(directory); describe('Create entries csv file with flags', () => { test - .stdout({ print: true }) + .stdout({ print: process.env.PRINT === "true" || false }) .stub(fs, 'createWriteStream', () => new PassThrough()) .nock(cma, (api) => { api @@ -141,14 +140,11 @@ describe('export-to-csv with action entries', () => { .nock(cma, (api) => { api.get('/v3/environments').reply(200, {environments: mockData.environments}); }) - // .nock(cma, (api) => { - // api.get('/v3/stack/branches').reply(200, { branch: mockData.branch }); - // }) .nock(cma, (api) => { api.get('/v3/content_types?count=true').reply(200, { content_types: 2 }); }) .nock(cma, (api) => { - api.get('/v3/content_types').reply(200, {res: mockData.contentTypes} ); + api.get('/v3/content_types').reply(200, {content_types: mockData.contentTypes} ); }) .nock(cma, (api) => { api @@ -180,53 +176,57 @@ describe('export-to-csv with action entries', () => { describe('Create entries csv file with prompt', () => { test - .stdout({ print: true }) + .stdout({ print: process.env.PRINT === "true" || false }) .stub(fs, 'createWriteStream', () => new PassThrough()) .stub(inquirer, 'registerPrompt', () => {}) .stub(inquirer, 'prompt', () => { return Promise.resolve({ action: 'entries', chosenOrg: mockData.organizations[0].name, + chosenLanguage: mockData.locales[0].name, chosenStack: mockData.stacks[0].name, - chosenContentTypes: mockData.contentTypes[0] + chosenContentTypes: [mockData.contentTypes[0].uid], + branch: mockData.branch.uid, }); }) + .nock(cma, (api) => { + api.get(`/v3/organizations?limit=100`).reply(200, { organizations: mockData.organizations }); + }) .nock(cma, (api) => { api .get(`/v3/stacks?&query={"org_uid":"${mockData.organizations[0].uid}"}`) .reply(200, { stacks: mockData.stacks }); }) .nock(cma, (api) => { - api.get(`/v3/organizations?limit=100`).reply(200, { organizations: mockData.organizations }); + api.get('/v3/environments').reply(200, { environments: mockData.environments }); }) .nock(cma, (api) => { - api - .get(`/v3/stacks?&query={"org_uid":"${mockData.organizations[0].uid}"}`) - .reply(200, { stacks: mockData.stacks }); + api.get('/v3/locales').reply(200, { locales: mockData.locales }); }) .nock(cma, (api) => { - api.get('/v3/environments').reply(200, {environments: mockData.environments}); + api.get('/v3/stacks/branches').reply(200, { branches: mockData.branch }); }) - // .nock(cma, (api) => { - // api.get('/v3/stack/branches').reply(200, { branch: mockData.branch }); - // }) .nock(cma, (api) => { api.get('/v3/content_types?count=true').reply(200, { content_types: 2 }); }) .nock(cma, (api) => { - api.get('/v3/content_types?skip=0&include_branch=true').reply(200, {contentTypes: mockData.contentTypes} ); + api.get('/v3/content_types?skip=0&include_branch=true').reply(200, { content_types: mockData.contentTypes }); }) .nock(cma, (api) => { api - .get(`/v3/content_types/${mockData.contentTypes[0].uid}/entries?include_publish_details=true&locale=en1&count=true`) - .reply(200, { entries: 2 }); + .get( + `/v3/content_types/${mockData.contentTypes[0].uid}/entries?include_publish_details=true&locale=${mockData.locales[0].code}&count=true`, + ) + .reply(200, { entries: 1 }); }) .nock(cma, (api) => { api - .get(`/v3/content_types/${mockData.contentTypes[0].uid}/entries?include_publish_details=true&locale=en1&skip=0&limit=100&include_workflow=true`) - .reply(200, {entries: mockData.entry}); + .get( + `/v3/content_types/${mockData.contentTypes[0].uid}/entries?include_publish_details=true&locale=${mockData.locales[0].code}&skip=0&limit=100&include_workflow=true`, + ) + .reply(200, { entries: mockData.entry }); }) - .command(['cm:export-to-csv', '--locale', 'en1']) + .command(['cm:export-to-csv']) .it('Entries CSV file should be created with prompt'); }); }); @@ -253,7 +253,7 @@ describe('export-to-csv with action users', () => { .command(['cm:export-to-csv', '--action', 'users', '--org', mockData.organizations[0].uid]) .it('Users csv file should be successfully created'); }); - + describe('Export users csv file with prompt', () => { test .stdout({ print: process.env.PRINT === "true" || false }) diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 71c060cd31..9c9bb5be8e 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -617,9 +617,12 @@ importers: '@contentstack/cli-command': ~1.2.12 '@contentstack/cli-utilities': ~1.5.2 '@oclif/test': ^2.2.10 - chai: ^4.2.0 + '@types/chai': ^4.3.6 + '@types/mocha': ^10.0.1 + chai: ^4.3.8 chalk: ^4.1.0 debug: ^4.3.1 + dotenv: ^16.3.1 eslint: ^7.32.0 eslint-config-oclif: ^4.0.0 fast-csv: ^4.3.6 @@ -627,7 +630,7 @@ importers: inquirer: 8.2.4 inquirer-checkbox-plus-prompt: 1.0.1 mkdirp: ^3.0.1 - mocha: ^10.0.0 + mocha: ^10.2.0 nyc: ^15.1.0 oclif: ^3.8.1 dependencies: @@ -640,12 +643,15 @@ importers: mkdirp: 3.0.1 devDependencies: '@oclif/test': 2.3.31 - chai: 4.3.7 + '@types/chai': 4.3.6 + '@types/mocha': 10.0.1 + chai: 4.3.8 debug: 4.3.4 + dotenv: 16.3.1 eslint: 7.32.0 eslint-config-oclif: 4.0.0_eslint@7.32.0 globby: 10.0.2 - mocha: 10.1.0 + mocha: 10.2.0 nyc: 15.1.0 oclif: 3.9.1 @@ -3241,6 +3247,10 @@ packages: /@types/chai/4.3.5: resolution: {integrity: sha512-mEo1sAde+UCE6b2hxn332f1g1E8WfYRu6p5SvTKr2ZKC1f7gFJXk4h5PyGP9Dt6gCaG8y8XhwnXWC6Iy2cmBng==} + /@types/chai/4.3.6: + resolution: {integrity: sha512-VOVRLM1mBxIRxydiViqPcKn6MIxZytrbMpd6RJLIWKxUNr3zux8no0Oc7kJx0WAPIitgZ0gkrDS+btlqQpubpw==} + dev: true + /@types/cli-progress/3.11.0: resolution: {integrity: sha512-XhXhBv1R/q2ahF3BM7qT5HLzJNlIL0wbcGyZVjqOTqAybAnsLisd7gy1UCyIqpL+5Iv6XhlSyzjLCnI2sIdbCg==} dependencies: @@ -3413,6 +3423,10 @@ packages: '@types/node': 14.18.53 dev: true + /@types/mocha/10.0.1: + resolution: {integrity: sha512-/fvYntiO1GeICvqbQ3doGDIP97vWmvFt83GKguJ6prmQM2iXZfFcq6YE8KteFyRtX2/h5Hf91BYvPodJKFYv5Q==} + dev: true + /@types/mocha/8.2.3: resolution: {integrity: sha512-ekGvFhFgrc2zYQoX4JeZPmVzZxw6Dtllga7iGHzfbYIYkAMUx/sAFP2GdFpLff+vdHXu5fl7WX9AT+TtqYcsyw==} dev: true @@ -4737,6 +4751,19 @@ packages: type-detect: 4.0.8 dev: true + /chai/4.3.8: + resolution: {integrity: sha512-vX4YvVVtxlfSZ2VecZgFUTU5qPCYsobVI2O9FmwEXBhDigYGQA6jRXCycIs1yJnnWbZ6/+a2zNIF5DfVCcJBFQ==} + engines: {node: '>=4'} + dependencies: + assertion-error: 1.1.0 + check-error: 1.0.2 + deep-eql: 4.1.3 + get-func-name: 2.0.0 + loupe: 2.3.6 + pathval: 1.1.1 + type-detect: 4.0.8 + dev: true + /chalk/1.1.3: resolution: {integrity: sha512-U3lRVLMSlsCfjqYPbLyVv11M9CPW4I728d6TCKMAOJueEeB9/8o+eSsMnxPJD+Q+K909sdESg7C+tIkoH6on1A==} engines: {node: '>=0.10.0'} @@ -9308,6 +9335,34 @@ packages: yargs-unparser: 2.0.0 dev: true + /mocha/10.2.0: + resolution: {integrity: sha512-IDY7fl/BecMwFHzoqF2sg/SHHANeBoMMXFlS9r0OXKDssYE1M5O43wUY/9BVPeIvfH2zmEbBfseqN9gBQZzXkg==} + engines: {node: '>= 14.0.0'} + hasBin: true + dependencies: + ansi-colors: 4.1.1 + browser-stdout: 1.3.1 + chokidar: 3.5.3 + debug: 4.3.4_supports-color@8.1.1 + diff: 5.0.0 + escape-string-regexp: 4.0.0 + find-up: 5.0.0 + glob: 7.2.0 + he: 1.2.0 + js-yaml: 4.1.0 + log-symbols: 4.1.0 + minimatch: 5.0.1 + ms: 2.1.3 + nanoid: 3.3.3 + serialize-javascript: 6.0.0 + strip-json-comments: 3.1.1 + supports-color: 8.1.1 + workerpool: 6.2.1 + yargs: 16.2.0 + yargs-parser: 20.2.4 + yargs-unparser: 2.0.0 + dev: true + /mock-stdin/1.0.0: resolution: {integrity: sha512-tukRdb9Beu27t6dN+XztSRHq9J0B/CoAOySGzHfn8UTfmqipA5yNT/sDUEyYdAV3Hpka6Wx6kOMxuObdOex60Q==} From a179496bd96150cd0aaa690079fd42c936216879 Mon Sep 17 00:00:00 2001 From: Aman Kumar Date: Wed, 27 Sep 2023 15:17:40 +0530 Subject: [PATCH 21/45] refactor: updated package --- .../contentstack-export-to-csv/package.json | 4 +-- .../test/unit/commands/export-to-csv.test.js | 30 +++++++++++-------- 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/packages/contentstack-export-to-csv/package.json b/packages/contentstack-export-to-csv/package.json index 475db8c2fd..84f3af0dab 100644 --- a/packages/contentstack-export-to-csv/package.json +++ b/packages/contentstack-export-to-csv/package.json @@ -47,8 +47,8 @@ "postpack": "rm -f oclif.manifest.json", "prepack": "oclif manifest && oclif readme", "test": "nyc mocha --forbid-only \"test/**/*.test.js\"", - "test:unit": "mocha --timeout 10000 --forbid-only \"test/unit/**/*.test.js\"", - "test:unit:report": "nyc --extension .js mocha --forbid-only \"test/unit/**/*.test.js\"", + "test:unit": "mocha --timeout 10000 --forbid-only \"test/unit/**/*.test.js\" \"test/util/common-utils.test.js\"", + "test:unit:report": "nyc --extension .js mocha --forbid-only \"test/unit/**/*.test.js\" \"test/util/common-utils.test.js\"", "version": "oclif readme && git add README.md", "clean": "rm -rf ./node_modules tsconfig.build.tsbuildinfo" }, diff --git a/packages/contentstack-export-to-csv/test/unit/commands/export-to-csv.test.js b/packages/contentstack-export-to-csv/test/unit/commands/export-to-csv.test.js index 96ffca2c46..4def3630f3 100644 --- a/packages/contentstack-export-to-csv/test/unit/commands/export-to-csv.test.js +++ b/packages/contentstack-export-to-csv/test/unit/commands/export-to-csv.test.js @@ -13,11 +13,11 @@ const directory = join(process.cwd(), 'data'); const taxonomyFileName = join(process.cwd(), 'data', mockData.stacks[0].name); describe('export-to-csv with action taxonomies', () => { - if(!fs.existsSync(directory)) fs.mkdirSync(directory); + if (!fs.existsSync(directory)) fs.mkdirSync(directory); describe('Create taxonomies & terms csv file with all flags including taxonomy uid', () => { test - .stdout({ print: process.env.PRINT === "true" || false }) + .stdout({ print: process.env.PRINT === 'true' || false }) .stub(fs, 'createWriteStream', () => new PassThrough()) .nock(cma, (api) => { api @@ -53,7 +53,7 @@ describe('export-to-csv with action taxonomies', () => { describe('Create taxonomies & terms csv file with all flags excluding taxonomy uid', () => { test - .stdout({ print: process.env.PRINT === "true" || false }) + .stdout({ print: process.env.PRINT === 'true' || false }) .stub(fs, 'createWriteStream', () => new PassThrough()) .nock(cma, (api) => { api @@ -92,7 +92,7 @@ describe('export-to-csv with action taxonomies', () => { describe('Create taxonomies & terms csv file with prompt', () => { test - .stdout({ print: process.env.PRINT === "true" || false }) + .stdout({ print: process.env.PRINT === 'true' || false }) .stub(fs, 'createWriteStream', () => new PassThrough()) .stub(inquirer, 'registerPrompt', () => {}) .stub(inquirer, 'prompt', () => { @@ -130,7 +130,7 @@ describe('export-to-csv with action entries', () => { describe('Create entries csv file with flags', () => { test - .stdout({ print: process.env.PRINT === "true" || false }) + .stdout({ print: process.env.PRINT === 'true' || false }) .stub(fs, 'createWriteStream', () => new PassThrough()) .nock(cma, (api) => { api @@ -138,23 +138,27 @@ describe('export-to-csv with action entries', () => { .reply(200, { stacks: mockData.stacks }); }) .nock(cma, (api) => { - api.get('/v3/environments').reply(200, {environments: mockData.environments}); + api.get('/v3/environments').reply(200, { environments: mockData.environments }); }) .nock(cma, (api) => { api.get('/v3/content_types?count=true').reply(200, { content_types: 2 }); }) .nock(cma, (api) => { - api.get('/v3/content_types').reply(200, {content_types: mockData.contentTypes} ); + api.get('/v3/content_types').reply(200, { content_types: mockData.contentTypes }); }) .nock(cma, (api) => { api - .get(`/v3/content_types/${mockData.contentTypes[0].uid}/entries?include_publish_details=true&locale=en1&count=true`) + .get( + `/v3/content_types/${mockData.contentTypes[0].uid}/entries?include_publish_details=true&locale=en1&count=true`, + ) .reply(200, { entries: 2 }); }) .nock(cma, (api) => { api - .get(`/v3/content_types/${mockData.contentTypes[0].uid}/entries?include_publish_details=true&locale=en1&skip=0&limit=100&include_workflow=true`) - .reply(200, {entries: mockData.entry}); + .get( + `/v3/content_types/${mockData.contentTypes[0].uid}/entries?include_publish_details=true&locale=en1&skip=0&limit=100&include_workflow=true`, + ) + .reply(200, { entries: mockData.entry }); }) .command([ 'cm:export-to-csv', @@ -176,7 +180,7 @@ describe('export-to-csv with action entries', () => { describe('Create entries csv file with prompt', () => { test - .stdout({ print: process.env.PRINT === "true" || false }) + .stdout({ print: process.env.PRINT === 'true' || false }) .stub(fs, 'createWriteStream', () => new PassThrough()) .stub(inquirer, 'registerPrompt', () => {}) .stub(inquirer, 'prompt', () => { @@ -234,7 +238,7 @@ describe('export-to-csv with action entries', () => { describe('export-to-csv with action users', () => { describe('Export users csv file with flags', () => { test - .stdout({ print: process.env.PRINT === "true" || false }) + .stdout({ print: process.env.PRINT === 'true' || false }) .stub(fs, 'createWriteStream', () => new PassThrough()) .nock(cma, (api) => { api.get('/v3/user?include_orgs_roles=true').reply(200, { user: mockData.users[0] }); @@ -256,7 +260,7 @@ describe('export-to-csv with action users', () => { describe('Export users csv file with prompt', () => { test - .stdout({ print: process.env.PRINT === "true" || false }) + .stdout({ print: process.env.PRINT === 'true' || false }) .stub(fs, 'createWriteStream', () => new PassThrough()) .stub(inquirer, 'registerPrompt', () => {}) .stub(inquirer, 'prompt', () => { From d8646ded8ab05e3c5da817de667e7c47691e8ba2 Mon Sep 17 00:00:00 2001 From: Aman Kumar Date: Wed, 27 Sep 2023 16:02:56 +0530 Subject: [PATCH 22/45] fix: directory creation in test cases --- .../test/unit/commands/export-to-csv.test.js | 40 ++++++++++--------- 1 file changed, 22 insertions(+), 18 deletions(-) diff --git a/packages/contentstack-export-to-csv/test/unit/commands/export-to-csv.test.js b/packages/contentstack-export-to-csv/test/unit/commands/export-to-csv.test.js index 4def3630f3..bd562fc05f 100644 --- a/packages/contentstack-export-to-csv/test/unit/commands/export-to-csv.test.js +++ b/packages/contentstack-export-to-csv/test/unit/commands/export-to-csv.test.js @@ -1,4 +1,5 @@ const fs = require('fs'); +const mkdirp = require('mkdirp'); const { test, expect } = require('@oclif/test'); const { join } = require('path'); const { PassThrough } = require('stream'); @@ -9,16 +10,14 @@ const { cliux, configHandler } = require('@contentstack/cli-utilities'); const mockData = require('../../mock-data/common.mock.json'); const { cma } = configHandler.get('region'); -const directory = join(process.cwd(), 'data'); -const taxonomyFileName = join(process.cwd(), 'data', mockData.stacks[0].name); describe('export-to-csv with action taxonomies', () => { - if (!fs.existsSync(directory)) fs.mkdirSync(directory); - describe('Create taxonomies & terms csv file with all flags including taxonomy uid', () => { test .stdout({ print: process.env.PRINT === 'true' || false }) .stub(fs, 'createWriteStream', () => new PassThrough()) + .stub(mkdirp, 'sync', () => {}) + .stub(process, 'chdir', () => {}) .nock(cma, (api) => { api .get(`/v3/stacks?&query={"org_uid":"${mockData.organizations[0].uid}"}`) @@ -26,12 +25,12 @@ describe('export-to-csv with action taxonomies', () => { }) .nock(cma, (api) => { api - .get('/v3/taxonomies/taxonomy_uid_1?include_count=true&skip=0&limit=30') + .get(`/v3/taxonomies/${mockData.taxonomiesResp.taxonomies[0].uid}?include_count=true&skip=0&limit=30`) .reply(200, { taxonomy: mockData.taxonomiesResp.taxonomies[0] }); }) .nock(cma, (api) => { api - .get('/v3/taxonomies/taxonomy_uid_1/terms?include_count=true&skip=0&limit=100') + .get(`/v3/taxonomies/${mockData.taxonomiesResp.taxonomies[0].uid}/terms?include_count=true&skip=0&limit=100`) .reply(200, { terms: mockData.termsResp.terms, count: mockData.termsResp.count }); }) .command([ @@ -39,15 +38,12 @@ describe('export-to-csv with action taxonomies', () => { '--action', 'taxonomies', '--taxonomy-uid', - 'taxonomy_uid_1', + mockData.taxonomiesResp.taxonomies[0].uid, '--stack-api-key', mockData.stacks[0].api_key, '--org', mockData.organizations[0].uid, ]) - .do(({ stdout }) => { - expect(stdout).to.contain(`Writing taxonomies to file: ${taxonomyFileName}_taxonomies.csv`); - }) .it('CSV file should be created'); }); @@ -55,6 +51,8 @@ describe('export-to-csv with action taxonomies', () => { test .stdout({ print: process.env.PRINT === 'true' || false }) .stub(fs, 'createWriteStream', () => new PassThrough()) + .stub(mkdirp, 'sync', () => {}) + .stub(process, 'chdir', () => {}) .nock(cma, (api) => { api .get(`/v3/stacks?&query={"org_uid":"${mockData.organizations[0].uid}"}`) @@ -67,12 +65,12 @@ describe('export-to-csv with action taxonomies', () => { }) .nock(cma, (api) => { api - .get('/v3/taxonomies/taxonomy_uid_1/terms?include_count=true&skip=0&limit=100') + .get(`/v3/taxonomies/${mockData.taxonomiesResp.taxonomies[0].uid}/terms?include_count=true&skip=0&limit=100`) .reply(200, { terms: mockData.termsResp.terms, count: mockData.termsResp.count }); }) .nock(cma, (api) => { api - .get('/v3/taxonomies/taxonomy_uid_2/terms?include_count=true&skip=0&limit=100') + .get(`/v3/taxonomies/${mockData.taxonomiesResp.taxonomies[1].uid}/terms?include_count=true&skip=0&limit=100`) .reply(200, { terms: [], count: 0 }); }) .command([ @@ -84,9 +82,6 @@ describe('export-to-csv with action taxonomies', () => { '--org', mockData.organizations[0].uid, ]) - .do(({ stdout }) => { - expect(stdout).to.contain(`Writing taxonomies to file: ${taxonomyFileName}_taxonomies.csv`); - }) .it('file should be created'); }); @@ -94,6 +89,8 @@ describe('export-to-csv with action taxonomies', () => { test .stdout({ print: process.env.PRINT === 'true' || false }) .stub(fs, 'createWriteStream', () => new PassThrough()) + .stub(mkdirp, 'sync', () => {}) + .stub(process, 'chdir', () => {}) .stub(inquirer, 'registerPrompt', () => {}) .stub(inquirer, 'prompt', () => { return Promise.resolve({ @@ -112,12 +109,12 @@ describe('export-to-csv with action taxonomies', () => { }) .nock(cma, (api) => { api - .get('/v3/taxonomies/taxonomy_uid_1?include_count=true&skip=0&limit=30') + .get(`/v3/taxonomies/${mockData.taxonomiesResp.taxonomies[0].uid}?include_count=true&skip=0&limit=30`) .reply(200, { taxonomy: mockData.taxonomiesResp.taxonomies[0] }); }) .nock(cma, (api) => { api - .get('/v3/taxonomies/taxonomy_uid_1/terms?include_count=true&skip=0&limit=100') + .get(`/v3/taxonomies/${mockData.taxonomiesResp.taxonomies[0].uid}/terms?include_count=true&skip=0&limit=100`) .reply(200, { terms: mockData.termsResp.terms, count: mockData.termsResp.count }); }) .command(['cm:export-to-csv', '--taxonomy-uid', 'taxonomy_uid_1']) @@ -126,12 +123,13 @@ describe('export-to-csv with action taxonomies', () => { }); describe('export-to-csv with action entries', () => { - if (!fs.existsSync(directory)) fs.mkdirSync(directory); describe('Create entries csv file with flags', () => { test .stdout({ print: process.env.PRINT === 'true' || false }) .stub(fs, 'createWriteStream', () => new PassThrough()) + .stub(mkdirp, 'sync', () => {}) + .stub(process, 'chdir', () => {}) .nock(cma, (api) => { api .get(`/v3/stacks?&query={"org_uid":"${mockData.organizations[0].uid}"}`) @@ -182,6 +180,8 @@ describe('export-to-csv with action entries', () => { test .stdout({ print: process.env.PRINT === 'true' || false }) .stub(fs, 'createWriteStream', () => new PassThrough()) + .stub(mkdirp, 'sync', () => {}) + .stub(process, 'chdir', () => {}) .stub(inquirer, 'registerPrompt', () => {}) .stub(inquirer, 'prompt', () => { return Promise.resolve({ @@ -240,6 +240,8 @@ describe('export-to-csv with action users', () => { test .stdout({ print: process.env.PRINT === 'true' || false }) .stub(fs, 'createWriteStream', () => new PassThrough()) + .stub(mkdirp, 'sync', () => {}) + .stub(process, 'chdir', () => {}) .nock(cma, (api) => { api.get('/v3/user?include_orgs_roles=true').reply(200, { user: mockData.users[0] }); }) @@ -262,6 +264,8 @@ describe('export-to-csv with action users', () => { test .stdout({ print: process.env.PRINT === 'true' || false }) .stub(fs, 'createWriteStream', () => new PassThrough()) + .stub(mkdirp, 'sync', () => {}) + .stub(process, 'chdir', () => {}) .stub(inquirer, 'registerPrompt', () => {}) .stub(inquirer, 'prompt', () => { return Promise.resolve({ action: 'users', chosenOrg: mockData.organizations[0].name }); From b49746e0571527e0b0627bd5f0f83183072d3577 Mon Sep 17 00:00:00 2001 From: Aman Kumar Date: Wed, 27 Sep 2023 16:19:08 +0530 Subject: [PATCH 23/45] refactor: identical user api call --- .../test/unit/commands/export-to-csv.test.js | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/packages/contentstack-export-to-csv/test/unit/commands/export-to-csv.test.js b/packages/contentstack-export-to-csv/test/unit/commands/export-to-csv.test.js index bd562fc05f..0f88eb5311 100644 --- a/packages/contentstack-export-to-csv/test/unit/commands/export-to-csv.test.js +++ b/packages/contentstack-export-to-csv/test/unit/commands/export-to-csv.test.js @@ -243,14 +243,11 @@ describe('export-to-csv with action users', () => { .stub(mkdirp, 'sync', () => {}) .stub(process, 'chdir', () => {}) .nock(cma, (api) => { - api.get('/v3/user?include_orgs_roles=true').reply(200, { user: mockData.users[0] }); + api.get('/v3/user?include_orgs_roles=true').reply(200, { user: mockData.users[0] }).persist(); }) .nock(cma, (api) => { api.get(`/v3/organizations/${mockData.organizations[0].uid}/roles`).reply(200, { roles: mockData.roles }); }) - .nock(cma, (api) => { - api.get('/v3/user?include_orgs_roles=true').reply(200, { user: mockData.users[0] }); - }) .nock(cma, (api) => { api .get(`/v3/organizations/${mockData.organizations[0].uid}/share?skip=0&page=1&limit=100`) @@ -274,14 +271,11 @@ describe('export-to-csv with action users', () => { api.get(`/v3/organizations?limit=100`).reply(200, { organizations: mockData.organizations }); }) .nock(cma, (api) => { - api.get('/v3/user?include_orgs_roles=true').reply(200, { user: mockData.users[0] }); + api.get('/v3/user?include_orgs_roles=true').reply(200, { user: mockData.users[0] }).persist(); }) .nock(cma, (api) => { api.get(`/v3/organizations/${mockData.organizations[0].uid}/roles`).reply(200, { roles: mockData.roles }); }) - .nock(cma, (api) => { - api.get('/v3/user?include_orgs_roles=true').reply(200, { user: mockData.users[0] }); - }) .nock(cma, (api) => { api .get(`/v3/organizations/${mockData.organizations[0].uid}/share?skip=0&page=1&limit=100`) From 97cd3fd066f376bad6803e32bf7c88836b90685b Mon Sep 17 00:00:00 2001 From: Aman Kumar Date: Tue, 3 Oct 2023 11:21:55 +0530 Subject: [PATCH 24/45] feat: content type module support taxonomy --- .../src/import/modules/content-types.ts | 11 ++++-- .../src/import/modules/taxonomies.ts | 8 ++--- .../contentstack-import/src/utils/index.ts | 1 + .../src/utils/taxonomies-helper.ts | 34 +++++++++++++++++++ 4 files changed, 47 insertions(+), 7 deletions(-) create mode 100644 packages/contentstack-import/src/utils/taxonomies-helper.ts diff --git a/packages/contentstack-import/src/import/modules/content-types.ts b/packages/contentstack-import/src/import/modules/content-types.ts index 86557f6a8b..0faff4e7ff 100644 --- a/packages/contentstack-import/src/import/modules/content-types.ts +++ b/packages/contentstack-import/src/import/modules/content-types.ts @@ -7,7 +7,7 @@ import * as path from 'path'; import { isEmpty, find, cloneDeep, map } from 'lodash'; -import { fsUtil, log, formatError, schemaTemplate, lookupExtension } from '../../utils'; +import { fsUtil, log, formatError, schemaTemplate, lookupExtension, lookUpTaxonomy } from '../../utils'; import { ImportConfig, ModuleClassParams } from '../../types'; import BaseClass, { ApiOptions } from './base-class'; import { updateFieldRules } from '../../utils/content-type-helper'; @@ -49,6 +49,8 @@ export default class ContentTypesImport extends BaseClass { limit: number; writeConcurrency?: number; }; + private taxonomiesPath: string; + public taxonomies: Record[] = []; constructor({ importConfig, stackAPIClient }: ModuleClassParams) { super({ importConfig, stackAPIClient }); @@ -75,6 +77,7 @@ export default class ContentTypesImport extends BaseClass { this.gFs = []; this.createdGFs = []; this.pendingGFs = []; + this.taxonomiesPath = path.join(importConfig.data, 'mapper/taxonomies', 'success.json'); } async start(): Promise { @@ -85,7 +88,6 @@ export default class ContentTypesImport extends BaseClass { * Update pending global fields * write field rules */ - this.cTs = fsUtil.readFile(path.join(this.cTsFolderPath, 'schema.json')) as Record[]; if (!this.cTs || isEmpty(this.cTs)) { log(this.importConfig, 'No content type found to import', 'info'); @@ -95,6 +97,7 @@ export default class ContentTypesImport extends BaseClass { this.installedExtensions = ( ((await fsUtil.readFile(this.marketplaceAppMapperPath)) as any) || { extension_uid: {} } ).extension_uid; + this.taxonomies = fsUtil.readFile(this.taxonomiesPath) as Record[]; await this.seedCTs(); log(this.importConfig, 'Created content types', 'success'); @@ -152,7 +155,7 @@ export default class ContentTypesImport extends BaseClass { async updateCTs(): Promise { const onSuccess = ({ response: contentType, apiData: { uid } }: any) => { - log(this.importConfig, `${uid} updated with references`, 'success'); + log(this.importConfig, `'${uid}' updated with references`, 'success'); }; const onReject = ({ error, apiData: { uid } }: any) => { log(this.importConfig, formatError(error), 'error'); @@ -186,6 +189,8 @@ export default class ContentTypesImport extends BaseClass { } this.fieldRules.push(contentType.uid); } + //update taxonomy field and display warning msg if taxonomy creation failed before removing + lookUpTaxonomy(contentType.schema, this.taxonomies); lookupExtension( this.importConfig, contentType.schema, diff --git a/packages/contentstack-import/src/import/modules/taxonomies.ts b/packages/contentstack-import/src/import/modules/taxonomies.ts index 37cade6c80..80e923e33b 100644 --- a/packages/contentstack-import/src/import/modules/taxonomies.ts +++ b/packages/contentstack-import/src/import/modules/taxonomies.ts @@ -13,7 +13,7 @@ import { ModuleClassParams, TaxonomiesConfig, TermsConfig } from '../../types'; type TaxonomyPayload = { baseUrl: string; url: string; - mgToken: string; + mgToken?: string; reqPayload: Record; headers: Record; }; @@ -118,7 +118,7 @@ export default class ImportTaxonomies extends BaseClass { return; } - const apiContent = values(this.taxonomies); + const apiContent = values(this.taxonomies) as Record[];; this.taxonomyUIDs = keys(this.taxonomies); const onSuccess = ({ @@ -131,7 +131,7 @@ export default class ImportTaxonomies extends BaseClass { this.taxonomiesSuccess[taxonomy.uid] = taxonomy; log(this.importConfig, `Taxonomy '${name}' imported successfully`, 'success'); } else { - let errorMsg; + let errorMsg:any; if ([500, 503, 502].includes(status)) errorMsg = data?.message || data; else errorMsg = data?.error_message; if (errorMsg === undefined) { @@ -259,7 +259,7 @@ export default class ImportTaxonomies extends BaseClass { ) as Record; if (this.terms !== undefined && !isEmpty(this.terms)) { - const apiContent = values(this.terms); + const apiContent = values(this.terms) as Record[]; await this.makeConcurrentCall( { apiContent, diff --git a/packages/contentstack-import/src/utils/index.ts b/packages/contentstack-import/src/utils/index.ts index 19cfca611e..1cfe9a620e 100644 --- a/packages/contentstack-import/src/utils/index.ts +++ b/packages/contentstack-import/src/utils/index.ts @@ -27,3 +27,4 @@ export { restoreJsonRteEntryRefs, } from './entries-helper'; export * from './common-helper'; +export { lookUpTaxonomy } from './taxonomies-helper'; diff --git a/packages/contentstack-import/src/utils/taxonomies-helper.ts b/packages/contentstack-import/src/utils/taxonomies-helper.ts new file mode 100644 index 0000000000..df9ad6fc48 --- /dev/null +++ b/packages/contentstack-import/src/utils/taxonomies-helper.ts @@ -0,0 +1,34 @@ +/** + * taxonomy lookup + */ +import { cliux } from '@contentstack/cli-utilities'; + +export const lookUpTaxonomy = function (schema: any, taxonomies: Record[]) { + for (let i in schema) { + if (schema[i].data_type === 'taxonomy') { + const taxonomyFieldData = schema[i].taxonomies as Record[]; + if (!taxonomyFieldData.length) break; + for (let index = 0; index < taxonomyFieldData.length; index++) { + const taxonomyData = taxonomyFieldData[index]; + if (taxonomies === undefined || !taxonomies[taxonomyData?.taxonomy_uid]) { + // remove taxonomy from taxonomies field data with warning + cliux.print( + `Taxonomy '${taxonomyData?.taxonomy_uid}' does not exist. Removing the data from taxonomy field`, + { color: 'yellow' }, + ); + taxonomyFieldData.splice(index, 1); + --index; + } + } + //Case 1:- if no taxonomy exist and trying to create taxonomies field API error -> The 'taxonomies' property must have atleast one taxonomy object. + if (!taxonomyFieldData?.length) { + cliux.print(`Content type related Taxonomy does not exist in stack. Removing the field from schema`, { + color: 'yellow', + }); + schema.splice(i, 1); + } else { + schema[i].taxonomies = taxonomyFieldData; + } + } + } +}; From 22c1db1de1ad70fb61db14e7ff10bbaba6eecf0d Mon Sep 17 00:00:00 2001 From: Aman Kumar Date: Wed, 4 Oct 2023 16:01:43 +0530 Subject: [PATCH 25/45] feat: entries module supports taxonomy --- .../src/import/modules/content-types.ts | 4 +- .../src/import/modules/entries.ts | 8 ++ .../src/import/modules/taxonomies.ts | 5 +- .../contentstack-import/src/utils/index.ts | 2 +- .../src/utils/taxonomies-helper.ts | 120 +++++++++++++++--- 5 files changed, 114 insertions(+), 25 deletions(-) diff --git a/packages/contentstack-import/src/import/modules/content-types.ts b/packages/contentstack-import/src/import/modules/content-types.ts index 0faff4e7ff..4440813098 100644 --- a/packages/contentstack-import/src/import/modules/content-types.ts +++ b/packages/contentstack-import/src/import/modules/content-types.ts @@ -50,7 +50,7 @@ export default class ContentTypesImport extends BaseClass { writeConcurrency?: number; }; private taxonomiesPath: string; - public taxonomies: Record[] = []; + public taxonomies: Record; constructor({ importConfig, stackAPIClient }: ModuleClassParams) { super({ importConfig, stackAPIClient }); @@ -97,7 +97,7 @@ export default class ContentTypesImport extends BaseClass { this.installedExtensions = ( ((await fsUtil.readFile(this.marketplaceAppMapperPath)) as any) || { extension_uid: {} } ).extension_uid; - this.taxonomies = fsUtil.readFile(this.taxonomiesPath) as Record[]; + this.taxonomies = fsUtil.readFile(this.taxonomiesPath) as Record; await this.seedCTs(); log(this.importConfig, 'Created content types', 'success'); diff --git a/packages/contentstack-import/src/import/modules/entries.ts b/packages/contentstack-import/src/import/modules/entries.ts index fa83c02d06..84d06fd2e0 100644 --- a/packages/contentstack-import/src/import/modules/entries.ts +++ b/packages/contentstack-import/src/import/modules/entries.ts @@ -20,6 +20,7 @@ import { lookupEntries, lookupAssets, fileHelper, + lookUpTerms, } from '../../utils'; import { ModuleClassParams } from '../../types'; import BaseClass, { ApiOptions } from './base-class'; @@ -53,6 +54,8 @@ export default class EntriesImport extends BaseClass { private entriesUidMapper: Record; private envs: Record; private autoCreatedEntries: Record[]; + private taxonomiesPath: string; + public taxonomies: Record; constructor({ importConfig, stackAPIClient }: ModuleClassParams) { super({ importConfig, stackAPIClient }); @@ -64,6 +67,7 @@ export default class EntriesImport extends BaseClass { this.uniqueUidMapperPath = path.join(this.entriesMapperPath, 'unique-mapping.json'); this.modifiedCTsPath = path.join(this.entriesMapperPath, 'modified-schemas.json'); this.marketplaceAppMapperPath = path.join(this.importConfig.data, 'mapper', 'marketplace_apps', 'uid-mapping.json'); + this.taxonomiesPath = path.join(this.importConfig.data, 'mapper', 'taxonomies', 'terms', 'success.json'); this.entriesConfig = importConfig.modules.entries; this.entriesPath = path.resolve(importConfig.data, this.entriesConfig.dirName); this.cTsPath = path.resolve(importConfig.data, importConfig.modules['content-types'].dirName); @@ -96,6 +100,8 @@ export default class EntriesImport extends BaseClass { this.assetUidMapper = (fsUtil.readFile(this.assetUidMapperPath) as Record) || {}; this.assetUrlMapper = (fsUtil.readFile(this.assetUrlMapperPath) as Record) || {}; + + this.taxonomies = (fsUtil.readFile(this.taxonomiesPath) as Record); fsUtil.makeDirectory(this.entriesMapperPath); await this.disableMandatoryCTReferences(); @@ -360,6 +366,8 @@ export default class EntriesImport extends BaseClass { if (this.jsonRteCTsWithRef.indexOf(cTUid) > -1) { entry = removeEntryRefsFromJSONRTE(entry, contentType.schema); } + //will remove term if term doesn't exists in taxonomy + lookUpTerms(contentType?.schema, entry, this.taxonomies); // will replace all old asset uid/urls with new ones entry = lookupAssets( { diff --git a/packages/contentstack-import/src/import/modules/taxonomies.ts b/packages/contentstack-import/src/import/modules/taxonomies.ts index 80e923e33b..fa80f66a34 100644 --- a/packages/contentstack-import/src/import/modules/taxonomies.ts +++ b/packages/contentstack-import/src/import/modules/taxonomies.ts @@ -1,4 +1,5 @@ import keys from 'lodash/keys'; +import pick from 'lodash/pick'; import { join } from 'node:path'; import values from 'lodash/values'; import isEmpty from 'lodash/isEmpty'; @@ -128,7 +129,7 @@ export default class ImportTaxonomies extends BaseClass { //NOTE - Temp code to handle error thru API. Will remove this once sdk is ready if ([200, 201, 202].includes(status)) { const { taxonomy } = data; - this.taxonomiesSuccess[taxonomy.uid] = taxonomy; + this.taxonomiesSuccess[taxonomy.uid] = pick(taxonomy, ['name', 'description']); log(this.importConfig, `Taxonomy '${name}' imported successfully`, 'success'); } else { let errorMsg:any; @@ -223,7 +224,7 @@ export default class ImportTaxonomies extends BaseClass { if ([200, 201, 202].includes(status)) { if (!this.termsSuccess[taxonomy_uid]) this.termsSuccess[taxonomy_uid] = {}; const { term } = data; - this.termsSuccess[taxonomy_uid][term.uid] = term; + this.termsSuccess[taxonomy_uid][term.uid] = pick(term, ['name']); log(this.importConfig, `Term '${name}' imported successfully`, 'success'); } else { if (!this.termsFailed[taxonomy_uid]) this.termsFailed[taxonomy_uid] = {}; diff --git a/packages/contentstack-import/src/utils/index.ts b/packages/contentstack-import/src/utils/index.ts index 1cfe9a620e..c76ee3340d 100644 --- a/packages/contentstack-import/src/utils/index.ts +++ b/packages/contentstack-import/src/utils/index.ts @@ -27,4 +27,4 @@ export { restoreJsonRteEntryRefs, } from './entries-helper'; export * from './common-helper'; -export { lookUpTaxonomy } from './taxonomies-helper'; +export { lookUpTaxonomy, lookUpTerms } from './taxonomies-helper'; diff --git a/packages/contentstack-import/src/utils/taxonomies-helper.ts b/packages/contentstack-import/src/utils/taxonomies-helper.ts index df9ad6fc48..ea046c7643 100644 --- a/packages/contentstack-import/src/utils/taxonomies-helper.ts +++ b/packages/contentstack-import/src/utils/taxonomies-helper.ts @@ -3,32 +3,112 @@ */ import { cliux } from '@contentstack/cli-utilities'; -export const lookUpTaxonomy = function (schema: any, taxonomies: Record[]) { +/** + * check and remove if referenced taxonomy doesn't exists in stack + * @param {any} schema content type schema + * @param {Record} taxonomies created taxonomies + */ +export const lookUpTaxonomy = function (schema: any, taxonomies: Record) { for (let i in schema) { if (schema[i].data_type === 'taxonomy') { const taxonomyFieldData = schema[i].taxonomies as Record[]; - if (!taxonomyFieldData.length) break; - for (let index = 0; index < taxonomyFieldData.length; index++) { - const taxonomyData = taxonomyFieldData[index]; - if (taxonomies === undefined || !taxonomies[taxonomyData?.taxonomy_uid]) { - // remove taxonomy from taxonomies field data with warning - cliux.print( - `Taxonomy '${taxonomyData?.taxonomy_uid}' does not exist. Removing the data from taxonomy field`, - { color: 'yellow' }, - ); - taxonomyFieldData.splice(index, 1); - --index; - } - } - //Case 1:- if no taxonomy exist and trying to create taxonomies field API error -> The 'taxonomies' property must have atleast one taxonomy object. - if (!taxonomyFieldData?.length) { - cliux.print(`Content type related Taxonomy does not exist in stack. Removing the field from schema`, { - color: 'yellow', - }); + const { updatedTaxonomyData, isTaxonomyFieldRemoved } = verifyAndRemoveTaxonomy(taxonomyFieldData, taxonomies); + + //Handle API error -> The 'taxonomies' property must have atleast one taxonomy object. Remove taxonomy field from schema. + if (isTaxonomyFieldRemoved) { schema.splice(i, 1); } else { - schema[i].taxonomies = taxonomyFieldData; + schema[i].taxonomies = updatedTaxonomyData; } } } }; + +/** + * verify and remove referenced taxonomy with warning from respective content type + * @param {Record[]} taxonomyFieldData + * @param {Record} taxonomies created taxonomies + * @returns + */ +const verifyAndRemoveTaxonomy = function ( + taxonomyFieldData: Record[], + taxonomies: Record, +): { + updatedTaxonomyData: Record[]; + isTaxonomyFieldRemoved: boolean; +} { + let isTaxonomyFieldRemoved: boolean = false; + + for (let index = 0; index < taxonomyFieldData?.length; index++) { + const taxonomyData = taxonomyFieldData[index]; + + if (taxonomies === undefined || !taxonomies.hasOwnProperty(taxonomyData?.taxonomy_uid)) { + // remove taxonomy from taxonomies field data with warning if respective taxonomy doesn't exists + cliux.print(`Taxonomy '${taxonomyData?.taxonomy_uid}' does not exist. Removing the data from taxonomies field`, { + color: 'yellow', + }); + taxonomyFieldData.splice(index, 1); + --index; + } + } + + if (!taxonomyFieldData?.length) { + cliux.print('Taxonomy does not exist. Removing the field from content type', { color: 'yellow' }); + isTaxonomyFieldRemoved = true; + } + + return { + updatedTaxonomyData: taxonomyFieldData, + isTaxonomyFieldRemoved, + }; +}; + +/** + * check and remove if referenced terms doesn't exists in taxonomy + * @param {Record[]} ctSchema content type schema + * @param {any} entry + * @param {Record} taxonomiesAndTermData created taxonomies and terms + */ +export const lookUpTerms = function ( + ctSchema: Record[], + entry: any, + taxonomiesAndTermData: Record, +) { + for (let index = 0; index < ctSchema?.length; index++) { + if (ctSchema[index].data_type === 'taxonomy') { + const taxonomyFieldData = entry[ctSchema[index].uid]; + const updatedTaxonomyData = verifyAndRemoveTerms(taxonomyFieldData, taxonomiesAndTermData); + entry[ctSchema[index].uid] = updatedTaxonomyData; + } + } +}; + +/** + * verify and remove referenced term with warning from respective entry + * @param {Record[]} taxonomyFieldData entry taxonomies data + * @param {Record} taxonomiesAndTermData created taxonomies and terms + * @returns { Record[]} + */ +const verifyAndRemoveTerms = function ( + taxonomyFieldData: Record[], + taxonomiesAndTermData: Record, +): Record[] { + for (let index = 0; index < taxonomyFieldData?.length; index++) { + const taxonomyData = taxonomyFieldData[index]; + const taxUID = taxonomyData?.taxonomy_uid; + const termUID = taxonomyData?.term_uid; + + if ( + taxonomiesAndTermData === undefined || + !taxonomiesAndTermData.hasOwnProperty(taxUID) || + (taxonomiesAndTermData.hasOwnProperty(taxUID) && !taxonomiesAndTermData[taxUID].hasOwnProperty(termUID)) + ) { + // remove term from taxonomies field data with warning if respective term doesn't exists + cliux.print(`Term '${termUID}' does not exist. Removing it from taxonomy - '${taxUID}'`, { color: 'yellow' }); + taxonomyFieldData.splice(index, 1); + --index; + } + } + + return taxonomyFieldData; +}; From 1147c446362f2ffdbd6299b91ccc964b75fdda1d Mon Sep 17 00:00:00 2001 From: Aman Kumar Date: Wed, 4 Oct 2023 16:44:21 +0530 Subject: [PATCH 26/45] refactor: comment message --- .../contentstack-import/src/import/modules/content-types.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/packages/contentstack-import/src/import/modules/content-types.ts b/packages/contentstack-import/src/import/modules/content-types.ts index 4440813098..049fd0eedb 100644 --- a/packages/contentstack-import/src/import/modules/content-types.ts +++ b/packages/contentstack-import/src/import/modules/content-types.ts @@ -189,7 +189,7 @@ export default class ContentTypesImport extends BaseClass { } this.fieldRules.push(contentType.uid); } - //update taxonomy field and display warning msg if taxonomy creation failed before removing + //will remove taxonomy if taxonomy doesn't exists in stack lookUpTaxonomy(contentType.schema, this.taxonomies); lookupExtension( this.importConfig, From 6f1c6c5212efd1b2ed126fd7ad6150ed88b44348 Mon Sep 17 00:00:00 2001 From: Aman Kumar Date: Thu, 5 Oct 2023 15:40:29 +0530 Subject: [PATCH 27/45] feat: sanitize taxonomies & terms against csv injection --- .../src/util/index.js | 76 ++++++++++--------- 1 file changed, 40 insertions(+), 36 deletions(-) diff --git a/packages/contentstack-export-to-csv/src/util/index.js b/packages/contentstack-export-to-csv/src/util/index.js index 812ae97064..8eb09fc902 100644 --- a/packages/contentstack-export-to-csv/src/util/index.js +++ b/packages/contentstack-export-to-csv/src/util/index.js @@ -376,20 +376,20 @@ function exitProgram() { process.exit(); } -function sanitizeEntries(flatEntry) { +function sanitizeData(flatData) { // sanitize against CSV Injections - const CSVRegex = /^[\\+\\=@\\-]/ - for (key in flatEntry) { - if (typeof flatEntry[key] === 'string' && flatEntry[key].match(CSVRegex)) { - flatEntry[key] = flatEntry[key].replace(/\"/g, "\"\""); - flatEntry[key] = `"'${flatEntry[key]}"` - } else if (typeof flatEntry[key] === 'object') { + const CSVRegex = /^[\\+\\=@\\-]/; + for (key in flatData) { + if (typeof flatData[key] === 'string' && flatData[key].match(CSVRegex)) { + flatData[key] = flatData[key].replace(/\"/g, '""'); + flatData[key] = `"'${flatData[key]}"`; + } else if (typeof flatData[key] === 'object') { // convert any objects or arrays to string // to store this data correctly in csv - flatEntry[key] = JSON.stringify(flatEntry[key]); + flatData[key] = JSON.stringify(flatData[key]); } } - return flatEntry; + return flatData; } function cleanEntries(entries, language, environments, contentTypeUid) { @@ -414,7 +414,7 @@ function cleanEntries(entries, language, environments, contentTypeUid) { } } entry = flatten(entry); - entry = sanitizeEntries(entry); + entry = sanitizeData(entry); entry['publish_details'] = envArr; entry['_workflow'] = workflow; entry['ACL'] = JSON.stringify({}); // setting ACL to empty obj @@ -685,15 +685,15 @@ function wait(time) { /** * fetch all taxonomies in the provided stack - * @param {object} payload - * @param {number} skip - * @param {number} limit - * @param {array} taxonomies - * @returns + * @param {object} payload + * @param {number} skip + * @param {number} limit + * @param {array} taxonomies + * @returns */ async function getAllTaxonomies(payload, skip = 0, limit = 100, taxonomies = []) { const response = await apiRequestHandler(payload, skip, limit); - if(response){ + if (response) { skip += config.limit || 100; taxonomies = [...taxonomies, ...response.taxonomies]; if (skip >= response?.count) { @@ -707,15 +707,15 @@ async function getAllTaxonomies(payload, skip = 0, limit = 100, taxonomies = []) /** * fetch terms of related taxonomy - * @param {object} payload - * @param {number} skip - * @param {number} limit - * @param {array} terms - * @returns + * @param {object} payload + * @param {number} skip + * @param {number} limit + * @param {array} terms + * @returns */ async function getAllTermsOfTaxonomy(payload, skip = 0, limit = 100, terms = []) { const response = await apiRequestHandler(payload, skip, limit); - if(response){ + if (response) { skip += config.limit || 100; terms = [...terms, ...response.terms]; if (skip >= response?.count) { @@ -729,9 +729,9 @@ async function getAllTermsOfTaxonomy(payload, skip = 0, limit = 100, terms = []) /** * Verify the existence of a taxonomy. Obtain its details if it exists and return - * @param {object} payload - * @param {string} taxonomyUID - * @returns + * @param {object} payload + * @param {string} taxonomyUID + * @returns */ async function getTaxonomy(payload, taxonomyUID) { payload['url'] = `${payload.baseUrl}/${taxonomyUID}`; @@ -769,7 +769,7 @@ async function apiRequestHandler(payload, skip, limit) { let errorMsg; if ([500, 503, 502].includes(status)) errorMsg = data?.message || data; else errorMsg = data?.error_message; - if(errorMsg === undefined){ + if (errorMsg === undefined) { errorMsg = Object.values(data?.errors) && flat(Object.values(data.errors)); } cliux.print(`Error: ${errorMsg}`, { color: 'red' }); @@ -781,17 +781,19 @@ async function apiRequestHandler(payload, skip, limit) { /** * Change taxonomies data in required CSV headers format - * @param {array} taxonomies - * @returns + * @param {array} taxonomies + * @returns */ function formatTaxonomiesData(taxonomies) { - if(taxonomies?.length){ + if (taxonomies?.length) { const formattedTaxonomies = taxonomies.map((taxonomy) => { - return { + let obj = { 'Taxonomy UID': taxonomy.uid, Name: taxonomy.name, Description: taxonomy.description, }; + obj = sanitizeData(obj); + return obj; }); return formattedTaxonomies; } @@ -799,19 +801,21 @@ function formatTaxonomiesData(taxonomies) { /** * Modify the linked taxonomy data's terms in required CSV headers format - * @param {array} terms - * @param {string} taxonomyUID - * @returns + * @param {array} terms + * @param {string} taxonomyUID + * @returns */ function formatTermsOfTaxonomyData(terms, taxonomyUID) { - if(terms?.length){ + if (terms?.length) { const formattedTerms = terms.map((term) => { - return { + let obj = { 'Taxonomy UID': taxonomyUID, UID: term.uid, Name: term.name, 'Parent UID': term.parent_uid, - }; + } + obj = sanitizeData(obj); + return obj; }); return formattedTerms; } From 5454d149edbf322926aeec77f6997a0f787b4e9b Mon Sep 17 00:00:00 2001 From: Aman Kumar Date: Thu, 5 Oct 2023 16:39:51 +0530 Subject: [PATCH 28/45] refactor: sanitize taxonomies & term --- .../contentstack-export-to-csv/src/util/index.js | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/packages/contentstack-export-to-csv/src/util/index.js b/packages/contentstack-export-to-csv/src/util/index.js index 8eb09fc902..c9132be470 100644 --- a/packages/contentstack-export-to-csv/src/util/index.js +++ b/packages/contentstack-export-to-csv/src/util/index.js @@ -787,13 +787,11 @@ async function apiRequestHandler(payload, skip, limit) { function formatTaxonomiesData(taxonomies) { if (taxonomies?.length) { const formattedTaxonomies = taxonomies.map((taxonomy) => { - let obj = { + return sanitizeData({ 'Taxonomy UID': taxonomy.uid, Name: taxonomy.name, Description: taxonomy.description, - }; - obj = sanitizeData(obj); - return obj; + }); }); return formattedTaxonomies; } @@ -808,14 +806,12 @@ function formatTaxonomiesData(taxonomies) { function formatTermsOfTaxonomyData(terms, taxonomyUID) { if (terms?.length) { const formattedTerms = terms.map((term) => { - let obj = { + return sanitizeData({ 'Taxonomy UID': taxonomyUID, UID: term.uid, Name: term.name, 'Parent UID': term.parent_uid, - } - obj = sanitizeData(obj); - return obj; + }); }); return formattedTerms; } @@ -864,5 +860,5 @@ module.exports = { formatTaxonomiesData, formatTermsOfTaxonomyData, getTaxonomy, - getStacks + getStacks, }; From 0a5caa03df4797c9903b343f431b99530897cc0a Mon Sep 17 00:00:00 2001 From: Aman Kumar Date: Fri, 6 Oct 2023 13:21:13 +0530 Subject: [PATCH 29/45] refactor: taxonomy UI text messages --- .../src/commands/cm/export-to-csv.js | 10 +++---- .../src/export/modules/taxonomies.ts | 14 +++++----- .../src/import/modules/taxonomies.ts | 26 +++++++++---------- .../src/utils/taxonomies-helper.ts | 2 +- 4 files changed, 26 insertions(+), 26 deletions(-) diff --git a/packages/contentstack-export-to-csv/src/commands/cm/export-to-csv.js b/packages/contentstack-export-to-csv/src/commands/cm/export-to-csv.js index 1898bbeacc..dde0f49ae1 100644 --- a/packages/contentstack-export-to-csv/src/commands/cm/export-to-csv.js +++ b/packages/contentstack-export-to-csv/src/commands/cm/export-to-csv.js @@ -60,7 +60,7 @@ class ExportToCsvCommand extends Command { required: false, }), 'taxonomy-uid': flags.string({ - description: 'Provide taxonomy UID for which terms need to be exported', + description: 'Provide the taxonomy UID of the related terms you want to export', }), }; @@ -382,7 +382,7 @@ class ExportToCsvCommand extends Command { const fileName = `${stackName ? stackName : stack.name}_taxonomies.csv`; util.write(this, formattedTaxonomiesData, fileName, 'taxonomies'); } else { - cliux.print('info: No taxonomies found. Please provide a valid stack!', { color: 'blue' }); + cliux.print('info: No taxonomies found! Please provide a valid stack.', { color: 'blue' }); } for (let index = 0; index < taxonomies?.length; index++) { @@ -397,7 +397,7 @@ class ExportToCsvCommand extends Command { if (formattedTermsData?.length) { util.write(this, formattedTermsData, termFileName, 'terms'); } else { - cliux.print(`info: No terms found for the taxonomy UID - '${taxonomyUID}'`, { color: 'blue' }); + cliux.print(`info: No terms found for the taxonomy UID - '${taxonomyUID}'!`, { color: 'blue' }); } } } @@ -420,9 +420,9 @@ ExportToCsvCommand.examples = [ '', 'Exporting organization users to csv with organization name provided', 'csdx cm:export-to-csv --action --org --org-name ', - 'Exporting taxonomies and related terms to csv with taxonomy uid provided', + 'Exporting taxonomies and related terms to a .CSV file with the provided taxonomy UID', 'csdx cm:export-to-csv --action --alias --taxonomy-uid ', - 'Exporting taxonomies and respective terms to csv', + 'Exporting taxonomies and respective terms to a .CSV file', 'csdx cm:export-to-csv --action --alias ', ]; diff --git a/packages/contentstack-export/src/export/modules/taxonomies.ts b/packages/contentstack-export/src/export/modules/taxonomies.ts index 09aa0db139..e7d540911c 100644 --- a/packages/contentstack-export/src/export/modules/taxonomies.ts +++ b/packages/contentstack-export/src/export/modules/taxonomies.ts @@ -60,11 +60,11 @@ export default class ExportTaxonomies extends BaseClass { //fetch all taxonomies and write into taxonomies folder await this.getAllTaxonomies(this.taxonomyPayload); if (this.taxonomies === undefined || isEmpty(this.taxonomies)) { - log(this.exportConfig, 'No taxonomies found', 'info'); + log(this.exportConfig, 'No taxonomies found!', 'info'); return; } else { fsUtil.writeFile(pResolve(this.taxonomiesFolderPath, this.taxonomiesConfig.fileName), this.taxonomies); - log(this.exportConfig, 'All the taxonomies have been exported successfully!', 'success'); + log(this.exportConfig, 'All taxonomies exported successfully!', 'success'); } //fetch all terms of respective and write into taxonomies/terms folder @@ -101,7 +101,7 @@ export default class ExportTaxonomies extends BaseClass { const taxonomyUID = taxonomies[index].uid; const taxonomyName = taxonomies[index]?.name; this.taxonomies[taxonomyUID] = omit(taxonomies[index], this.taxonomiesConfig.invalidKeys); - log(this.exportConfig, `'${taxonomyName}' taxonomy was exported successfully`, 'success'); + log(this.exportConfig, `'${taxonomyName}' taxonomy exported successfully!`, 'success'); } } @@ -117,13 +117,13 @@ export default class ExportTaxonomies extends BaseClass { await this.fetchTermsOfTaxonomy(this.taxonomyPayload); if (this.terms === undefined || isEmpty(this.terms)) { - log(this.exportConfig, `No terms found for taxonomy - '${taxonomyUID}'`, 'info'); + log(this.exportConfig, `No terms found for taxonomy - '${taxonomyUID}'!`, 'info'); } else { fsUtil.writeFile(pResolve(this.termsFolderPath, `${taxonomyUID}-${this.termsConfig.fileName}`), this.terms); - log(this.exportConfig, `Terms from taxonomy '${taxonomyUID}' were exported successfully.`, 'success'); + log(this.exportConfig, `Terms from taxonomy '${taxonomyUID}' exported successfully!`, 'success'); } } - log(this.exportConfig, `All the terms have been exported successfully!`, 'success'); + log(this.exportConfig, `All the terms exported successfully!`, 'success'); } /** @@ -205,7 +205,7 @@ export default class ExportTaxonomies extends BaseClass { } else if (err?.message) { cliux.print(`Error: ${err.message}`, { color: 'red' }); } else { - cliux.print(`Error: Something went wrong. Please try again.`, { color: 'red' }); + cliux.print(`Error: Something went wrong! Please try again.`, { color: 'red' }); } } } diff --git a/packages/contentstack-import/src/import/modules/taxonomies.ts b/packages/contentstack-import/src/import/modules/taxonomies.ts index fa80f66a34..b3f3453989 100644 --- a/packages/contentstack-import/src/import/modules/taxonomies.ts +++ b/packages/contentstack-import/src/import/modules/taxonomies.ts @@ -68,7 +68,7 @@ export default class ImportTaxonomies extends BaseClass { * @returns {Promise} Promise */ async start(): Promise { - log(this.importConfig, 'Migrating taxonomies', 'info'); + log(this.importConfig, 'Migrating taxonomies...', 'info'); //Step1 check folder exists or not if (fileHelper.fileExistsSync(this.taxonomiesFolderPath)) { @@ -104,7 +104,7 @@ export default class ImportTaxonomies extends BaseClass { await this.importTerms(); this.createTermSuccessAndFailedFile(); - log(this.importConfig, 'Taxonomies have been imported successfully!', 'success'); + log(this.importConfig, 'Taxonomies imported successfully!', 'success'); } /** @@ -115,7 +115,7 @@ export default class ImportTaxonomies extends BaseClass { */ async importTaxonomies(): Promise { if (this.taxonomies === undefined || isEmpty(this.taxonomies)) { - log(this.importConfig, 'No Taxonomies Found', 'info'); + log(this.importConfig, 'No Taxonomies Found!', 'info'); return; } @@ -130,7 +130,7 @@ export default class ImportTaxonomies extends BaseClass { if ([200, 201, 202].includes(status)) { const { taxonomy } = data; this.taxonomiesSuccess[taxonomy.uid] = pick(taxonomy, ['name', 'description']); - log(this.importConfig, `Taxonomy '${name}' imported successfully`, 'success'); + log(this.importConfig, `Taxonomy '${name}' imported successfully!`, 'success'); } else { let errorMsg:any; if ([500, 503, 502].includes(status)) errorMsg = data?.message || data; @@ -138,8 +138,8 @@ export default class ImportTaxonomies extends BaseClass { if (errorMsg === undefined) { errorMsg = Object.values(data?.errors) && flatten(Object.values(data.errors)); } - this.taxonomiesFailed[uid] = `Taxonomy '${name}' failed to be import. ${JSON.stringify(errorMsg)}`; - log(this.importConfig, `Taxonomy '${name}' failed to be import. ${JSON.stringify(errorMsg)}`, 'error'); + this.taxonomiesFailed[uid] = `Taxonomy '${name}' failed to be import! ${JSON.stringify(errorMsg)}`; + log(this.importConfig, `Taxonomy '${name}' failed to be import! ${JSON.stringify(errorMsg)}`, 'error'); } }; @@ -147,10 +147,10 @@ export default class ImportTaxonomies extends BaseClass { const err = error?.message ? JSON.parse(error.message) : error; const { name } = apiData; if (err?.errors?.name) { - log(this.importConfig, `Taxonomy '${name}' already exists`, 'info'); + log(this.importConfig, `Taxonomy '${name}' already exists!`, 'info'); } else { this.taxonomiesFailed[apiData.uid] = apiData; - log(this.importConfig, `Taxonomy '${name}' failed to be import ${formatError(error)}`, 'error'); + log(this.importConfig, `Taxonomy '${name}' failed to be import! ${formatError(error)}`, 'error'); log(this.importConfig, error, 'error'); } }; @@ -225,7 +225,7 @@ export default class ImportTaxonomies extends BaseClass { if (!this.termsSuccess[taxonomy_uid]) this.termsSuccess[taxonomy_uid] = {}; const { term } = data; this.termsSuccess[taxonomy_uid][term.uid] = pick(term, ['name']); - log(this.importConfig, `Term '${name}' imported successfully`, 'success'); + log(this.importConfig, `Term '${name}' imported successfully!`, 'success'); } else { if (!this.termsFailed[taxonomy_uid]) this.termsFailed[taxonomy_uid] = {}; let errorMsg; @@ -234,8 +234,8 @@ export default class ImportTaxonomies extends BaseClass { if (errorMsg === undefined) { errorMsg = Object.values(data?.errors) && flatten(Object.values(data.errors)); } - this.termsFailed[taxonomy_uid][uid] = `Terms '${name}' failed to be import. ${JSON.stringify(errorMsg)}`; - log(this.importConfig, `Terms '${name}' failed to be import. ${JSON.stringify(errorMsg)}`, 'error'); + this.termsFailed[taxonomy_uid][uid] = `Terms '${name}' failed to be import! ${JSON.stringify(errorMsg)}`; + log(this.importConfig, `Terms '${name}' failed to be import! ${JSON.stringify(errorMsg)}`, 'error'); } }; @@ -244,10 +244,10 @@ export default class ImportTaxonomies extends BaseClass { if (!this.termsFailed[taxonomy_uid]) this.termsFailed[taxonomy_uid] = {}; const err = error?.message ? JSON.parse(error.message) : error; if (err?.errors?.name) { - log(this.importConfig, `Term '${name}' already exists`, 'info'); + log(this.importConfig, `Term '${name}' already exists!`, 'info'); } else { this.termsFailed[taxonomy_uid][apiData.uid] = apiData; - log(this.importConfig, `Term '${name}' failed to be import ${formatError(error)}`, 'error'); + log(this.importConfig, `Term '${name}' failed to be import! ${formatError(error)}`, 'error'); log(this.importConfig, error, 'error'); } }; diff --git a/packages/contentstack-import/src/utils/taxonomies-helper.ts b/packages/contentstack-import/src/utils/taxonomies-helper.ts index ea046c7643..7d62242927 100644 --- a/packages/contentstack-import/src/utils/taxonomies-helper.ts +++ b/packages/contentstack-import/src/utils/taxonomies-helper.ts @@ -44,7 +44,7 @@ const verifyAndRemoveTaxonomy = function ( if (taxonomies === undefined || !taxonomies.hasOwnProperty(taxonomyData?.taxonomy_uid)) { // remove taxonomy from taxonomies field data with warning if respective taxonomy doesn't exists - cliux.print(`Taxonomy '${taxonomyData?.taxonomy_uid}' does not exist. Removing the data from taxonomies field`, { + cliux.print(`Taxonomy '${taxonomyData?.taxonomy_uid}' does not exist. Removing the data from the taxonomies field`, { color: 'yellow', }); taxonomyFieldData.splice(index, 1); From e0e3eb1067b3181a5a82b65089b2415113f3eabe Mon Sep 17 00:00:00 2001 From: Aman Kumar Date: Mon, 9 Oct 2023 19:23:31 +0530 Subject: [PATCH 30/45] feat: replace taxonomy & term http call with sdk --- package-lock.json | 40 ++--- .../src/commands/cm/export-to-csv.js | 19 +- .../src/util/index.js | 130 ++++++++------ .../src/export/modules/taxonomies.ts | 163 +++++++----------- packages/contentstack-import/package.json | 2 +- .../src/import/modules/base-class.ts | 21 +-- .../src/import/modules/taxonomies.ts | 114 +++--------- packages/contentstack-utilities/package.json | 2 +- packages/contentstack/package.json | 2 +- pnpm-lock.yaml | 22 +-- 10 files changed, 219 insertions(+), 296 deletions(-) diff --git a/package-lock.json b/package-lock.json index 5dad81b9fe..75c29afcc1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -811,11 +811,11 @@ } }, "node_modules/@contentstack/management": { - "version": "1.10.2", - "resolved": "https://registry.npmjs.org/@contentstack/management/-/management-1.10.2.tgz", - "integrity": "sha512-jO24EqcCJhOjqdsqw8y3T0SPPAd0DG4BByjUcV0S28W2yoa8aBbcjcbZioRPzRLYKTmZWsAZissl18cIJm5djQ==", + "version": "1.11.0", + "resolved": "https://registry.npmjs.org/@contentstack/management/-/management-1.11.0.tgz", + "integrity": "sha512-tv4At2Q5iGgkzL1MFGil/o36URKZfO6DY/KtpNJFYjmTHitZNv7uotH8OXkOPBMxB4xz58SG58lWB6fNTkLrpw==", "dependencies": { - "axios": "^1.4.0", + "axios": "^1.5.1", "form-data": "^3.0.1", "lodash": "^4.17.21", "qs": "^6.11.2" @@ -825,9 +825,9 @@ } }, "node_modules/@contentstack/management/node_modules/axios": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/axios/-/axios-1.5.0.tgz", - "integrity": "sha512-D4DdjDo5CY50Qms0qGQTTw6Q44jl7zRwY7bthds06pUGfChBCTcQs+N743eFWGEd6pRTMd6A+I87aWyFV5wiZQ==", + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.5.1.tgz", + "integrity": "sha512-Q28iYCWzNHjAm+yEAot5QaAMxhMghWLFVf7rRdwhUI+c2jix2DUXjAHXVi+s1ibs3mjPO/cCgbA++3BjD0vP/A==", "dependencies": { "follow-redirects": "^1.15.0", "form-data": "^4.0.0", @@ -22760,7 +22760,7 @@ "@contentstack/cli-launch": "~1.0.12", "@contentstack/cli-migration": "~1.3.13", "@contentstack/cli-utilities": "~1.5.3", - "@contentstack/management": "~1.10.0", + "@contentstack/management": "~1.11.0", "@oclif/core": "^2.9.3", "@oclif/plugin-help": "^5", "@oclif/plugin-not-found": "^2.4.0", @@ -24587,7 +24587,7 @@ "dependencies": { "@contentstack/cli-command": "~1.2.13", "@contentstack/cli-utilities": "~1.5.3", - "@contentstack/management": "~1.10.2", + "@contentstack/management": "~1.11.0", "@oclif/core": "^2.9.3", "big-json": "^3.2.0", "bluebird": "^3.7.2", @@ -25386,7 +25386,7 @@ "version": "1.5.3", "license": "MIT", "dependencies": { - "@contentstack/management": "~1.10.2", + "@contentstack/management": "~1.11.0", "@oclif/core": "^2.9.3", "axios": "1.3.4", "chalk": "^4.0.0", @@ -26999,7 +26999,7 @@ "@contentstack/cli-launch": "~1.0.12", "@contentstack/cli-migration": "~1.3.13", "@contentstack/cli-utilities": "~1.5.3", - "@contentstack/management": "~1.10.0", + "@contentstack/management": "~1.11.0", "@oclif/core": "^2.9.3", "@oclif/plugin-help": "^5", "@oclif/plugin-not-found": "^2.4.0", @@ -28122,7 +28122,7 @@ "requires": { "@contentstack/cli-command": "~1.2.13", "@contentstack/cli-utilities": "~1.5.3", - "@contentstack/management": "~1.10.2", + "@contentstack/management": "~1.11.0", "@oclif/core": "^2.9.3", "@oclif/test": "^1.2.6", "@types/big-json": "^3.2.0", @@ -29009,7 +29009,7 @@ "@contentstack/cli-utilities": { "version": "file:packages/contentstack-utilities", "requires": { - "@contentstack/management": "~1.10.2", + "@contentstack/management": "~1.11.0", "@oclif/core": "^2.9.3", "@oclif/test": "^2.2.10", "@types/chai": "^4.2.18", @@ -29863,20 +29863,20 @@ } }, "@contentstack/management": { - "version": "1.10.2", - "resolved": "https://registry.npmjs.org/@contentstack/management/-/management-1.10.2.tgz", - "integrity": "sha512-jO24EqcCJhOjqdsqw8y3T0SPPAd0DG4BByjUcV0S28W2yoa8aBbcjcbZioRPzRLYKTmZWsAZissl18cIJm5djQ==", + "version": "1.11.0", + "resolved": "https://registry.npmjs.org/@contentstack/management/-/management-1.11.0.tgz", + "integrity": "sha512-tv4At2Q5iGgkzL1MFGil/o36URKZfO6DY/KtpNJFYjmTHitZNv7uotH8OXkOPBMxB4xz58SG58lWB6fNTkLrpw==", "requires": { - "axios": "^1.4.0", + "axios": "^1.5.1", "form-data": "^3.0.1", "lodash": "^4.17.21", "qs": "^6.11.2" }, "dependencies": { "axios": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/axios/-/axios-1.5.0.tgz", - "integrity": "sha512-D4DdjDo5CY50Qms0qGQTTw6Q44jl7zRwY7bthds06pUGfChBCTcQs+N743eFWGEd6pRTMd6A+I87aWyFV5wiZQ==", + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.5.1.tgz", + "integrity": "sha512-Q28iYCWzNHjAm+yEAot5QaAMxhMghWLFVf7rRdwhUI+c2jix2DUXjAHXVi+s1ibs3mjPO/cCgbA++3BjD0vP/A==", "requires": { "follow-redirects": "^1.15.0", "form-data": "^4.0.0", diff --git a/packages/contentstack-export-to-csv/src/commands/cm/export-to-csv.js b/packages/contentstack-export-to-csv/src/commands/cm/export-to-csv.js index dde0f49ae1..520f53d881 100644 --- a/packages/contentstack-export-to-csv/src/commands/cm/export-to-csv.js +++ b/packages/contentstack-export-to-csv/src/commands/cm/export-to-csv.js @@ -214,7 +214,6 @@ class ExportToCsvCommand extends Command { case 'taxonomies': { let stack; let stackAPIClient; - let taxUID; if (managementTokenAlias) { const { stackDetails, apiClient } = await this.getAliasDetails(managementTokenAlias, stackName); managementAPIClient = apiClient; @@ -224,7 +223,7 @@ class ExportToCsvCommand extends Command { } stackAPIClient = this.getStackClient(managementAPIClient, stack); - await this.createTaxonomyAndTermCsvFile(stackName, stack, taxonomyUID); + await this.createTaxonomyAndTermCsvFile(stackAPIClient, stackName, stack, taxonomyUID); break; } } @@ -242,8 +241,8 @@ class ExportToCsvCommand extends Command { getStackClient(managementAPIClient, stack) { const stackInit = { api_key: stack.apiKey, - branch_uid: stack.branch_uid, }; + if(stack?.branch_uid) stackInit['branch_uid'] = stack.branch_uid; if (stack.token) { return managementAPIClient.stack({ ...stackInit, @@ -360,20 +359,18 @@ class ExportToCsvCommand extends Command { * @param {object} stack * @param {string} taxUID */ - async createTaxonomyAndTermCsvFile(stackName, stack, taxUID) { - const { cma } = configHandler.get('region') || {}; + async createTaxonomyAndTermCsvFile(stackAPIClient, stackName, stack, taxUID) { const payload = { - baseUrl: `${cma}/v3/taxonomies`, - apiKey: stack.apiKey, - mgToken: stack?.token, + stackAPIClient, + type: '' }; //check whether the taxonomy is valid or not let taxonomies = []; if (taxUID) { - const taxonomy = await util.getTaxonomy(payload, taxUID); + payload['taxonomyUID'] = taxUID; + const taxonomy = await util.getTaxonomy(payload); taxonomies.push(taxonomy); } else { - payload['url'] = payload.baseUrl; taxonomies = await util.getAllTaxonomies(payload); } @@ -389,7 +386,7 @@ class ExportToCsvCommand extends Command { const taxonomy = taxonomies[index]; const taxonomyUID = taxonomy?.uid; if (taxonomyUID) { - payload['url'] = `${payload.baseUrl}/${taxonomyUID}/terms`; + payload['taxonomyUID'] = taxonomyUID; const terms = await util.getAllTermsOfTaxonomy(payload); const formattedTermsData = util.formatTermsOfTaxonomyData(terms, taxonomyUID); const taxonomyName = taxonomy?.name ? taxonomy.name : ''; diff --git a/packages/contentstack-export-to-csv/src/util/index.js b/packages/contentstack-export-to-csv/src/util/index.js index c9132be470..42912fd80f 100644 --- a/packages/contentstack-export-to-csv/src/util/index.js +++ b/packages/contentstack-export-to-csv/src/util/index.js @@ -9,7 +9,14 @@ const debug = require('debug')('export-to-csv'); const checkboxPlus = require('inquirer-checkbox-plus-prompt'); const config = require('./config.js'); -const { cliux, configHandler, HttpClient, messageHandler } = require('@contentstack/cli-utilities'); +const { + cliux, + configHandler, + HttpClient, + messageHandler, + managementSDKClient, + ContentstackClient, +} = require('@contentstack/cli-utilities'); const directory = './data'; const delimeter = os.platform() === 'win32' ? '\\' : '/'; @@ -687,41 +694,43 @@ function wait(time) { * fetch all taxonomies in the provided stack * @param {object} payload * @param {number} skip - * @param {number} limit * @param {array} taxonomies * @returns */ -async function getAllTaxonomies(payload, skip = 0, limit = 100, taxonomies = []) { - const response = await apiRequestHandler(payload, skip, limit); +async function getAllTaxonomies(payload, skip=0, taxonomies = []) { + payload['type'] = 'taxonomies'; + const response = await taxonomySDKHandler(payload); if (response) { skip += config.limit || 100; + //TODO - replace reponse.taxonomies with items taxonomies = [...taxonomies, ...response.taxonomies]; if (skip >= response?.count) { return taxonomies; } else { - return getAllTaxonomies(payload, skip, limit, taxonomies); + return getAllTaxonomies(payload, skip, taxonomies); } } return taxonomies; } /** - * fetch terms of related taxonomy + * fetch taxonomy related terms * @param {object} payload * @param {number} skip * @param {number} limit * @param {array} terms * @returns */ -async function getAllTermsOfTaxonomy(payload, skip = 0, limit = 100, terms = []) { - const response = await apiRequestHandler(payload, skip, limit); - if (response) { +async function getAllTermsOfTaxonomy(payload, skip=0, terms = []) { + payload['type'] = 'terms'; + const {items, count} = await taxonomySDKHandler(payload); + if (items) { skip += config.limit || 100; - terms = [...terms, ...response.terms]; - if (skip >= response?.count) { + terms = [...terms, ...items]; + if (skip >= count) { return terms; } else { - return getAllTermsOfTaxonomy(payload, skip, limit, terms); + return getAllTermsOfTaxonomy(payload, skip, terms); } } return terms; @@ -733,50 +742,56 @@ async function getAllTermsOfTaxonomy(payload, skip = 0, limit = 100, terms = []) * @param {string} taxonomyUID * @returns */ -async function getTaxonomy(payload, taxonomyUID) { - payload['url'] = `${payload.baseUrl}/${taxonomyUID}`; - const resp = await apiRequestHandler(payload); - return resp?.taxonomy || ''; -} - -async function apiRequestHandler(payload, skip, limit) { - const headers = { - api_key: payload.apiKey, - 'Content-Type': 'application/json', - }; - - if (payload?.mgToken) headers['authorization'] = payload.mgToken; - else headers['authToken'] = configHandler.get('authtoken'); - - const params = { - include_count: true, - skip: 0, - limit: 30, - }; - - if (skip >= 0) params['skip'] = skip; - if (limit >= 0) params['limit'] = limit; - - return await new HttpClient() - .headers(headers) - .queryParams(params) - .get(payload.url) - .then((res) => { - //NOTE - temporary code for handling api errors response - const { status, data } = res; - if ([200, 201, 202].includes(status)) return data; - else { - let errorMsg; - if ([500, 503, 502].includes(status)) errorMsg = data?.message || data; - else errorMsg = data?.error_message; - if (errorMsg === undefined) { - errorMsg = Object.values(data?.errors) && flat(Object.values(data.errors)); - } - cliux.print(`Error: ${errorMsg}`, { color: 'red' }); - process.exit(1); - } - }) - .catch((err) => handleErrorMsg(err)); +async function getTaxonomy(payload) { + payload['type'] = 'taxonomy'; + const resp = await taxonomySDKHandler(payload); + return resp; +} + +/** + * taxonomy & term sdk handler + * @async + * @method + * @param payload + * @param skip + * @param limit + * @returns {*} Promise + */ +async function taxonomySDKHandler(payload, skip) { + const { stackAPIClient,taxonomyUID, type } = payload; + + const queryParams = { include_count: true }; + queryParams['limit'] = config.limit || 100; + if (skip >= 0) queryParams['skip'] = skip; + + switch (type) { + case 'taxonomies': + //TODO - replace count with find + return await stackAPIClient + .taxonomy() + .query(queryParams) + .count() + .then((data) => data) + .catch((err) => handleErrorMsg(err)); + case 'taxonomy': + return await stackAPIClient + .taxonomy(taxonomyUID) + .fetch() + .then((data) => data) + .catch((err) => handleErrorMsg(err)); + case 'terms': + console.log("taxonomyUID---",taxonomyUID) + queryParams['depth'] = 0; + return await stackAPIClient + .taxonomy(taxonomyUID) + .terms() + .query(queryParams) + .find() + .then((data) => data) + .catch((err) => handleErrorMsg(err)); + default: + handleErrorMsg({ errorMessage: 'Invalid module!' }); + } } /** @@ -821,7 +836,8 @@ function handleErrorMsg(err) { if (err?.errorMessage) { cliux.print(`Error: ${err.errorMessage}`, { color: 'red' }); } else if (err?.message) { - cliux.print(`Error: ${err.message}`, { color: 'red' }); + const errorMsg = err?.errors?.taxonomy || err?.errors?.term || err?.message; + cliux.print(`Error: ${errorMsg}`, { color: 'red' }); } else { console.log(err); cliux.print(`Error: ${messageHandler.parse('CLI_EXPORT_CSV_API_FAILED')}`, { color: 'red' }); diff --git a/packages/contentstack-export/src/export/modules/taxonomies.ts b/packages/contentstack-export/src/export/modules/taxonomies.ts index e7d540911c..dc0de7378d 100644 --- a/packages/contentstack-export/src/export/modules/taxonomies.ts +++ b/packages/contentstack-export/src/export/modules/taxonomies.ts @@ -1,28 +1,23 @@ import omit from 'lodash/omit'; import keys from 'lodash/keys'; import isEmpty from 'lodash/isEmpty'; -import flatten from 'lodash/flatten'; import { resolve as pResolve } from 'node:path'; -import { cliux, configHandler, HttpClient } from '@contentstack/cli-utilities'; import BaseClass from './base-class'; import { log, fsUtil } from '../../utils'; import { TaxonomiesConfig, TermsConfig, ModuleClassParams } from '../../types'; -//NOTE: Temp types need to remove once sdk available -type TaxonomyPayload = { - baseUrl: string; - url: string; - mgToken: string; - apiKey: string; -}; - export default class ExportTaxonomies extends BaseClass { private taxonomies: Record>; private terms: Record>; private taxonomiesConfig: TaxonomiesConfig; - private taxonomyPayload: TaxonomyPayload; private termsConfig: TermsConfig; + private qs: { + include_count: boolean; + skip: number; + asc: string; + depth?: number; + }; public taxonomiesFolderPath: string; public termsFolderPath: string; @@ -32,12 +27,7 @@ export default class ExportTaxonomies extends BaseClass { this.terms = {}; this.taxonomiesConfig = exportConfig.modules.taxonomies; this.termsConfig = exportConfig.modules.terms; - this.taxonomyPayload = { - baseUrl: '', - url: '', - mgToken: exportConfig.management_token, - apiKey: exportConfig.source_stack, - }; + this.qs = { include_count: true, skip: 0, asc: 'created_at' }; } async start(): Promise { @@ -53,12 +43,8 @@ export default class ExportTaxonomies extends BaseClass { this.termsFolderPath = pResolve(this.taxonomiesFolderPath, this.termsConfig.dirName); await fsUtil.makeDirectory(this.termsFolderPath); - const { cma } = configHandler.get('region') || {}; - this.taxonomyPayload.baseUrl = `${cma}/v3/taxonomies`; - this.taxonomyPayload.url = this.taxonomyPayload.baseUrl; - //fetch all taxonomies and write into taxonomies folder - await this.getAllTaxonomies(this.taxonomyPayload); + await this.getAllTaxonomies(); if (this.taxonomies === undefined || isEmpty(this.taxonomies)) { log(this.exportConfig, 'No taxonomies found!', 'info'); return; @@ -73,22 +59,34 @@ export default class ExportTaxonomies extends BaseClass { /** * fetch all taxonomies in the provided stack - * @param {TaxonomyPayload} payload * @param {number} skip - * @returns + * @returns {Promise} */ - async getAllTaxonomies(payload: TaxonomyPayload, skip = 0): Promise { - const response = await this.apiRequestHandler(payload, skip); - if (response?.taxonomies) { - skip += this.taxonomiesConfig.limit || 100; - this.sanitizeTaxonomiesAttribs(response.taxonomies); - if (skip >= response?.count) { - return; - } else { - return await this.getAllTaxonomies(payload, skip); - } + async getAllTaxonomies(skip = 0): Promise { + if (skip) { + this.qs.skip = skip; } - return; + //TODO - replace count with find + await this.stack + .taxonomy() + .query(this.qs) + .count() + .then(async (data: any) => { + const { taxonomies, count } = data; + const taxonomiesCount = count !== undefined ? count : taxonomies?.length; + + if (taxonomies?.length) { + this.sanitizeTaxonomiesAttribs(taxonomies); + skip += this.taxonomiesConfig.limit || 100; + if (skip >= taxonomiesCount) { + return; + } + return await this.getAllTaxonomies(skip); + } + }) + .catch((error: any) => { + this.handleErrorMsg(error); + }); } /** @@ -107,15 +105,16 @@ export default class ExportTaxonomies extends BaseClass { /** * fetch all terms of respective taxonomy and write it into -terms file + * @returns {Promise} */ async getAllTerms() { const taxonomiesUID = keys(this.taxonomies) || []; + this.qs.depth = 0; + for (let index = 0; index < taxonomiesUID?.length; index++) { const taxonomyUID = taxonomiesUID[index]; - this.taxonomyPayload.url = `${this.taxonomyPayload.baseUrl}/${taxonomyUID}/terms`; this.terms = {}; - await this.fetchTermsOfTaxonomy(this.taxonomyPayload); - + await this.fetchTermsOfTaxonomy(taxonomyUID); if (this.terms === undefined || isEmpty(this.terms)) { log(this.exportConfig, `No terms found for taxonomy - '${taxonomyUID}'!`, 'info'); } else { @@ -128,22 +127,36 @@ export default class ExportTaxonomies extends BaseClass { /** * fetch all terms of the provided taxonomy uid - * @param {TaxonomyPayload} payload + * @async + * @param {string} taxonomyUID * @param {number} skip - * @returns + * @returns {Promise} */ - async fetchTermsOfTaxonomy(payload: TaxonomyPayload, skip = 0): Promise { - const response = await this.apiRequestHandler(payload, skip); - if (response?.terms) { - skip += this.termsConfig.limit || 100; - this.sanitizeTermsAttribs(response.terms); - if (skip >= response?.count) { - return; - } else { - return await this.fetchTermsOfTaxonomy(payload, skip); - } + async fetchTermsOfTaxonomy(taxonomyUID: string, skip = 0): Promise { + if (skip) { + this.qs.skip = skip; } - return; + await this.stack + .taxonomy(taxonomyUID) + .terms() + .query(this.qs) + .find() + .then(async (data: any) => { + const { items, count } = data; + const termsCount = count !== undefined ? count : items?.length; + + if (items?.length) { + this.sanitizeTermsAttribs(items); + skip += this.taxonomiesConfig.limit || 100; + if (skip >= termsCount) { + return; + } + return await this.fetchTermsOfTaxonomy(taxonomyUID, skip); + } + }) + .catch((error: any) => { + this.handleErrorMsg(error); + }); } /** @@ -158,54 +171,12 @@ export default class ExportTaxonomies extends BaseClass { } } - //NOTE: Temp code need to remove once sdk available - async apiRequestHandler(payload: TaxonomyPayload, skip: number) { - const headers: any = { - api_key: payload.apiKey, - 'Content-Type': 'application/json', - }; - - if (payload?.mgToken) headers['authorization'] = payload.mgToken; - else headers['authToken'] = configHandler.get('authtoken'); - - const params = { - include_count: true, - skip: 0, - limit: this.taxonomiesConfig.limit || 100, - depth: 0 // include all the terms if set to 0 - }; - - if (skip >= 0) params['skip'] = skip; - - return await new HttpClient() - .headers(headers) - .queryParams(params) - .get(payload.url) - .then((res: any) => { - //NOTE - temporary code for handling api errors response - const { status, data } = res; - if ([200, 201, 202].includes(status)) return data; - else { - let errorMsg; - if ([500, 503, 502].includes(status)) errorMsg = data?.message || data; - else errorMsg = data?.error_message; - if (errorMsg === undefined) { - errorMsg = Object.values(data?.errors) && flatten(Object.values(data.errors)); - } - cliux.print(`Error: ${errorMsg}`, { color: 'red' }); - } - }) - .catch((err: any) => this.handleErrorMsg(err)); - } - - //NOTE: Temp code need to remove once sdk available handleErrorMsg(err: any) { if (err?.errorMessage) { - cliux.print(`Error: ${err.errorMessage}`, { color: 'red' }); + log(this.exportConfig, `Failed to export. ${err.errorMessage}`, 'error'); } else if (err?.message) { - cliux.print(`Error: ${err.message}`, { color: 'red' }); - } else { - cliux.print(`Error: Something went wrong! Please try again.`, { color: 'red' }); + const errorMsg = err?.errors?.taxonomy || err?.errors?.term || err?.message; + log(this.exportConfig, `Failed to export. ${errorMsg}`, 'error'); } } } diff --git a/packages/contentstack-import/package.json b/packages/contentstack-import/package.json index 599d2ea456..7facb1d483 100644 --- a/packages/contentstack-import/package.json +++ b/packages/contentstack-import/package.json @@ -7,7 +7,7 @@ "dependencies": { "@contentstack/cli-command": "~1.2.13", "@contentstack/cli-utilities": "~1.5.3", - "@contentstack/management": "~1.10.2", + "@contentstack/management": "~1.11.0", "@oclif/core": "^2.9.3", "big-json": "^3.2.0", "bluebird": "^3.7.2", diff --git a/packages/contentstack-import/src/import/modules/base-class.ts b/packages/contentstack-import/src/import/modules/base-class.ts index e47f8790ab..7a758fb8a6 100644 --- a/packages/contentstack-import/src/import/modules/base-class.ts +++ b/packages/contentstack-import/src/import/modules/base-class.ts @@ -18,7 +18,6 @@ import { LabelData } from '@contentstack/management/types/stack/label'; import { WebhookData } from '@contentstack/management/types/stack/webhook'; import { WorkflowData } from '@contentstack/management/types/stack/workflow'; import { RoleData } from '@contentstack/management/types/stack/role'; -import { HttpClient } from '@contentstack/cli-utilities'; import { log } from '../../utils'; import { ImportConfig, ModuleClassParams } from '../../types'; @@ -385,18 +384,16 @@ export default abstract class BaseClass { .then(onSuccess) .catch(onReject); case 'create-taxonomies': - return new HttpClient() - .headers(additionalInfo.headers) - .post(additionalInfo.url, { taxonomy: apiData }) - .then(onSuccess) - .catch(onReject); + return this.stack.taxonomy().create({ taxonomy: apiData }).then(onSuccess).catch(onReject); case 'create-terms': - const url = `${additionalInfo.baseUrl}/${apiData.taxonomy_uid}/terms`; - return new HttpClient() - .headers(additionalInfo.headers) - .post(url, { term: apiData }) - .then(onSuccess) - .catch(onReject); + if (apiData?.taxonomy_uid) { + return this.stack + .taxonomy(apiData.taxonomy_uid) + .terms() + .create({ term: apiData }) + .then(onSuccess) + .catch(onReject); + } default: return Promise.resolve(); } diff --git a/packages/contentstack-import/src/import/modules/taxonomies.ts b/packages/contentstack-import/src/import/modules/taxonomies.ts index b3f3453989..97231f79c5 100644 --- a/packages/contentstack-import/src/import/modules/taxonomies.ts +++ b/packages/contentstack-import/src/import/modules/taxonomies.ts @@ -3,21 +3,11 @@ import pick from 'lodash/pick'; import { join } from 'node:path'; import values from 'lodash/values'; import isEmpty from 'lodash/isEmpty'; -import flatten from 'lodash/flatten'; -import { configHandler } from '@contentstack/cli-utilities'; import BaseClass, { ApiOptions } from './base-class'; import { log, formatError, fsUtil, fileHelper } from '../../utils'; import { ModuleClassParams, TaxonomiesConfig, TermsConfig } from '../../types'; -//NOTE: Temp types need to remove once sdk available -type TaxonomyPayload = { - baseUrl: string; - url: string; - mgToken?: string; - reqPayload: Record; - headers: Record; -}; export default class ImportTaxonomies extends BaseClass { private taxonomiesMapperDirPath: string; @@ -31,7 +21,6 @@ export default class ImportTaxonomies extends BaseClass { private termsConfig: TermsConfig; private termsSuccessPath: string; private termsFailsPath: string; - private taxonomyPayload: TaxonomyPayload; public taxonomiesSuccess: Record = {}; public taxonomiesFailed: Record = {}; public termsSuccess: Record> = {}; @@ -51,16 +40,6 @@ export default class ImportTaxonomies extends BaseClass { this.taxFailsPath = join(this.taxonomiesMapperDirPath, 'fails.json'); this.termsSuccessPath = join(this.termsMapperDirPath, 'success.json'); this.termsFailsPath = join(this.termsMapperDirPath, 'fails.json'); - this.taxonomyPayload = { - baseUrl: '', - url: '', - mgToken: importConfig.management_token, - reqPayload: {}, - headers: { - 'Content-Type': 'application/json', - api_key: importConfig.target_stack, - }, - }; } /** @@ -81,13 +60,6 @@ export default class ImportTaxonomies extends BaseClass { return; } - //NOTE - Temp code for api request - const { cma } = configHandler.get('region') || {}; - this.taxonomyPayload.baseUrl = `${cma}/v3/taxonomies`; - this.taxonomyPayload.url = this.taxonomyPayload.baseUrl; - if (this.taxonomyPayload?.mgToken) this.taxonomyPayload.headers['authorization'] = this.taxonomyPayload.mgToken; - else this.taxonomyPayload.headers['authtoken'] = configHandler.get('authtoken'); - //Step 2 create taxonomies & terms mapper directory await fsUtil.makeDirectory(this.taxonomiesMapperDirPath); await fsUtil.makeDirectory(this.termsMapperDirPath); @@ -119,39 +91,28 @@ export default class ImportTaxonomies extends BaseClass { return; } - const apiContent = values(this.taxonomies) as Record[];; + const apiContent = values(this.taxonomies) as Record[]; this.taxonomyUIDs = keys(this.taxonomies); - const onSuccess = ({ - response: { data, status } = { data: null, status: null }, - apiData: { uid, name } = { uid: null, name: '' }, - }: any) => { - //NOTE - Temp code to handle error thru API. Will remove this once sdk is ready - if ([200, 201, 202].includes(status)) { - const { taxonomy } = data; - this.taxonomiesSuccess[taxonomy.uid] = pick(taxonomy, ['name', 'description']); - log(this.importConfig, `Taxonomy '${name}' imported successfully!`, 'success'); - } else { - let errorMsg:any; - if ([500, 503, 502].includes(status)) errorMsg = data?.message || data; - else errorMsg = data?.error_message; - if (errorMsg === undefined) { - errorMsg = Object.values(data?.errors) && flatten(Object.values(data.errors)); - } - this.taxonomiesFailed[uid] = `Taxonomy '${name}' failed to be import! ${JSON.stringify(errorMsg)}`; - log(this.importConfig, `Taxonomy '${name}' failed to be import! ${JSON.stringify(errorMsg)}`, 'error'); - } + const onSuccess = ({ response }: any) => { + const { uid, name } = response; + this.taxonomiesSuccess[uid] = pick(response, ['name', 'description']); + log(this.importConfig, `Taxonomy '${name}' imported successfully!`, 'success'); }; const onReject = ({ error, apiData }: any) => { const err = error?.message ? JSON.parse(error.message) : error; const { name } = apiData; - if (err?.errors?.name) { - log(this.importConfig, `Taxonomy '${name}' already exists!`, 'info'); + if (err?.errors?.taxonomy) { + if (err?.errorCode === 0) { + log(this.importConfig, `Taxonomy '${name}' already exists!`, 'info'); + } else { + this.taxonomiesFailed[apiData.uid] = apiData; + log(this.importConfig, `Taxonomy '${name}' failed to be import! ${err.errors.taxonomy}`, 'error'); + } } else { this.taxonomiesFailed[apiData.uid] = apiData; log(this.importConfig, `Taxonomy '${name}' failed to be import! ${formatError(error)}`, 'error'); - log(this.importConfig, error, 'error'); } }; @@ -164,8 +125,7 @@ export default class ImportTaxonomies extends BaseClass { reject: onReject, resolve: onSuccess, entity: 'create-taxonomies', - includeParamOnCompletion: true, - additionalInfo: this.taxonomyPayload, + includeParamOnCompletion: true }, concurrencyLimit: this.importConfig.concurrency || this.importConfig.fetchConcurrency || 1, }, @@ -181,12 +141,6 @@ export default class ImportTaxonomies extends BaseClass { */ serializeTaxonomy(apiOptions: ApiOptions): ApiOptions { const { apiData: taxonomy } = apiOptions; - if (this.taxonomiesSuccess.hasOwnProperty(taxonomy.uid)) { - log(this.importConfig, `Taxonomy '${taxonomy.name}' already exists. Skipping it to avoid duplicates!`, 'info'); - apiOptions.entity = undefined; - } else { - apiOptions.apiData = taxonomy; - } apiOptions.apiData = taxonomy; return apiOptions; } @@ -216,39 +170,28 @@ export default class ImportTaxonomies extends BaseClass { return; } - const onSuccess = ({ - response: { data, status } = { data: null, status: null }, - apiData: { uid, name, taxonomy_uid } = { uid: null, name: '', taxonomy_uid: null }, - }: any) => { - //NOTE - Temp code to handle error thru API. Will remove this once sdk is ready - if ([200, 201, 202].includes(status)) { - if (!this.termsSuccess[taxonomy_uid]) this.termsSuccess[taxonomy_uid] = {}; - const { term } = data; - this.termsSuccess[taxonomy_uid][term.uid] = pick(term, ['name']); - log(this.importConfig, `Term '${name}' imported successfully!`, 'success'); - } else { - if (!this.termsFailed[taxonomy_uid]) this.termsFailed[taxonomy_uid] = {}; - let errorMsg; - if ([500, 503, 502].includes(status)) errorMsg = data?.message || data; - else errorMsg = data?.error_message; - if (errorMsg === undefined) { - errorMsg = Object.values(data?.errors) && flatten(Object.values(data.errors)); - } - this.termsFailed[taxonomy_uid][uid] = `Terms '${name}' failed to be import! ${JSON.stringify(errorMsg)}`; - log(this.importConfig, `Terms '${name}' failed to be import! ${JSON.stringify(errorMsg)}`, 'error'); - } + const onSuccess = ({ response, apiData: { taxonomy_uid } = { taxonomy_uid: null } }: any) => { + const { uid, name } = response; + if (!this.termsSuccess[taxonomy_uid]) this.termsSuccess[taxonomy_uid] = {}; + this.termsSuccess[taxonomy_uid][uid] = pick(response, ['name']); + log(this.importConfig, `Term '${name}' imported successfully!`, 'success'); }; const onReject = ({ error, apiData }: any) => { - const { uid, taxonomy_uid, name } = apiData; + const { taxonomy_uid, name } = apiData; if (!this.termsFailed[taxonomy_uid]) this.termsFailed[taxonomy_uid] = {}; const err = error?.message ? JSON.parse(error.message) : error; - if (err?.errors?.name) { - log(this.importConfig, `Term '${name}' already exists!`, 'info'); + + if (err?.errors?.term) { + if (err?.errorCode === 0) { + log(this.importConfig, `Term '${name}' already exists!`, 'info'); + } else { + this.termsFailed[taxonomy_uid][apiData.uid] = apiData; + log(this.importConfig, `Term '${name}' failed to be import! ${err.errors.term}`, 'error'); + } } else { this.termsFailed[taxonomy_uid][apiData.uid] = apiData; log(this.importConfig, `Term '${name}' failed to be import! ${formatError(error)}`, 'error'); - log(this.importConfig, error, 'error'); } }; @@ -270,8 +213,7 @@ export default class ImportTaxonomies extends BaseClass { reject: onReject, resolve: onSuccess, entity: 'create-terms', - includeParamOnCompletion: true, - additionalInfo: this.taxonomyPayload, + includeParamOnCompletion: true }, concurrencyLimit: this.importConfig.concurrency || this.importConfig.fetchConcurrency || 1, }, diff --git a/packages/contentstack-utilities/package.json b/packages/contentstack-utilities/package.json index c9445b92f1..4fc63b60f8 100644 --- a/packages/contentstack-utilities/package.json +++ b/packages/contentstack-utilities/package.json @@ -32,7 +32,7 @@ "author": "contentstack", "license": "MIT", "dependencies": { - "@contentstack/management": "~1.10.2", + "@contentstack/management": "~1.11.0", "@oclif/core": "^2.9.3", "axios": "1.3.4", "chalk": "^4.0.0", diff --git a/packages/contentstack/package.json b/packages/contentstack/package.json index 5a7e6b70c7..d43ad79951 100755 --- a/packages/contentstack/package.json +++ b/packages/contentstack/package.json @@ -38,7 +38,7 @@ "@contentstack/cli-launch": "~1.0.12", "@contentstack/cli-migration": "~1.3.13", "@contentstack/cli-utilities": "~1.5.3", - "@contentstack/management": "~1.10.0", + "@contentstack/management": "~1.11.0", "@oclif/core": "^2.9.3", "@oclif/plugin-help": "^5", "@oclif/plugin-not-found": "^2.4.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 85de711d9a..12e9519d03 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -26,7 +26,7 @@ importers: '@contentstack/cli-launch': ~1.0.12 '@contentstack/cli-migration': ~1.3.13 '@contentstack/cli-utilities': ~1.5.3 - '@contentstack/management': ~1.10.0 + '@contentstack/management': ~1.11.0 '@oclif/core': ^2.9.3 '@oclif/plugin-help': ^5 '@oclif/plugin-not-found': ^2.4.0 @@ -80,7 +80,7 @@ importers: '@contentstack/cli-launch': link:../contentstack-launch '@contentstack/cli-migration': link:../contentstack-migration '@contentstack/cli-utilities': link:../contentstack-utilities - '@contentstack/management': 1.10.2_debug@4.3.4 + '@contentstack/management': 1.11.0_debug@4.3.4 '@oclif/core': 2.15.0_4qcp7qp4jxxdgb4qbxgwox4hwq '@oclif/plugin-help': 5.2.14_4qcp7qp4jxxdgb4qbxgwox4hwq '@oclif/plugin-not-found': 2.4.0_4qcp7qp4jxxdgb4qbxgwox4hwq @@ -710,7 +710,7 @@ importers: specifiers: '@contentstack/cli-command': ~1.2.13 '@contentstack/cli-utilities': ~1.5.3 - '@contentstack/management': ~1.10.2 + '@contentstack/management': ~1.11.0 '@oclif/core': ^2.9.3 '@oclif/test': ^1.2.6 '@types/big-json': ^3.2.0 @@ -750,7 +750,7 @@ importers: dependencies: '@contentstack/cli-command': link:../contentstack-command '@contentstack/cli-utilities': link:../contentstack-utilities - '@contentstack/management': 1.10.2_debug@4.3.4 + '@contentstack/management': 1.11.0_debug@4.3.4 '@oclif/core': 2.9.4_4qcp7qp4jxxdgb4qbxgwox4hwq big-json: 3.2.0 bluebird: 3.7.2 @@ -1017,7 +1017,7 @@ importers: packages/contentstack-utilities: specifiers: - '@contentstack/management': ~1.10.2 + '@contentstack/management': ~1.11.0 '@oclif/core': ^2.9.3 '@oclif/test': ^2.2.10 '@types/chai': ^4.2.18 @@ -1061,7 +1061,7 @@ importers: winston: ^3.7.2 xdg-basedir: ^4.0.0 dependencies: - '@contentstack/management': 1.10.2_debug@4.3.4 + '@contentstack/management': 1.11.0_debug@4.3.4 '@oclif/core': 2.9.4_4qcp7qp4jxxdgb4qbxgwox4hwq axios: 1.3.4_debug@4.3.4 chalk: 4.1.2 @@ -1548,11 +1548,11 @@ packages: uuid: 8.3.2 dev: false - /@contentstack/management/1.10.2_debug@4.3.4: - resolution: {integrity: sha512-jO24EqcCJhOjqdsqw8y3T0SPPAd0DG4BByjUcV0S28W2yoa8aBbcjcbZioRPzRLYKTmZWsAZissl18cIJm5djQ==} + /@contentstack/management/1.11.0_debug@4.3.4: + resolution: {integrity: sha512-tv4At2Q5iGgkzL1MFGil/o36URKZfO6DY/KtpNJFYjmTHitZNv7uotH8OXkOPBMxB4xz58SG58lWB6fNTkLrpw==} engines: {node: '>=8.0.0'} dependencies: - axios: 1.5.0_debug@4.3.4 + axios: 1.5.1_debug@4.3.4 form-data: 3.0.1 lodash: 4.17.21 qs: 6.11.2 @@ -4512,8 +4512,8 @@ packages: - debug dev: false - /axios/1.5.0_debug@4.3.4: - resolution: {integrity: sha512-D4DdjDo5CY50Qms0qGQTTw6Q44jl7zRwY7bthds06pUGfChBCTcQs+N743eFWGEd6pRTMd6A+I87aWyFV5wiZQ==} + /axios/1.5.1_debug@4.3.4: + resolution: {integrity: sha512-Q28iYCWzNHjAm+yEAot5QaAMxhMghWLFVf7rRdwhUI+c2jix2DUXjAHXVi+s1ibs3mjPO/cCgbA++3BjD0vP/A==} dependencies: follow-redirects: 1.15.2_debug@4.3.4 form-data: 4.0.0 From de56c8818d976bb5b2d5c4d32760fdc17dc25d1d Mon Sep 17 00:00:00 2001 From: Aman Kumar Date: Tue, 10 Oct 2023 17:59:18 +0530 Subject: [PATCH 31/45] feat: taxonomies & multiple field support in migration --- .../docs/api-reference.md | 55 ++++++++++++++++--- .../src/config/default-options.js | 2 +- .../src/modules/content-types.js | 16 +++--- .../src/modules/fields.js | 37 ++++++++++++- .../src/utils/constants.js | 2 + 5 files changed, 93 insertions(+), 19 deletions(-) diff --git a/packages/contentstack-migration/docs/api-reference.md b/packages/contentstack-migration/docs/api-reference.md index 6db907e107..366a7f734b 100644 --- a/packages/contentstack-migration/docs/api-reference.md +++ b/packages/contentstack-migration/docs/api-reference.md @@ -53,10 +53,12 @@ Creates content type by passing content type name and options **Example** ```js -module.exports = {migrations} => { - const blog = migrations.createContentType('blog', { - title: 'blog' - }) +module.exports = ({migration}) => { + const blog = migration + .createContentType(blogUID) + .title(blogTitle) + .description('blog 1') + blog.createField('title').display_name('Title').data_type('text').mandatory(true); } ``` @@ -98,10 +100,8 @@ Edits content type by passing content type name and options **Example** ```js -module.exports = {migrations} => { - const blog = migrations.editContentType('blog', { - title: 'blog' - }); +module.exports = ({migration}) => { + const blog = migration.editContentType('blog'); blog.description('Changed description'); } ``` @@ -177,6 +177,8 @@ Chained function takes boolean value for force while deleting content type * [.unique(value)](#Field+unique) ⇒ [Field](#Field) * [.reference_to(value)](#Field+reference_to) ⇒ [Field](#Field) * [.ref_multiple(value)](#Field+ref_multiple) ⇒ [Field](#Field) + * [.taxonomies(value)](#Field+taxonomies) ⇒ [Field](#Field) + * [.multiple(value)](#Field+multiple) ⇒ [Field](#Field) * [.ref_multipleContentType(value)](#Field+ref_multipleContentType) ⇒ [Field](#Field) * [.getTaskDefinition()](#Field+getTaskDefinition) ⇒ [Task](#Task) @@ -203,11 +205,24 @@ Creates a field with provided uid. module.exports =({ migration })=> { const blog = migration.editContentType('blog'); - blog.createField('author'); + blog.createField('author') .display_name('Author') .data_type('text') .mandatory(false); }; + +Create a taxonomy field + + module.exports =({ migration })=> { + const blog = migration.editContentType('blog'); + + blog.createField('taxonomies') + .display_name('Taxonomy1') + .data_type('taxonomy') + .taxonomies([{ "taxonomy_uid": "test_taxonomy1", "max_terms": 2, "mandatory": false}]) + .multiple(true) + .mandatory(false); +}; ``` @@ -356,6 +371,28 @@ module.exports = ({migration}) => { | --- | --- | --- | | value | string | set true if accepts multiple entries as reference | + + +### field.taxonomies(value) ⇒ [Field](#Field) +The 'taxonomies' property should contain at least one taxonomy object + +**Kind**: instance method of [Field](#Field) +**Returns**: [Field](#Field) - current instance of field object to chain further methods. + +| Param | Type | Description | +| --- | --- | --- | +| value | string \| Array.<string> | list of taxonomies. | + + + +### field.multiple(value) ⇒ [Field](#Field) +**Kind**: instance method of [Field](#Field) +**Returns**: [Field](#Field) - current instance of field object to chain further methods. + +| Param | Type | Description | +| --- | --- | --- | +| value | boolean | set true if field is multiple | + ### field.ref\_multipleContentType(value) ⇒ [Field](#Field) diff --git a/packages/contentstack-migration/src/config/default-options.js b/packages/contentstack-migration/src/config/default-options.js index 90d34e835f..72dc159fe8 100644 --- a/packages/contentstack-migration/src/config/default-options.js +++ b/packages/contentstack-migration/src/config/default-options.js @@ -2,6 +2,6 @@ 'use strict'; module.exports = { - is_page: true, + is_page: false, singleton: false, }; diff --git a/packages/contentstack-migration/src/modules/content-types.js b/packages/contentstack-migration/src/modules/content-types.js index b47b0598c3..c865a2b576 100644 --- a/packages/contentstack-migration/src/modules/content-types.js +++ b/packages/contentstack-migration/src/modules/content-types.js @@ -38,10 +38,12 @@ class ContentType extends Base { * @param {Object} opts Optional: Content type fields definition * @returns {Field} instance of Field * @example - * module.exports = {migrations} => { - * const blog = migrations.createContentType('blog', { - * title: 'blog' - * }) + * module.exports = ({migration}) => { + * const blog = migration + * .createContentType(blogUID) + * .title(blogTitle) + * .description('blog 1') + * blog.createField('title').display_name('Title').data_type('text').mandatory(true); * } */ createContentType(id, opts = {}) { @@ -114,10 +116,8 @@ class ContentType extends Base { * @param {Object} opts Optional: Content type fields definition * @returns {Field} instance of Field * @example - * module.exports = {migrations} => { - * const blog = migrations.editContentType('blog', { - * title: 'blog' - * }); + * module.exports = ({migration}) => { + * const blog = migration.editContentType('blog'); * blog.description('Changed description'); * } */ diff --git a/packages/contentstack-migration/src/modules/fields.js b/packages/contentstack-migration/src/modules/fields.js index 9ace6c74d9..0b6e69f9e6 100644 --- a/packages/contentstack-migration/src/modules/fields.js +++ b/packages/contentstack-migration/src/modules/fields.js @@ -16,6 +16,8 @@ const { field_metadata, reference_to, actions: _actions, + taxonomies, + multiple, } = constants; // Base class @@ -52,11 +54,24 @@ class Field extends Base { * module.exports =({ migration })=> { * const blog = migration.editContentType('blog'); * - * blog.createField('author'); + * blog.createField('author') * .display_name('Author') * .data_type('text') * .mandatory(false); * }; + * + * Create a taxonomy field + * + * module.exports =({ migration })=> { + * const blog = migration.editContentType('blog'); + * + * blog.createField('taxonomies') + * .display_name('Taxonomy1') + * .data_type('taxonomy') + * .taxonomies([{ "taxonomy_uid": "test_taxonomy1", "max_terms": 2, "mandatory": false}]) + * .multiple(true) + * .mandatory(false); + * }; */ createField(field, opts) { this.updateContentTypeSchema(field); @@ -227,6 +242,26 @@ class Field extends Base { return this; } + /** + * The 'taxonomies' property should contain at least one taxonomy object + * @param {string | string[]} value list of taxonomies. + * @returns {Field} current instance of field object to chain further methods. + */ + taxonomies(value) { + this.buildSchema(taxonomies, this.field, value); + return this; + } + + /** + * + * @param {boolean} value set true if field is multiple + * @returns {Field} current instance of field object to chain further methods. + */ + multiple(value) { + this.buildSchema(multiple, this.field, value); + return this; + } + /** * * @param {boolean} value set true if refer to multiple content types diff --git a/packages/contentstack-migration/src/utils/constants.js b/packages/contentstack-migration/src/utils/constants.js index 1ebc59fa80..0705c0117c 100644 --- a/packages/contentstack-migration/src/utils/constants.js +++ b/packages/contentstack-migration/src/utils/constants.js @@ -21,6 +21,8 @@ exports.unique = 'unique'; exports.display_name = 'display_name'; exports.reference_to = 'reference_to'; exports.field_metadata = 'field_metadata'; +exports.taxonomies = 'taxonomies'; +exports.multiple = 'multiple'; exports.actions = { CUSTOM_TASK: 'CUSTOM_TASK', From 85371cb01278ed5f84519a7053c092e1fba0dfc8 Mon Sep 17 00:00:00 2001 From: Aman Kumar Date: Tue, 10 Oct 2023 18:03:43 +0530 Subject: [PATCH 32/45] fix: create content type example --- packages/contentstack-migration/docs/api-reference.md | 4 ++-- packages/contentstack-migration/src/modules/content-types.js | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/packages/contentstack-migration/docs/api-reference.md b/packages/contentstack-migration/docs/api-reference.md index 366a7f734b..da8e9a473f 100644 --- a/packages/contentstack-migration/docs/api-reference.md +++ b/packages/contentstack-migration/docs/api-reference.md @@ -55,8 +55,8 @@ Creates content type by passing content type name and options ```js module.exports = ({migration}) => { const blog = migration - .createContentType(blogUID) - .title(blogTitle) + .createContentType('blog') + .title('blog title') .description('blog 1') blog.createField('title').display_name('Title').data_type('text').mandatory(true); } diff --git a/packages/contentstack-migration/src/modules/content-types.js b/packages/contentstack-migration/src/modules/content-types.js index c865a2b576..e597ece69a 100644 --- a/packages/contentstack-migration/src/modules/content-types.js +++ b/packages/contentstack-migration/src/modules/content-types.js @@ -40,8 +40,8 @@ class ContentType extends Base { * @example * module.exports = ({migration}) => { * const blog = migration - * .createContentType(blogUID) - * .title(blogTitle) + * .createContentType('blog') + * .title('blog title') * .description('blog 1') * blog.createField('title').display_name('Title').data_type('text').mandatory(true); * } From 31794326ddac5147f64a129402c9dc44cae7e5b2 Mon Sep 17 00:00:00 2001 From: Aman Kumar Date: Tue, 10 Oct 2023 19:11:57 +0530 Subject: [PATCH 33/45] Revert "feat: replace taxonomy & term http call with sdk" This reverts commit e0e3eb1067b3181a5a82b65089b2415113f3eabe. --- package-lock.json | 40 ++--- .../src/commands/cm/export-to-csv.js | 19 +- .../src/util/index.js | 130 ++++++-------- .../src/export/modules/taxonomies.ts | 163 +++++++++++------- packages/contentstack-import/package.json | 2 +- .../src/import/modules/base-class.ts | 21 ++- .../src/import/modules/taxonomies.ts | 114 +++++++++--- packages/contentstack-utilities/package.json | 2 +- packages/contentstack/package.json | 2 +- pnpm-lock.yaml | 22 +-- 10 files changed, 296 insertions(+), 219 deletions(-) diff --git a/package-lock.json b/package-lock.json index 75c29afcc1..5dad81b9fe 100644 --- a/package-lock.json +++ b/package-lock.json @@ -811,11 +811,11 @@ } }, "node_modules/@contentstack/management": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/@contentstack/management/-/management-1.11.0.tgz", - "integrity": "sha512-tv4At2Q5iGgkzL1MFGil/o36URKZfO6DY/KtpNJFYjmTHitZNv7uotH8OXkOPBMxB4xz58SG58lWB6fNTkLrpw==", + "version": "1.10.2", + "resolved": "https://registry.npmjs.org/@contentstack/management/-/management-1.10.2.tgz", + "integrity": "sha512-jO24EqcCJhOjqdsqw8y3T0SPPAd0DG4BByjUcV0S28W2yoa8aBbcjcbZioRPzRLYKTmZWsAZissl18cIJm5djQ==", "dependencies": { - "axios": "^1.5.1", + "axios": "^1.4.0", "form-data": "^3.0.1", "lodash": "^4.17.21", "qs": "^6.11.2" @@ -825,9 +825,9 @@ } }, "node_modules/@contentstack/management/node_modules/axios": { - "version": "1.5.1", - "resolved": "https://registry.npmjs.org/axios/-/axios-1.5.1.tgz", - "integrity": "sha512-Q28iYCWzNHjAm+yEAot5QaAMxhMghWLFVf7rRdwhUI+c2jix2DUXjAHXVi+s1ibs3mjPO/cCgbA++3BjD0vP/A==", + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.5.0.tgz", + "integrity": "sha512-D4DdjDo5CY50Qms0qGQTTw6Q44jl7zRwY7bthds06pUGfChBCTcQs+N743eFWGEd6pRTMd6A+I87aWyFV5wiZQ==", "dependencies": { "follow-redirects": "^1.15.0", "form-data": "^4.0.0", @@ -22760,7 +22760,7 @@ "@contentstack/cli-launch": "~1.0.12", "@contentstack/cli-migration": "~1.3.13", "@contentstack/cli-utilities": "~1.5.3", - "@contentstack/management": "~1.11.0", + "@contentstack/management": "~1.10.0", "@oclif/core": "^2.9.3", "@oclif/plugin-help": "^5", "@oclif/plugin-not-found": "^2.4.0", @@ -24587,7 +24587,7 @@ "dependencies": { "@contentstack/cli-command": "~1.2.13", "@contentstack/cli-utilities": "~1.5.3", - "@contentstack/management": "~1.11.0", + "@contentstack/management": "~1.10.2", "@oclif/core": "^2.9.3", "big-json": "^3.2.0", "bluebird": "^3.7.2", @@ -25386,7 +25386,7 @@ "version": "1.5.3", "license": "MIT", "dependencies": { - "@contentstack/management": "~1.11.0", + "@contentstack/management": "~1.10.2", "@oclif/core": "^2.9.3", "axios": "1.3.4", "chalk": "^4.0.0", @@ -26999,7 +26999,7 @@ "@contentstack/cli-launch": "~1.0.12", "@contentstack/cli-migration": "~1.3.13", "@contentstack/cli-utilities": "~1.5.3", - "@contentstack/management": "~1.11.0", + "@contentstack/management": "~1.10.0", "@oclif/core": "^2.9.3", "@oclif/plugin-help": "^5", "@oclif/plugin-not-found": "^2.4.0", @@ -28122,7 +28122,7 @@ "requires": { "@contentstack/cli-command": "~1.2.13", "@contentstack/cli-utilities": "~1.5.3", - "@contentstack/management": "~1.11.0", + "@contentstack/management": "~1.10.2", "@oclif/core": "^2.9.3", "@oclif/test": "^1.2.6", "@types/big-json": "^3.2.0", @@ -29009,7 +29009,7 @@ "@contentstack/cli-utilities": { "version": "file:packages/contentstack-utilities", "requires": { - "@contentstack/management": "~1.11.0", + "@contentstack/management": "~1.10.2", "@oclif/core": "^2.9.3", "@oclif/test": "^2.2.10", "@types/chai": "^4.2.18", @@ -29863,20 +29863,20 @@ } }, "@contentstack/management": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/@contentstack/management/-/management-1.11.0.tgz", - "integrity": "sha512-tv4At2Q5iGgkzL1MFGil/o36URKZfO6DY/KtpNJFYjmTHitZNv7uotH8OXkOPBMxB4xz58SG58lWB6fNTkLrpw==", + "version": "1.10.2", + "resolved": "https://registry.npmjs.org/@contentstack/management/-/management-1.10.2.tgz", + "integrity": "sha512-jO24EqcCJhOjqdsqw8y3T0SPPAd0DG4BByjUcV0S28W2yoa8aBbcjcbZioRPzRLYKTmZWsAZissl18cIJm5djQ==", "requires": { - "axios": "^1.5.1", + "axios": "^1.4.0", "form-data": "^3.0.1", "lodash": "^4.17.21", "qs": "^6.11.2" }, "dependencies": { "axios": { - "version": "1.5.1", - "resolved": "https://registry.npmjs.org/axios/-/axios-1.5.1.tgz", - "integrity": "sha512-Q28iYCWzNHjAm+yEAot5QaAMxhMghWLFVf7rRdwhUI+c2jix2DUXjAHXVi+s1ibs3mjPO/cCgbA++3BjD0vP/A==", + "version": "1.5.0", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.5.0.tgz", + "integrity": "sha512-D4DdjDo5CY50Qms0qGQTTw6Q44jl7zRwY7bthds06pUGfChBCTcQs+N743eFWGEd6pRTMd6A+I87aWyFV5wiZQ==", "requires": { "follow-redirects": "^1.15.0", "form-data": "^4.0.0", diff --git a/packages/contentstack-export-to-csv/src/commands/cm/export-to-csv.js b/packages/contentstack-export-to-csv/src/commands/cm/export-to-csv.js index 520f53d881..dde0f49ae1 100644 --- a/packages/contentstack-export-to-csv/src/commands/cm/export-to-csv.js +++ b/packages/contentstack-export-to-csv/src/commands/cm/export-to-csv.js @@ -214,6 +214,7 @@ class ExportToCsvCommand extends Command { case 'taxonomies': { let stack; let stackAPIClient; + let taxUID; if (managementTokenAlias) { const { stackDetails, apiClient } = await this.getAliasDetails(managementTokenAlias, stackName); managementAPIClient = apiClient; @@ -223,7 +224,7 @@ class ExportToCsvCommand extends Command { } stackAPIClient = this.getStackClient(managementAPIClient, stack); - await this.createTaxonomyAndTermCsvFile(stackAPIClient, stackName, stack, taxonomyUID); + await this.createTaxonomyAndTermCsvFile(stackName, stack, taxonomyUID); break; } } @@ -241,8 +242,8 @@ class ExportToCsvCommand extends Command { getStackClient(managementAPIClient, stack) { const stackInit = { api_key: stack.apiKey, + branch_uid: stack.branch_uid, }; - if(stack?.branch_uid) stackInit['branch_uid'] = stack.branch_uid; if (stack.token) { return managementAPIClient.stack({ ...stackInit, @@ -359,18 +360,20 @@ class ExportToCsvCommand extends Command { * @param {object} stack * @param {string} taxUID */ - async createTaxonomyAndTermCsvFile(stackAPIClient, stackName, stack, taxUID) { + async createTaxonomyAndTermCsvFile(stackName, stack, taxUID) { + const { cma } = configHandler.get('region') || {}; const payload = { - stackAPIClient, - type: '' + baseUrl: `${cma}/v3/taxonomies`, + apiKey: stack.apiKey, + mgToken: stack?.token, }; //check whether the taxonomy is valid or not let taxonomies = []; if (taxUID) { - payload['taxonomyUID'] = taxUID; - const taxonomy = await util.getTaxonomy(payload); + const taxonomy = await util.getTaxonomy(payload, taxUID); taxonomies.push(taxonomy); } else { + payload['url'] = payload.baseUrl; taxonomies = await util.getAllTaxonomies(payload); } @@ -386,7 +389,7 @@ class ExportToCsvCommand extends Command { const taxonomy = taxonomies[index]; const taxonomyUID = taxonomy?.uid; if (taxonomyUID) { - payload['taxonomyUID'] = taxonomyUID; + payload['url'] = `${payload.baseUrl}/${taxonomyUID}/terms`; const terms = await util.getAllTermsOfTaxonomy(payload); const formattedTermsData = util.formatTermsOfTaxonomyData(terms, taxonomyUID); const taxonomyName = taxonomy?.name ? taxonomy.name : ''; diff --git a/packages/contentstack-export-to-csv/src/util/index.js b/packages/contentstack-export-to-csv/src/util/index.js index 42912fd80f..c9132be470 100644 --- a/packages/contentstack-export-to-csv/src/util/index.js +++ b/packages/contentstack-export-to-csv/src/util/index.js @@ -9,14 +9,7 @@ const debug = require('debug')('export-to-csv'); const checkboxPlus = require('inquirer-checkbox-plus-prompt'); const config = require('./config.js'); -const { - cliux, - configHandler, - HttpClient, - messageHandler, - managementSDKClient, - ContentstackClient, -} = require('@contentstack/cli-utilities'); +const { cliux, configHandler, HttpClient, messageHandler } = require('@contentstack/cli-utilities'); const directory = './data'; const delimeter = os.platform() === 'win32' ? '\\' : '/'; @@ -694,43 +687,41 @@ function wait(time) { * fetch all taxonomies in the provided stack * @param {object} payload * @param {number} skip + * @param {number} limit * @param {array} taxonomies * @returns */ -async function getAllTaxonomies(payload, skip=0, taxonomies = []) { - payload['type'] = 'taxonomies'; - const response = await taxonomySDKHandler(payload); +async function getAllTaxonomies(payload, skip = 0, limit = 100, taxonomies = []) { + const response = await apiRequestHandler(payload, skip, limit); if (response) { skip += config.limit || 100; - //TODO - replace reponse.taxonomies with items taxonomies = [...taxonomies, ...response.taxonomies]; if (skip >= response?.count) { return taxonomies; } else { - return getAllTaxonomies(payload, skip, taxonomies); + return getAllTaxonomies(payload, skip, limit, taxonomies); } } return taxonomies; } /** - * fetch taxonomy related terms + * fetch terms of related taxonomy * @param {object} payload * @param {number} skip * @param {number} limit * @param {array} terms * @returns */ -async function getAllTermsOfTaxonomy(payload, skip=0, terms = []) { - payload['type'] = 'terms'; - const {items, count} = await taxonomySDKHandler(payload); - if (items) { +async function getAllTermsOfTaxonomy(payload, skip = 0, limit = 100, terms = []) { + const response = await apiRequestHandler(payload, skip, limit); + if (response) { skip += config.limit || 100; - terms = [...terms, ...items]; - if (skip >= count) { + terms = [...terms, ...response.terms]; + if (skip >= response?.count) { return terms; } else { - return getAllTermsOfTaxonomy(payload, skip, terms); + return getAllTermsOfTaxonomy(payload, skip, limit, terms); } } return terms; @@ -742,56 +733,50 @@ async function getAllTermsOfTaxonomy(payload, skip=0, terms = []) { * @param {string} taxonomyUID * @returns */ -async function getTaxonomy(payload) { - payload['type'] = 'taxonomy'; - const resp = await taxonomySDKHandler(payload); - return resp; -} - -/** - * taxonomy & term sdk handler - * @async - * @method - * @param payload - * @param skip - * @param limit - * @returns {*} Promise - */ -async function taxonomySDKHandler(payload, skip) { - const { stackAPIClient,taxonomyUID, type } = payload; - - const queryParams = { include_count: true }; - queryParams['limit'] = config.limit || 100; - if (skip >= 0) queryParams['skip'] = skip; - - switch (type) { - case 'taxonomies': - //TODO - replace count with find - return await stackAPIClient - .taxonomy() - .query(queryParams) - .count() - .then((data) => data) - .catch((err) => handleErrorMsg(err)); - case 'taxonomy': - return await stackAPIClient - .taxonomy(taxonomyUID) - .fetch() - .then((data) => data) - .catch((err) => handleErrorMsg(err)); - case 'terms': - console.log("taxonomyUID---",taxonomyUID) - queryParams['depth'] = 0; - return await stackAPIClient - .taxonomy(taxonomyUID) - .terms() - .query(queryParams) - .find() - .then((data) => data) - .catch((err) => handleErrorMsg(err)); - default: - handleErrorMsg({ errorMessage: 'Invalid module!' }); - } +async function getTaxonomy(payload, taxonomyUID) { + payload['url'] = `${payload.baseUrl}/${taxonomyUID}`; + const resp = await apiRequestHandler(payload); + return resp?.taxonomy || ''; +} + +async function apiRequestHandler(payload, skip, limit) { + const headers = { + api_key: payload.apiKey, + 'Content-Type': 'application/json', + }; + + if (payload?.mgToken) headers['authorization'] = payload.mgToken; + else headers['authToken'] = configHandler.get('authtoken'); + + const params = { + include_count: true, + skip: 0, + limit: 30, + }; + + if (skip >= 0) params['skip'] = skip; + if (limit >= 0) params['limit'] = limit; + + return await new HttpClient() + .headers(headers) + .queryParams(params) + .get(payload.url) + .then((res) => { + //NOTE - temporary code for handling api errors response + const { status, data } = res; + if ([200, 201, 202].includes(status)) return data; + else { + let errorMsg; + if ([500, 503, 502].includes(status)) errorMsg = data?.message || data; + else errorMsg = data?.error_message; + if (errorMsg === undefined) { + errorMsg = Object.values(data?.errors) && flat(Object.values(data.errors)); + } + cliux.print(`Error: ${errorMsg}`, { color: 'red' }); + process.exit(1); + } + }) + .catch((err) => handleErrorMsg(err)); } /** @@ -836,8 +821,7 @@ function handleErrorMsg(err) { if (err?.errorMessage) { cliux.print(`Error: ${err.errorMessage}`, { color: 'red' }); } else if (err?.message) { - const errorMsg = err?.errors?.taxonomy || err?.errors?.term || err?.message; - cliux.print(`Error: ${errorMsg}`, { color: 'red' }); + cliux.print(`Error: ${err.message}`, { color: 'red' }); } else { console.log(err); cliux.print(`Error: ${messageHandler.parse('CLI_EXPORT_CSV_API_FAILED')}`, { color: 'red' }); diff --git a/packages/contentstack-export/src/export/modules/taxonomies.ts b/packages/contentstack-export/src/export/modules/taxonomies.ts index dc0de7378d..e7d540911c 100644 --- a/packages/contentstack-export/src/export/modules/taxonomies.ts +++ b/packages/contentstack-export/src/export/modules/taxonomies.ts @@ -1,23 +1,28 @@ import omit from 'lodash/omit'; import keys from 'lodash/keys'; import isEmpty from 'lodash/isEmpty'; +import flatten from 'lodash/flatten'; import { resolve as pResolve } from 'node:path'; +import { cliux, configHandler, HttpClient } from '@contentstack/cli-utilities'; import BaseClass from './base-class'; import { log, fsUtil } from '../../utils'; import { TaxonomiesConfig, TermsConfig, ModuleClassParams } from '../../types'; +//NOTE: Temp types need to remove once sdk available +type TaxonomyPayload = { + baseUrl: string; + url: string; + mgToken: string; + apiKey: string; +}; + export default class ExportTaxonomies extends BaseClass { private taxonomies: Record>; private terms: Record>; private taxonomiesConfig: TaxonomiesConfig; + private taxonomyPayload: TaxonomyPayload; private termsConfig: TermsConfig; - private qs: { - include_count: boolean; - skip: number; - asc: string; - depth?: number; - }; public taxonomiesFolderPath: string; public termsFolderPath: string; @@ -27,7 +32,12 @@ export default class ExportTaxonomies extends BaseClass { this.terms = {}; this.taxonomiesConfig = exportConfig.modules.taxonomies; this.termsConfig = exportConfig.modules.terms; - this.qs = { include_count: true, skip: 0, asc: 'created_at' }; + this.taxonomyPayload = { + baseUrl: '', + url: '', + mgToken: exportConfig.management_token, + apiKey: exportConfig.source_stack, + }; } async start(): Promise { @@ -43,8 +53,12 @@ export default class ExportTaxonomies extends BaseClass { this.termsFolderPath = pResolve(this.taxonomiesFolderPath, this.termsConfig.dirName); await fsUtil.makeDirectory(this.termsFolderPath); + const { cma } = configHandler.get('region') || {}; + this.taxonomyPayload.baseUrl = `${cma}/v3/taxonomies`; + this.taxonomyPayload.url = this.taxonomyPayload.baseUrl; + //fetch all taxonomies and write into taxonomies folder - await this.getAllTaxonomies(); + await this.getAllTaxonomies(this.taxonomyPayload); if (this.taxonomies === undefined || isEmpty(this.taxonomies)) { log(this.exportConfig, 'No taxonomies found!', 'info'); return; @@ -59,34 +73,22 @@ export default class ExportTaxonomies extends BaseClass { /** * fetch all taxonomies in the provided stack + * @param {TaxonomyPayload} payload * @param {number} skip - * @returns {Promise} + * @returns */ - async getAllTaxonomies(skip = 0): Promise { - if (skip) { - this.qs.skip = skip; + async getAllTaxonomies(payload: TaxonomyPayload, skip = 0): Promise { + const response = await this.apiRequestHandler(payload, skip); + if (response?.taxonomies) { + skip += this.taxonomiesConfig.limit || 100; + this.sanitizeTaxonomiesAttribs(response.taxonomies); + if (skip >= response?.count) { + return; + } else { + return await this.getAllTaxonomies(payload, skip); + } } - //TODO - replace count with find - await this.stack - .taxonomy() - .query(this.qs) - .count() - .then(async (data: any) => { - const { taxonomies, count } = data; - const taxonomiesCount = count !== undefined ? count : taxonomies?.length; - - if (taxonomies?.length) { - this.sanitizeTaxonomiesAttribs(taxonomies); - skip += this.taxonomiesConfig.limit || 100; - if (skip >= taxonomiesCount) { - return; - } - return await this.getAllTaxonomies(skip); - } - }) - .catch((error: any) => { - this.handleErrorMsg(error); - }); + return; } /** @@ -105,16 +107,15 @@ export default class ExportTaxonomies extends BaseClass { /** * fetch all terms of respective taxonomy and write it into -terms file - * @returns {Promise} */ async getAllTerms() { const taxonomiesUID = keys(this.taxonomies) || []; - this.qs.depth = 0; - for (let index = 0; index < taxonomiesUID?.length; index++) { const taxonomyUID = taxonomiesUID[index]; + this.taxonomyPayload.url = `${this.taxonomyPayload.baseUrl}/${taxonomyUID}/terms`; this.terms = {}; - await this.fetchTermsOfTaxonomy(taxonomyUID); + await this.fetchTermsOfTaxonomy(this.taxonomyPayload); + if (this.terms === undefined || isEmpty(this.terms)) { log(this.exportConfig, `No terms found for taxonomy - '${taxonomyUID}'!`, 'info'); } else { @@ -127,36 +128,22 @@ export default class ExportTaxonomies extends BaseClass { /** * fetch all terms of the provided taxonomy uid - * @async - * @param {string} taxonomyUID + * @param {TaxonomyPayload} payload * @param {number} skip - * @returns {Promise} + * @returns */ - async fetchTermsOfTaxonomy(taxonomyUID: string, skip = 0): Promise { - if (skip) { - this.qs.skip = skip; + async fetchTermsOfTaxonomy(payload: TaxonomyPayload, skip = 0): Promise { + const response = await this.apiRequestHandler(payload, skip); + if (response?.terms) { + skip += this.termsConfig.limit || 100; + this.sanitizeTermsAttribs(response.terms); + if (skip >= response?.count) { + return; + } else { + return await this.fetchTermsOfTaxonomy(payload, skip); + } } - await this.stack - .taxonomy(taxonomyUID) - .terms() - .query(this.qs) - .find() - .then(async (data: any) => { - const { items, count } = data; - const termsCount = count !== undefined ? count : items?.length; - - if (items?.length) { - this.sanitizeTermsAttribs(items); - skip += this.taxonomiesConfig.limit || 100; - if (skip >= termsCount) { - return; - } - return await this.fetchTermsOfTaxonomy(taxonomyUID, skip); - } - }) - .catch((error: any) => { - this.handleErrorMsg(error); - }); + return; } /** @@ -171,12 +158,54 @@ export default class ExportTaxonomies extends BaseClass { } } + //NOTE: Temp code need to remove once sdk available + async apiRequestHandler(payload: TaxonomyPayload, skip: number) { + const headers: any = { + api_key: payload.apiKey, + 'Content-Type': 'application/json', + }; + + if (payload?.mgToken) headers['authorization'] = payload.mgToken; + else headers['authToken'] = configHandler.get('authtoken'); + + const params = { + include_count: true, + skip: 0, + limit: this.taxonomiesConfig.limit || 100, + depth: 0 // include all the terms if set to 0 + }; + + if (skip >= 0) params['skip'] = skip; + + return await new HttpClient() + .headers(headers) + .queryParams(params) + .get(payload.url) + .then((res: any) => { + //NOTE - temporary code for handling api errors response + const { status, data } = res; + if ([200, 201, 202].includes(status)) return data; + else { + let errorMsg; + if ([500, 503, 502].includes(status)) errorMsg = data?.message || data; + else errorMsg = data?.error_message; + if (errorMsg === undefined) { + errorMsg = Object.values(data?.errors) && flatten(Object.values(data.errors)); + } + cliux.print(`Error: ${errorMsg}`, { color: 'red' }); + } + }) + .catch((err: any) => this.handleErrorMsg(err)); + } + + //NOTE: Temp code need to remove once sdk available handleErrorMsg(err: any) { if (err?.errorMessage) { - log(this.exportConfig, `Failed to export. ${err.errorMessage}`, 'error'); + cliux.print(`Error: ${err.errorMessage}`, { color: 'red' }); } else if (err?.message) { - const errorMsg = err?.errors?.taxonomy || err?.errors?.term || err?.message; - log(this.exportConfig, `Failed to export. ${errorMsg}`, 'error'); + cliux.print(`Error: ${err.message}`, { color: 'red' }); + } else { + cliux.print(`Error: Something went wrong! Please try again.`, { color: 'red' }); } } } diff --git a/packages/contentstack-import/package.json b/packages/contentstack-import/package.json index 7facb1d483..599d2ea456 100644 --- a/packages/contentstack-import/package.json +++ b/packages/contentstack-import/package.json @@ -7,7 +7,7 @@ "dependencies": { "@contentstack/cli-command": "~1.2.13", "@contentstack/cli-utilities": "~1.5.3", - "@contentstack/management": "~1.11.0", + "@contentstack/management": "~1.10.2", "@oclif/core": "^2.9.3", "big-json": "^3.2.0", "bluebird": "^3.7.2", diff --git a/packages/contentstack-import/src/import/modules/base-class.ts b/packages/contentstack-import/src/import/modules/base-class.ts index 7a758fb8a6..e47f8790ab 100644 --- a/packages/contentstack-import/src/import/modules/base-class.ts +++ b/packages/contentstack-import/src/import/modules/base-class.ts @@ -18,6 +18,7 @@ import { LabelData } from '@contentstack/management/types/stack/label'; import { WebhookData } from '@contentstack/management/types/stack/webhook'; import { WorkflowData } from '@contentstack/management/types/stack/workflow'; import { RoleData } from '@contentstack/management/types/stack/role'; +import { HttpClient } from '@contentstack/cli-utilities'; import { log } from '../../utils'; import { ImportConfig, ModuleClassParams } from '../../types'; @@ -384,16 +385,18 @@ export default abstract class BaseClass { .then(onSuccess) .catch(onReject); case 'create-taxonomies': - return this.stack.taxonomy().create({ taxonomy: apiData }).then(onSuccess).catch(onReject); + return new HttpClient() + .headers(additionalInfo.headers) + .post(additionalInfo.url, { taxonomy: apiData }) + .then(onSuccess) + .catch(onReject); case 'create-terms': - if (apiData?.taxonomy_uid) { - return this.stack - .taxonomy(apiData.taxonomy_uid) - .terms() - .create({ term: apiData }) - .then(onSuccess) - .catch(onReject); - } + const url = `${additionalInfo.baseUrl}/${apiData.taxonomy_uid}/terms`; + return new HttpClient() + .headers(additionalInfo.headers) + .post(url, { term: apiData }) + .then(onSuccess) + .catch(onReject); default: return Promise.resolve(); } diff --git a/packages/contentstack-import/src/import/modules/taxonomies.ts b/packages/contentstack-import/src/import/modules/taxonomies.ts index 97231f79c5..b3f3453989 100644 --- a/packages/contentstack-import/src/import/modules/taxonomies.ts +++ b/packages/contentstack-import/src/import/modules/taxonomies.ts @@ -3,11 +3,21 @@ import pick from 'lodash/pick'; import { join } from 'node:path'; import values from 'lodash/values'; import isEmpty from 'lodash/isEmpty'; +import flatten from 'lodash/flatten'; +import { configHandler } from '@contentstack/cli-utilities'; import BaseClass, { ApiOptions } from './base-class'; import { log, formatError, fsUtil, fileHelper } from '../../utils'; import { ModuleClassParams, TaxonomiesConfig, TermsConfig } from '../../types'; +//NOTE: Temp types need to remove once sdk available +type TaxonomyPayload = { + baseUrl: string; + url: string; + mgToken?: string; + reqPayload: Record; + headers: Record; +}; export default class ImportTaxonomies extends BaseClass { private taxonomiesMapperDirPath: string; @@ -21,6 +31,7 @@ export default class ImportTaxonomies extends BaseClass { private termsConfig: TermsConfig; private termsSuccessPath: string; private termsFailsPath: string; + private taxonomyPayload: TaxonomyPayload; public taxonomiesSuccess: Record = {}; public taxonomiesFailed: Record = {}; public termsSuccess: Record> = {}; @@ -40,6 +51,16 @@ export default class ImportTaxonomies extends BaseClass { this.taxFailsPath = join(this.taxonomiesMapperDirPath, 'fails.json'); this.termsSuccessPath = join(this.termsMapperDirPath, 'success.json'); this.termsFailsPath = join(this.termsMapperDirPath, 'fails.json'); + this.taxonomyPayload = { + baseUrl: '', + url: '', + mgToken: importConfig.management_token, + reqPayload: {}, + headers: { + 'Content-Type': 'application/json', + api_key: importConfig.target_stack, + }, + }; } /** @@ -60,6 +81,13 @@ export default class ImportTaxonomies extends BaseClass { return; } + //NOTE - Temp code for api request + const { cma } = configHandler.get('region') || {}; + this.taxonomyPayload.baseUrl = `${cma}/v3/taxonomies`; + this.taxonomyPayload.url = this.taxonomyPayload.baseUrl; + if (this.taxonomyPayload?.mgToken) this.taxonomyPayload.headers['authorization'] = this.taxonomyPayload.mgToken; + else this.taxonomyPayload.headers['authtoken'] = configHandler.get('authtoken'); + //Step 2 create taxonomies & terms mapper directory await fsUtil.makeDirectory(this.taxonomiesMapperDirPath); await fsUtil.makeDirectory(this.termsMapperDirPath); @@ -91,28 +119,39 @@ export default class ImportTaxonomies extends BaseClass { return; } - const apiContent = values(this.taxonomies) as Record[]; + const apiContent = values(this.taxonomies) as Record[];; this.taxonomyUIDs = keys(this.taxonomies); - const onSuccess = ({ response }: any) => { - const { uid, name } = response; - this.taxonomiesSuccess[uid] = pick(response, ['name', 'description']); - log(this.importConfig, `Taxonomy '${name}' imported successfully!`, 'success'); + const onSuccess = ({ + response: { data, status } = { data: null, status: null }, + apiData: { uid, name } = { uid: null, name: '' }, + }: any) => { + //NOTE - Temp code to handle error thru API. Will remove this once sdk is ready + if ([200, 201, 202].includes(status)) { + const { taxonomy } = data; + this.taxonomiesSuccess[taxonomy.uid] = pick(taxonomy, ['name', 'description']); + log(this.importConfig, `Taxonomy '${name}' imported successfully!`, 'success'); + } else { + let errorMsg:any; + if ([500, 503, 502].includes(status)) errorMsg = data?.message || data; + else errorMsg = data?.error_message; + if (errorMsg === undefined) { + errorMsg = Object.values(data?.errors) && flatten(Object.values(data.errors)); + } + this.taxonomiesFailed[uid] = `Taxonomy '${name}' failed to be import! ${JSON.stringify(errorMsg)}`; + log(this.importConfig, `Taxonomy '${name}' failed to be import! ${JSON.stringify(errorMsg)}`, 'error'); + } }; const onReject = ({ error, apiData }: any) => { const err = error?.message ? JSON.parse(error.message) : error; const { name } = apiData; - if (err?.errors?.taxonomy) { - if (err?.errorCode === 0) { - log(this.importConfig, `Taxonomy '${name}' already exists!`, 'info'); - } else { - this.taxonomiesFailed[apiData.uid] = apiData; - log(this.importConfig, `Taxonomy '${name}' failed to be import! ${err.errors.taxonomy}`, 'error'); - } + if (err?.errors?.name) { + log(this.importConfig, `Taxonomy '${name}' already exists!`, 'info'); } else { this.taxonomiesFailed[apiData.uid] = apiData; log(this.importConfig, `Taxonomy '${name}' failed to be import! ${formatError(error)}`, 'error'); + log(this.importConfig, error, 'error'); } }; @@ -125,7 +164,8 @@ export default class ImportTaxonomies extends BaseClass { reject: onReject, resolve: onSuccess, entity: 'create-taxonomies', - includeParamOnCompletion: true + includeParamOnCompletion: true, + additionalInfo: this.taxonomyPayload, }, concurrencyLimit: this.importConfig.concurrency || this.importConfig.fetchConcurrency || 1, }, @@ -141,6 +181,12 @@ export default class ImportTaxonomies extends BaseClass { */ serializeTaxonomy(apiOptions: ApiOptions): ApiOptions { const { apiData: taxonomy } = apiOptions; + if (this.taxonomiesSuccess.hasOwnProperty(taxonomy.uid)) { + log(this.importConfig, `Taxonomy '${taxonomy.name}' already exists. Skipping it to avoid duplicates!`, 'info'); + apiOptions.entity = undefined; + } else { + apiOptions.apiData = taxonomy; + } apiOptions.apiData = taxonomy; return apiOptions; } @@ -170,28 +216,39 @@ export default class ImportTaxonomies extends BaseClass { return; } - const onSuccess = ({ response, apiData: { taxonomy_uid } = { taxonomy_uid: null } }: any) => { - const { uid, name } = response; - if (!this.termsSuccess[taxonomy_uid]) this.termsSuccess[taxonomy_uid] = {}; - this.termsSuccess[taxonomy_uid][uid] = pick(response, ['name']); - log(this.importConfig, `Term '${name}' imported successfully!`, 'success'); + const onSuccess = ({ + response: { data, status } = { data: null, status: null }, + apiData: { uid, name, taxonomy_uid } = { uid: null, name: '', taxonomy_uid: null }, + }: any) => { + //NOTE - Temp code to handle error thru API. Will remove this once sdk is ready + if ([200, 201, 202].includes(status)) { + if (!this.termsSuccess[taxonomy_uid]) this.termsSuccess[taxonomy_uid] = {}; + const { term } = data; + this.termsSuccess[taxonomy_uid][term.uid] = pick(term, ['name']); + log(this.importConfig, `Term '${name}' imported successfully!`, 'success'); + } else { + if (!this.termsFailed[taxonomy_uid]) this.termsFailed[taxonomy_uid] = {}; + let errorMsg; + if ([500, 503, 502].includes(status)) errorMsg = data?.message || data; + else errorMsg = data?.error_message; + if (errorMsg === undefined) { + errorMsg = Object.values(data?.errors) && flatten(Object.values(data.errors)); + } + this.termsFailed[taxonomy_uid][uid] = `Terms '${name}' failed to be import! ${JSON.stringify(errorMsg)}`; + log(this.importConfig, `Terms '${name}' failed to be import! ${JSON.stringify(errorMsg)}`, 'error'); + } }; const onReject = ({ error, apiData }: any) => { - const { taxonomy_uid, name } = apiData; + const { uid, taxonomy_uid, name } = apiData; if (!this.termsFailed[taxonomy_uid]) this.termsFailed[taxonomy_uid] = {}; const err = error?.message ? JSON.parse(error.message) : error; - - if (err?.errors?.term) { - if (err?.errorCode === 0) { - log(this.importConfig, `Term '${name}' already exists!`, 'info'); - } else { - this.termsFailed[taxonomy_uid][apiData.uid] = apiData; - log(this.importConfig, `Term '${name}' failed to be import! ${err.errors.term}`, 'error'); - } + if (err?.errors?.name) { + log(this.importConfig, `Term '${name}' already exists!`, 'info'); } else { this.termsFailed[taxonomy_uid][apiData.uid] = apiData; log(this.importConfig, `Term '${name}' failed to be import! ${formatError(error)}`, 'error'); + log(this.importConfig, error, 'error'); } }; @@ -213,7 +270,8 @@ export default class ImportTaxonomies extends BaseClass { reject: onReject, resolve: onSuccess, entity: 'create-terms', - includeParamOnCompletion: true + includeParamOnCompletion: true, + additionalInfo: this.taxonomyPayload, }, concurrencyLimit: this.importConfig.concurrency || this.importConfig.fetchConcurrency || 1, }, diff --git a/packages/contentstack-utilities/package.json b/packages/contentstack-utilities/package.json index 4fc63b60f8..c9445b92f1 100644 --- a/packages/contentstack-utilities/package.json +++ b/packages/contentstack-utilities/package.json @@ -32,7 +32,7 @@ "author": "contentstack", "license": "MIT", "dependencies": { - "@contentstack/management": "~1.11.0", + "@contentstack/management": "~1.10.2", "@oclif/core": "^2.9.3", "axios": "1.3.4", "chalk": "^4.0.0", diff --git a/packages/contentstack/package.json b/packages/contentstack/package.json index d43ad79951..5a7e6b70c7 100755 --- a/packages/contentstack/package.json +++ b/packages/contentstack/package.json @@ -38,7 +38,7 @@ "@contentstack/cli-launch": "~1.0.12", "@contentstack/cli-migration": "~1.3.13", "@contentstack/cli-utilities": "~1.5.3", - "@contentstack/management": "~1.11.0", + "@contentstack/management": "~1.10.0", "@oclif/core": "^2.9.3", "@oclif/plugin-help": "^5", "@oclif/plugin-not-found": "^2.4.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 12e9519d03..85de711d9a 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -26,7 +26,7 @@ importers: '@contentstack/cli-launch': ~1.0.12 '@contentstack/cli-migration': ~1.3.13 '@contentstack/cli-utilities': ~1.5.3 - '@contentstack/management': ~1.11.0 + '@contentstack/management': ~1.10.0 '@oclif/core': ^2.9.3 '@oclif/plugin-help': ^5 '@oclif/plugin-not-found': ^2.4.0 @@ -80,7 +80,7 @@ importers: '@contentstack/cli-launch': link:../contentstack-launch '@contentstack/cli-migration': link:../contentstack-migration '@contentstack/cli-utilities': link:../contentstack-utilities - '@contentstack/management': 1.11.0_debug@4.3.4 + '@contentstack/management': 1.10.2_debug@4.3.4 '@oclif/core': 2.15.0_4qcp7qp4jxxdgb4qbxgwox4hwq '@oclif/plugin-help': 5.2.14_4qcp7qp4jxxdgb4qbxgwox4hwq '@oclif/plugin-not-found': 2.4.0_4qcp7qp4jxxdgb4qbxgwox4hwq @@ -710,7 +710,7 @@ importers: specifiers: '@contentstack/cli-command': ~1.2.13 '@contentstack/cli-utilities': ~1.5.3 - '@contentstack/management': ~1.11.0 + '@contentstack/management': ~1.10.2 '@oclif/core': ^2.9.3 '@oclif/test': ^1.2.6 '@types/big-json': ^3.2.0 @@ -750,7 +750,7 @@ importers: dependencies: '@contentstack/cli-command': link:../contentstack-command '@contentstack/cli-utilities': link:../contentstack-utilities - '@contentstack/management': 1.11.0_debug@4.3.4 + '@contentstack/management': 1.10.2_debug@4.3.4 '@oclif/core': 2.9.4_4qcp7qp4jxxdgb4qbxgwox4hwq big-json: 3.2.0 bluebird: 3.7.2 @@ -1017,7 +1017,7 @@ importers: packages/contentstack-utilities: specifiers: - '@contentstack/management': ~1.11.0 + '@contentstack/management': ~1.10.2 '@oclif/core': ^2.9.3 '@oclif/test': ^2.2.10 '@types/chai': ^4.2.18 @@ -1061,7 +1061,7 @@ importers: winston: ^3.7.2 xdg-basedir: ^4.0.0 dependencies: - '@contentstack/management': 1.11.0_debug@4.3.4 + '@contentstack/management': 1.10.2_debug@4.3.4 '@oclif/core': 2.9.4_4qcp7qp4jxxdgb4qbxgwox4hwq axios: 1.3.4_debug@4.3.4 chalk: 4.1.2 @@ -1548,11 +1548,11 @@ packages: uuid: 8.3.2 dev: false - /@contentstack/management/1.11.0_debug@4.3.4: - resolution: {integrity: sha512-tv4At2Q5iGgkzL1MFGil/o36URKZfO6DY/KtpNJFYjmTHitZNv7uotH8OXkOPBMxB4xz58SG58lWB6fNTkLrpw==} + /@contentstack/management/1.10.2_debug@4.3.4: + resolution: {integrity: sha512-jO24EqcCJhOjqdsqw8y3T0SPPAd0DG4BByjUcV0S28W2yoa8aBbcjcbZioRPzRLYKTmZWsAZissl18cIJm5djQ==} engines: {node: '>=8.0.0'} dependencies: - axios: 1.5.1_debug@4.3.4 + axios: 1.5.0_debug@4.3.4 form-data: 3.0.1 lodash: 4.17.21 qs: 6.11.2 @@ -4512,8 +4512,8 @@ packages: - debug dev: false - /axios/1.5.1_debug@4.3.4: - resolution: {integrity: sha512-Q28iYCWzNHjAm+yEAot5QaAMxhMghWLFVf7rRdwhUI+c2jix2DUXjAHXVi+s1ibs3mjPO/cCgbA++3BjD0vP/A==} + /axios/1.5.0_debug@4.3.4: + resolution: {integrity: sha512-D4DdjDo5CY50Qms0qGQTTw6Q44jl7zRwY7bthds06pUGfChBCTcQs+N743eFWGEd6pRTMd6A+I87aWyFV5wiZQ==} dependencies: follow-redirects: 1.15.2_debug@4.3.4 form-data: 4.0.0 From 0781f40f6d6cae2b8c63e0a4ee172dfb69770137 Mon Sep 17 00:00:00 2001 From: Aman Kumar Date: Tue, 10 Oct 2023 19:19:54 +0530 Subject: [PATCH 34/45] refactor: removed console --- packages/contentstack-export-to-csv/src/util/index.js | 1 - 1 file changed, 1 deletion(-) diff --git a/packages/contentstack-export-to-csv/src/util/index.js b/packages/contentstack-export-to-csv/src/util/index.js index 42912fd80f..7b797d09c5 100644 --- a/packages/contentstack-export-to-csv/src/util/index.js +++ b/packages/contentstack-export-to-csv/src/util/index.js @@ -780,7 +780,6 @@ async function taxonomySDKHandler(payload, skip) { .then((data) => data) .catch((err) => handleErrorMsg(err)); case 'terms': - console.log("taxonomyUID---",taxonomyUID) queryParams['depth'] = 0; return await stackAPIClient .taxonomy(taxonomyUID) From ee88689a0e96d974a619ccda4a67e33447f6df8f Mon Sep 17 00:00:00 2001 From: Aman Kumar Date: Wed, 11 Oct 2023 12:22:16 +0530 Subject: [PATCH 35/45] fix: pagination & export to csv unit test cases --- .../src/commands/cm/export-to-csv.js | 3 ++- .../src/util/index.js | 23 +++++++++---------- .../test/unit/commands/export-to-csv.test.js | 22 +++++++++--------- 3 files changed, 24 insertions(+), 24 deletions(-) diff --git a/packages/contentstack-export-to-csv/src/commands/cm/export-to-csv.js b/packages/contentstack-export-to-csv/src/commands/cm/export-to-csv.js index 520f53d881..6b144a32a0 100644 --- a/packages/contentstack-export-to-csv/src/commands/cm/export-to-csv.js +++ b/packages/contentstack-export-to-csv/src/commands/cm/export-to-csv.js @@ -362,7 +362,8 @@ class ExportToCsvCommand extends Command { async createTaxonomyAndTermCsvFile(stackAPIClient, stackName, stack, taxUID) { const payload = { stackAPIClient, - type: '' + type: '', + limit: config.limit || 100 }; //check whether the taxonomy is valid or not let taxonomies = []; diff --git a/packages/contentstack-export-to-csv/src/util/index.js b/packages/contentstack-export-to-csv/src/util/index.js index 7b797d09c5..fb1cc21ceb 100644 --- a/packages/contentstack-export-to-csv/src/util/index.js +++ b/packages/contentstack-export-to-csv/src/util/index.js @@ -697,12 +697,12 @@ function wait(time) { * @param {array} taxonomies * @returns */ -async function getAllTaxonomies(payload, skip=0, taxonomies = []) { +async function getAllTaxonomies(payload, skip = 0, taxonomies = []) { payload['type'] = 'taxonomies'; - const response = await taxonomySDKHandler(payload); + const response = await taxonomySDKHandler(payload, skip); if (response) { - skip += config.limit || 100; - //TODO - replace reponse.taxonomies with items + skip += payload.limit; + //TODO - replace response.taxonomies with items taxonomies = [...taxonomies, ...response.taxonomies]; if (skip >= response?.count) { return taxonomies; @@ -721,11 +721,11 @@ async function getAllTaxonomies(payload, skip=0, taxonomies = []) { * @param {array} terms * @returns */ -async function getAllTermsOfTaxonomy(payload, skip=0, terms = []) { +async function getAllTermsOfTaxonomy(payload, skip = 0, terms = []) { payload['type'] = 'terms'; - const {items, count} = await taxonomySDKHandler(payload); + const { items, count } = await taxonomySDKHandler(payload, skip); if (items) { - skip += config.limit || 100; + skip += payload.limit; terms = [...terms, ...items]; if (skip >= count) { return terms; @@ -758,11 +758,10 @@ async function getTaxonomy(payload) { * @returns {*} Promise */ async function taxonomySDKHandler(payload, skip) { - const { stackAPIClient,taxonomyUID, type } = payload; + const { stackAPIClient, taxonomyUID, type } = payload; - const queryParams = { include_count: true }; - queryParams['limit'] = config.limit || 100; - if (skip >= 0) queryParams['skip'] = skip; + const queryParams = { include_count: true, limit: payload.limit }; + if (skip >= 0) queryParams['skip'] = skip || 0; switch (type) { case 'taxonomies': @@ -875,5 +874,5 @@ module.exports = { formatTaxonomiesData, formatTermsOfTaxonomyData, getTaxonomy, - getStacks, + getStacks }; diff --git a/packages/contentstack-export-to-csv/test/unit/commands/export-to-csv.test.js b/packages/contentstack-export-to-csv/test/unit/commands/export-to-csv.test.js index 0f88eb5311..31a4a888e3 100644 --- a/packages/contentstack-export-to-csv/test/unit/commands/export-to-csv.test.js +++ b/packages/contentstack-export-to-csv/test/unit/commands/export-to-csv.test.js @@ -25,13 +25,13 @@ describe('export-to-csv with action taxonomies', () => { }) .nock(cma, (api) => { api - .get(`/v3/taxonomies/${mockData.taxonomiesResp.taxonomies[0].uid}?include_count=true&skip=0&limit=30`) + .get(`/v3/taxonomies/${mockData.taxonomiesResp.taxonomies[0].uid}`) .reply(200, { taxonomy: mockData.taxonomiesResp.taxonomies[0] }); }) .nock(cma, (api) => { api - .get(`/v3/taxonomies/${mockData.taxonomiesResp.taxonomies[0].uid}/terms?include_count=true&skip=0&limit=100`) - .reply(200, { terms: mockData.termsResp.terms, count: mockData.termsResp.count }); + .get(`/v3/taxonomies/${mockData.taxonomiesResp.taxonomies[0].uid}/terms?include_count=true&limit=100&skip=0&depth=0`) + .reply(200, mockData.termsResp); }) .command([ 'cm:export-to-csv', @@ -60,17 +60,17 @@ describe('export-to-csv with action taxonomies', () => { }) .nock(cma, (api) => { api - .get('/v3/taxonomies?include_count=true&skip=0&limit=100') - .reply(200, { taxonomies: mockData.taxonomiesResp.taxonomies, count: mockData.taxonomiesResp.count }); + .get('/v3/taxonomies?include_count=true&limit=100&skip=0&count=true') + .reply(200, mockData.taxonomiesResp); }) .nock(cma, (api) => { api - .get(`/v3/taxonomies/${mockData.taxonomiesResp.taxonomies[0].uid}/terms?include_count=true&skip=0&limit=100`) - .reply(200, { terms: mockData.termsResp.terms, count: mockData.termsResp.count }); + .get(`/v3/taxonomies/${mockData.taxonomiesResp.taxonomies[0].uid}/terms?include_count=true&limit=100&skip=0&depth=0`) + .reply(200, mockData.termsResp); }) .nock(cma, (api) => { api - .get(`/v3/taxonomies/${mockData.taxonomiesResp.taxonomies[1].uid}/terms?include_count=true&skip=0&limit=100`) + .get(`/v3/taxonomies/${mockData.taxonomiesResp.taxonomies[1].uid}/terms?include_count=true&limit=100&skip=0&depth=0`) .reply(200, { terms: [], count: 0 }); }) .command([ @@ -109,13 +109,13 @@ describe('export-to-csv with action taxonomies', () => { }) .nock(cma, (api) => { api - .get(`/v3/taxonomies/${mockData.taxonomiesResp.taxonomies[0].uid}?include_count=true&skip=0&limit=30`) + .get(`/v3/taxonomies/${mockData.taxonomiesResp.taxonomies[0].uid}`) .reply(200, { taxonomy: mockData.taxonomiesResp.taxonomies[0] }); }) .nock(cma, (api) => { api - .get(`/v3/taxonomies/${mockData.taxonomiesResp.taxonomies[0].uid}/terms?include_count=true&skip=0&limit=100`) - .reply(200, { terms: mockData.termsResp.terms, count: mockData.termsResp.count }); + .get(`/v3/taxonomies/${mockData.taxonomiesResp.taxonomies[0].uid}/terms?include_count=true&limit=100&skip=0&depth=0`) + .reply(200, mockData.termsResp); }) .command(['cm:export-to-csv', '--taxonomy-uid', 'taxonomy_uid_1']) .it('CSV file should be created'); From 6712496a7bd73f3b6be7d9c51b1145e6c86e4996 Mon Sep 17 00:00:00 2001 From: Aman Kumar Date: Mon, 9 Oct 2023 19:23:31 +0530 Subject: [PATCH 36/45] feat: replace taxonomy & term http call with sdk --- package-lock.json | 40 ++--- .../src/commands/cm/export-to-csv.js | 14 +- .../src/util/index.js | 19 +- .../src/export/modules/taxonomies.ts | 163 +++++++----------- packages/contentstack-import/package.json | 2 +- .../src/import/modules/base-class.ts | 21 +-- .../src/import/modules/taxonomies.ts | 114 +++--------- packages/contentstack-utilities/package.json | 2 +- packages/contentstack/package.json | 2 +- pnpm-lock.yaml | 22 +-- 10 files changed, 157 insertions(+), 242 deletions(-) diff --git a/package-lock.json b/package-lock.json index 5dad81b9fe..75c29afcc1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -811,11 +811,11 @@ } }, "node_modules/@contentstack/management": { - "version": "1.10.2", - "resolved": "https://registry.npmjs.org/@contentstack/management/-/management-1.10.2.tgz", - "integrity": "sha512-jO24EqcCJhOjqdsqw8y3T0SPPAd0DG4BByjUcV0S28W2yoa8aBbcjcbZioRPzRLYKTmZWsAZissl18cIJm5djQ==", + "version": "1.11.0", + "resolved": "https://registry.npmjs.org/@contentstack/management/-/management-1.11.0.tgz", + "integrity": "sha512-tv4At2Q5iGgkzL1MFGil/o36URKZfO6DY/KtpNJFYjmTHitZNv7uotH8OXkOPBMxB4xz58SG58lWB6fNTkLrpw==", "dependencies": { - "axios": "^1.4.0", + "axios": "^1.5.1", "form-data": "^3.0.1", "lodash": "^4.17.21", "qs": "^6.11.2" @@ -825,9 +825,9 @@ } }, "node_modules/@contentstack/management/node_modules/axios": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/axios/-/axios-1.5.0.tgz", - "integrity": "sha512-D4DdjDo5CY50Qms0qGQTTw6Q44jl7zRwY7bthds06pUGfChBCTcQs+N743eFWGEd6pRTMd6A+I87aWyFV5wiZQ==", + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.5.1.tgz", + "integrity": "sha512-Q28iYCWzNHjAm+yEAot5QaAMxhMghWLFVf7rRdwhUI+c2jix2DUXjAHXVi+s1ibs3mjPO/cCgbA++3BjD0vP/A==", "dependencies": { "follow-redirects": "^1.15.0", "form-data": "^4.0.0", @@ -22760,7 +22760,7 @@ "@contentstack/cli-launch": "~1.0.12", "@contentstack/cli-migration": "~1.3.13", "@contentstack/cli-utilities": "~1.5.3", - "@contentstack/management": "~1.10.0", + "@contentstack/management": "~1.11.0", "@oclif/core": "^2.9.3", "@oclif/plugin-help": "^5", "@oclif/plugin-not-found": "^2.4.0", @@ -24587,7 +24587,7 @@ "dependencies": { "@contentstack/cli-command": "~1.2.13", "@contentstack/cli-utilities": "~1.5.3", - "@contentstack/management": "~1.10.2", + "@contentstack/management": "~1.11.0", "@oclif/core": "^2.9.3", "big-json": "^3.2.0", "bluebird": "^3.7.2", @@ -25386,7 +25386,7 @@ "version": "1.5.3", "license": "MIT", "dependencies": { - "@contentstack/management": "~1.10.2", + "@contentstack/management": "~1.11.0", "@oclif/core": "^2.9.3", "axios": "1.3.4", "chalk": "^4.0.0", @@ -26999,7 +26999,7 @@ "@contentstack/cli-launch": "~1.0.12", "@contentstack/cli-migration": "~1.3.13", "@contentstack/cli-utilities": "~1.5.3", - "@contentstack/management": "~1.10.0", + "@contentstack/management": "~1.11.0", "@oclif/core": "^2.9.3", "@oclif/plugin-help": "^5", "@oclif/plugin-not-found": "^2.4.0", @@ -28122,7 +28122,7 @@ "requires": { "@contentstack/cli-command": "~1.2.13", "@contentstack/cli-utilities": "~1.5.3", - "@contentstack/management": "~1.10.2", + "@contentstack/management": "~1.11.0", "@oclif/core": "^2.9.3", "@oclif/test": "^1.2.6", "@types/big-json": "^3.2.0", @@ -29009,7 +29009,7 @@ "@contentstack/cli-utilities": { "version": "file:packages/contentstack-utilities", "requires": { - "@contentstack/management": "~1.10.2", + "@contentstack/management": "~1.11.0", "@oclif/core": "^2.9.3", "@oclif/test": "^2.2.10", "@types/chai": "^4.2.18", @@ -29863,20 +29863,20 @@ } }, "@contentstack/management": { - "version": "1.10.2", - "resolved": "https://registry.npmjs.org/@contentstack/management/-/management-1.10.2.tgz", - "integrity": "sha512-jO24EqcCJhOjqdsqw8y3T0SPPAd0DG4BByjUcV0S28W2yoa8aBbcjcbZioRPzRLYKTmZWsAZissl18cIJm5djQ==", + "version": "1.11.0", + "resolved": "https://registry.npmjs.org/@contentstack/management/-/management-1.11.0.tgz", + "integrity": "sha512-tv4At2Q5iGgkzL1MFGil/o36URKZfO6DY/KtpNJFYjmTHitZNv7uotH8OXkOPBMxB4xz58SG58lWB6fNTkLrpw==", "requires": { - "axios": "^1.4.0", + "axios": "^1.5.1", "form-data": "^3.0.1", "lodash": "^4.17.21", "qs": "^6.11.2" }, "dependencies": { "axios": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/axios/-/axios-1.5.0.tgz", - "integrity": "sha512-D4DdjDo5CY50Qms0qGQTTw6Q44jl7zRwY7bthds06pUGfChBCTcQs+N743eFWGEd6pRTMd6A+I87aWyFV5wiZQ==", + "version": "1.5.1", + "resolved": "https://registry.npmjs.org/axios/-/axios-1.5.1.tgz", + "integrity": "sha512-Q28iYCWzNHjAm+yEAot5QaAMxhMghWLFVf7rRdwhUI+c2jix2DUXjAHXVi+s1ibs3mjPO/cCgbA++3BjD0vP/A==", "requires": { "follow-redirects": "^1.15.0", "form-data": "^4.0.0", diff --git a/packages/contentstack-export-to-csv/src/commands/cm/export-to-csv.js b/packages/contentstack-export-to-csv/src/commands/cm/export-to-csv.js index 74442ba553..6b144a32a0 100644 --- a/packages/contentstack-export-to-csv/src/commands/cm/export-to-csv.js +++ b/packages/contentstack-export-to-csv/src/commands/cm/export-to-csv.js @@ -214,7 +214,6 @@ class ExportToCsvCommand extends Command { case 'taxonomies': { let stack; let stackAPIClient; - let taxUID; if (managementTokenAlias) { const { stackDetails, apiClient } = await this.getAliasDetails(managementTokenAlias, stackName); managementAPIClient = apiClient; @@ -224,7 +223,7 @@ class ExportToCsvCommand extends Command { } stackAPIClient = this.getStackClient(managementAPIClient, stack); - await this.createTaxonomyAndTermCsvFile(stackName, stack, taxonomyUID); + await this.createTaxonomyAndTermCsvFile(stackAPIClient, stackName, stack, taxonomyUID); break; } } @@ -242,8 +241,8 @@ class ExportToCsvCommand extends Command { getStackClient(managementAPIClient, stack) { const stackInit = { api_key: stack.apiKey, - branch_uid: stack.branch_uid, }; + if(stack?.branch_uid) stackInit['branch_uid'] = stack.branch_uid; if (stack.token) { return managementAPIClient.stack({ ...stackInit, @@ -360,8 +359,7 @@ class ExportToCsvCommand extends Command { * @param {object} stack * @param {string} taxUID */ - async createTaxonomyAndTermCsvFile(stackName, stack, taxUID) { - const { cma } = configHandler.get('region') || {}; + async createTaxonomyAndTermCsvFile(stackAPIClient, stackName, stack, taxUID) { const payload = { stackAPIClient, type: '', @@ -370,10 +368,10 @@ class ExportToCsvCommand extends Command { //check whether the taxonomy is valid or not let taxonomies = []; if (taxUID) { - const taxonomy = await util.getTaxonomy(payload, taxUID); + payload['taxonomyUID'] = taxUID; + const taxonomy = await util.getTaxonomy(payload); taxonomies.push(taxonomy); } else { - payload['url'] = payload.baseUrl; taxonomies = await util.getAllTaxonomies(payload); } @@ -389,7 +387,7 @@ class ExportToCsvCommand extends Command { const taxonomy = taxonomies[index]; const taxonomyUID = taxonomy?.uid; if (taxonomyUID) { - payload['url'] = `${payload.baseUrl}/${taxonomyUID}/terms`; + payload['taxonomyUID'] = taxonomyUID; const terms = await util.getAllTermsOfTaxonomy(payload); const formattedTermsData = util.formatTermsOfTaxonomyData(terms, taxonomyUID); const taxonomyName = taxonomy?.name ? taxonomy.name : ''; diff --git a/packages/contentstack-export-to-csv/src/util/index.js b/packages/contentstack-export-to-csv/src/util/index.js index 1ad1343517..fb1cc21ceb 100644 --- a/packages/contentstack-export-to-csv/src/util/index.js +++ b/packages/contentstack-export-to-csv/src/util/index.js @@ -9,7 +9,14 @@ const debug = require('debug')('export-to-csv'); const checkboxPlus = require('inquirer-checkbox-plus-prompt'); const config = require('./config.js'); -const { cliux, configHandler, HttpClient, messageHandler } = require('@contentstack/cli-utilities'); +const { + cliux, + configHandler, + HttpClient, + messageHandler, + managementSDKClient, + ContentstackClient, +} = require('@contentstack/cli-utilities'); const directory = './data'; const delimeter = os.platform() === 'win32' ? '\\' : '/'; @@ -687,7 +694,6 @@ function wait(time) { * fetch all taxonomies in the provided stack * @param {object} payload * @param {number} skip - * @param {number} limit * @param {array} taxonomies * @returns */ @@ -701,14 +707,14 @@ async function getAllTaxonomies(payload, skip = 0, taxonomies = []) { if (skip >= response?.count) { return taxonomies; } else { - return getAllTaxonomies(payload, skip, limit, taxonomies); + return getAllTaxonomies(payload, skip, taxonomies); } } return taxonomies; } /** - * fetch terms of related taxonomy + * fetch taxonomy related terms * @param {object} payload * @param {number} skip * @param {number} limit @@ -724,7 +730,7 @@ async function getAllTermsOfTaxonomy(payload, skip = 0, terms = []) { if (skip >= count) { return terms; } else { - return getAllTermsOfTaxonomy(payload, skip, limit, terms); + return getAllTermsOfTaxonomy(payload, skip, terms); } } return terms; @@ -828,7 +834,8 @@ function handleErrorMsg(err) { if (err?.errorMessage) { cliux.print(`Error: ${err.errorMessage}`, { color: 'red' }); } else if (err?.message) { - cliux.print(`Error: ${err.message}`, { color: 'red' }); + const errorMsg = err?.errors?.taxonomy || err?.errors?.term || err?.message; + cliux.print(`Error: ${errorMsg}`, { color: 'red' }); } else { console.log(err); cliux.print(`Error: ${messageHandler.parse('CLI_EXPORT_CSV_API_FAILED')}`, { color: 'red' }); diff --git a/packages/contentstack-export/src/export/modules/taxonomies.ts b/packages/contentstack-export/src/export/modules/taxonomies.ts index e7d540911c..dc0de7378d 100644 --- a/packages/contentstack-export/src/export/modules/taxonomies.ts +++ b/packages/contentstack-export/src/export/modules/taxonomies.ts @@ -1,28 +1,23 @@ import omit from 'lodash/omit'; import keys from 'lodash/keys'; import isEmpty from 'lodash/isEmpty'; -import flatten from 'lodash/flatten'; import { resolve as pResolve } from 'node:path'; -import { cliux, configHandler, HttpClient } from '@contentstack/cli-utilities'; import BaseClass from './base-class'; import { log, fsUtil } from '../../utils'; import { TaxonomiesConfig, TermsConfig, ModuleClassParams } from '../../types'; -//NOTE: Temp types need to remove once sdk available -type TaxonomyPayload = { - baseUrl: string; - url: string; - mgToken: string; - apiKey: string; -}; - export default class ExportTaxonomies extends BaseClass { private taxonomies: Record>; private terms: Record>; private taxonomiesConfig: TaxonomiesConfig; - private taxonomyPayload: TaxonomyPayload; private termsConfig: TermsConfig; + private qs: { + include_count: boolean; + skip: number; + asc: string; + depth?: number; + }; public taxonomiesFolderPath: string; public termsFolderPath: string; @@ -32,12 +27,7 @@ export default class ExportTaxonomies extends BaseClass { this.terms = {}; this.taxonomiesConfig = exportConfig.modules.taxonomies; this.termsConfig = exportConfig.modules.terms; - this.taxonomyPayload = { - baseUrl: '', - url: '', - mgToken: exportConfig.management_token, - apiKey: exportConfig.source_stack, - }; + this.qs = { include_count: true, skip: 0, asc: 'created_at' }; } async start(): Promise { @@ -53,12 +43,8 @@ export default class ExportTaxonomies extends BaseClass { this.termsFolderPath = pResolve(this.taxonomiesFolderPath, this.termsConfig.dirName); await fsUtil.makeDirectory(this.termsFolderPath); - const { cma } = configHandler.get('region') || {}; - this.taxonomyPayload.baseUrl = `${cma}/v3/taxonomies`; - this.taxonomyPayload.url = this.taxonomyPayload.baseUrl; - //fetch all taxonomies and write into taxonomies folder - await this.getAllTaxonomies(this.taxonomyPayload); + await this.getAllTaxonomies(); if (this.taxonomies === undefined || isEmpty(this.taxonomies)) { log(this.exportConfig, 'No taxonomies found!', 'info'); return; @@ -73,22 +59,34 @@ export default class ExportTaxonomies extends BaseClass { /** * fetch all taxonomies in the provided stack - * @param {TaxonomyPayload} payload * @param {number} skip - * @returns + * @returns {Promise} */ - async getAllTaxonomies(payload: TaxonomyPayload, skip = 0): Promise { - const response = await this.apiRequestHandler(payload, skip); - if (response?.taxonomies) { - skip += this.taxonomiesConfig.limit || 100; - this.sanitizeTaxonomiesAttribs(response.taxonomies); - if (skip >= response?.count) { - return; - } else { - return await this.getAllTaxonomies(payload, skip); - } + async getAllTaxonomies(skip = 0): Promise { + if (skip) { + this.qs.skip = skip; } - return; + //TODO - replace count with find + await this.stack + .taxonomy() + .query(this.qs) + .count() + .then(async (data: any) => { + const { taxonomies, count } = data; + const taxonomiesCount = count !== undefined ? count : taxonomies?.length; + + if (taxonomies?.length) { + this.sanitizeTaxonomiesAttribs(taxonomies); + skip += this.taxonomiesConfig.limit || 100; + if (skip >= taxonomiesCount) { + return; + } + return await this.getAllTaxonomies(skip); + } + }) + .catch((error: any) => { + this.handleErrorMsg(error); + }); } /** @@ -107,15 +105,16 @@ export default class ExportTaxonomies extends BaseClass { /** * fetch all terms of respective taxonomy and write it into -terms file + * @returns {Promise} */ async getAllTerms() { const taxonomiesUID = keys(this.taxonomies) || []; + this.qs.depth = 0; + for (let index = 0; index < taxonomiesUID?.length; index++) { const taxonomyUID = taxonomiesUID[index]; - this.taxonomyPayload.url = `${this.taxonomyPayload.baseUrl}/${taxonomyUID}/terms`; this.terms = {}; - await this.fetchTermsOfTaxonomy(this.taxonomyPayload); - + await this.fetchTermsOfTaxonomy(taxonomyUID); if (this.terms === undefined || isEmpty(this.terms)) { log(this.exportConfig, `No terms found for taxonomy - '${taxonomyUID}'!`, 'info'); } else { @@ -128,22 +127,36 @@ export default class ExportTaxonomies extends BaseClass { /** * fetch all terms of the provided taxonomy uid - * @param {TaxonomyPayload} payload + * @async + * @param {string} taxonomyUID * @param {number} skip - * @returns + * @returns {Promise} */ - async fetchTermsOfTaxonomy(payload: TaxonomyPayload, skip = 0): Promise { - const response = await this.apiRequestHandler(payload, skip); - if (response?.terms) { - skip += this.termsConfig.limit || 100; - this.sanitizeTermsAttribs(response.terms); - if (skip >= response?.count) { - return; - } else { - return await this.fetchTermsOfTaxonomy(payload, skip); - } + async fetchTermsOfTaxonomy(taxonomyUID: string, skip = 0): Promise { + if (skip) { + this.qs.skip = skip; } - return; + await this.stack + .taxonomy(taxonomyUID) + .terms() + .query(this.qs) + .find() + .then(async (data: any) => { + const { items, count } = data; + const termsCount = count !== undefined ? count : items?.length; + + if (items?.length) { + this.sanitizeTermsAttribs(items); + skip += this.taxonomiesConfig.limit || 100; + if (skip >= termsCount) { + return; + } + return await this.fetchTermsOfTaxonomy(taxonomyUID, skip); + } + }) + .catch((error: any) => { + this.handleErrorMsg(error); + }); } /** @@ -158,54 +171,12 @@ export default class ExportTaxonomies extends BaseClass { } } - //NOTE: Temp code need to remove once sdk available - async apiRequestHandler(payload: TaxonomyPayload, skip: number) { - const headers: any = { - api_key: payload.apiKey, - 'Content-Type': 'application/json', - }; - - if (payload?.mgToken) headers['authorization'] = payload.mgToken; - else headers['authToken'] = configHandler.get('authtoken'); - - const params = { - include_count: true, - skip: 0, - limit: this.taxonomiesConfig.limit || 100, - depth: 0 // include all the terms if set to 0 - }; - - if (skip >= 0) params['skip'] = skip; - - return await new HttpClient() - .headers(headers) - .queryParams(params) - .get(payload.url) - .then((res: any) => { - //NOTE - temporary code for handling api errors response - const { status, data } = res; - if ([200, 201, 202].includes(status)) return data; - else { - let errorMsg; - if ([500, 503, 502].includes(status)) errorMsg = data?.message || data; - else errorMsg = data?.error_message; - if (errorMsg === undefined) { - errorMsg = Object.values(data?.errors) && flatten(Object.values(data.errors)); - } - cliux.print(`Error: ${errorMsg}`, { color: 'red' }); - } - }) - .catch((err: any) => this.handleErrorMsg(err)); - } - - //NOTE: Temp code need to remove once sdk available handleErrorMsg(err: any) { if (err?.errorMessage) { - cliux.print(`Error: ${err.errorMessage}`, { color: 'red' }); + log(this.exportConfig, `Failed to export. ${err.errorMessage}`, 'error'); } else if (err?.message) { - cliux.print(`Error: ${err.message}`, { color: 'red' }); - } else { - cliux.print(`Error: Something went wrong! Please try again.`, { color: 'red' }); + const errorMsg = err?.errors?.taxonomy || err?.errors?.term || err?.message; + log(this.exportConfig, `Failed to export. ${errorMsg}`, 'error'); } } } diff --git a/packages/contentstack-import/package.json b/packages/contentstack-import/package.json index 599d2ea456..7facb1d483 100644 --- a/packages/contentstack-import/package.json +++ b/packages/contentstack-import/package.json @@ -7,7 +7,7 @@ "dependencies": { "@contentstack/cli-command": "~1.2.13", "@contentstack/cli-utilities": "~1.5.3", - "@contentstack/management": "~1.10.2", + "@contentstack/management": "~1.11.0", "@oclif/core": "^2.9.3", "big-json": "^3.2.0", "bluebird": "^3.7.2", diff --git a/packages/contentstack-import/src/import/modules/base-class.ts b/packages/contentstack-import/src/import/modules/base-class.ts index e47f8790ab..7a758fb8a6 100644 --- a/packages/contentstack-import/src/import/modules/base-class.ts +++ b/packages/contentstack-import/src/import/modules/base-class.ts @@ -18,7 +18,6 @@ import { LabelData } from '@contentstack/management/types/stack/label'; import { WebhookData } from '@contentstack/management/types/stack/webhook'; import { WorkflowData } from '@contentstack/management/types/stack/workflow'; import { RoleData } from '@contentstack/management/types/stack/role'; -import { HttpClient } from '@contentstack/cli-utilities'; import { log } from '../../utils'; import { ImportConfig, ModuleClassParams } from '../../types'; @@ -385,18 +384,16 @@ export default abstract class BaseClass { .then(onSuccess) .catch(onReject); case 'create-taxonomies': - return new HttpClient() - .headers(additionalInfo.headers) - .post(additionalInfo.url, { taxonomy: apiData }) - .then(onSuccess) - .catch(onReject); + return this.stack.taxonomy().create({ taxonomy: apiData }).then(onSuccess).catch(onReject); case 'create-terms': - const url = `${additionalInfo.baseUrl}/${apiData.taxonomy_uid}/terms`; - return new HttpClient() - .headers(additionalInfo.headers) - .post(url, { term: apiData }) - .then(onSuccess) - .catch(onReject); + if (apiData?.taxonomy_uid) { + return this.stack + .taxonomy(apiData.taxonomy_uid) + .terms() + .create({ term: apiData }) + .then(onSuccess) + .catch(onReject); + } default: return Promise.resolve(); } diff --git a/packages/contentstack-import/src/import/modules/taxonomies.ts b/packages/contentstack-import/src/import/modules/taxonomies.ts index b3f3453989..97231f79c5 100644 --- a/packages/contentstack-import/src/import/modules/taxonomies.ts +++ b/packages/contentstack-import/src/import/modules/taxonomies.ts @@ -3,21 +3,11 @@ import pick from 'lodash/pick'; import { join } from 'node:path'; import values from 'lodash/values'; import isEmpty from 'lodash/isEmpty'; -import flatten from 'lodash/flatten'; -import { configHandler } from '@contentstack/cli-utilities'; import BaseClass, { ApiOptions } from './base-class'; import { log, formatError, fsUtil, fileHelper } from '../../utils'; import { ModuleClassParams, TaxonomiesConfig, TermsConfig } from '../../types'; -//NOTE: Temp types need to remove once sdk available -type TaxonomyPayload = { - baseUrl: string; - url: string; - mgToken?: string; - reqPayload: Record; - headers: Record; -}; export default class ImportTaxonomies extends BaseClass { private taxonomiesMapperDirPath: string; @@ -31,7 +21,6 @@ export default class ImportTaxonomies extends BaseClass { private termsConfig: TermsConfig; private termsSuccessPath: string; private termsFailsPath: string; - private taxonomyPayload: TaxonomyPayload; public taxonomiesSuccess: Record = {}; public taxonomiesFailed: Record = {}; public termsSuccess: Record> = {}; @@ -51,16 +40,6 @@ export default class ImportTaxonomies extends BaseClass { this.taxFailsPath = join(this.taxonomiesMapperDirPath, 'fails.json'); this.termsSuccessPath = join(this.termsMapperDirPath, 'success.json'); this.termsFailsPath = join(this.termsMapperDirPath, 'fails.json'); - this.taxonomyPayload = { - baseUrl: '', - url: '', - mgToken: importConfig.management_token, - reqPayload: {}, - headers: { - 'Content-Type': 'application/json', - api_key: importConfig.target_stack, - }, - }; } /** @@ -81,13 +60,6 @@ export default class ImportTaxonomies extends BaseClass { return; } - //NOTE - Temp code for api request - const { cma } = configHandler.get('region') || {}; - this.taxonomyPayload.baseUrl = `${cma}/v3/taxonomies`; - this.taxonomyPayload.url = this.taxonomyPayload.baseUrl; - if (this.taxonomyPayload?.mgToken) this.taxonomyPayload.headers['authorization'] = this.taxonomyPayload.mgToken; - else this.taxonomyPayload.headers['authtoken'] = configHandler.get('authtoken'); - //Step 2 create taxonomies & terms mapper directory await fsUtil.makeDirectory(this.taxonomiesMapperDirPath); await fsUtil.makeDirectory(this.termsMapperDirPath); @@ -119,39 +91,28 @@ export default class ImportTaxonomies extends BaseClass { return; } - const apiContent = values(this.taxonomies) as Record[];; + const apiContent = values(this.taxonomies) as Record[]; this.taxonomyUIDs = keys(this.taxonomies); - const onSuccess = ({ - response: { data, status } = { data: null, status: null }, - apiData: { uid, name } = { uid: null, name: '' }, - }: any) => { - //NOTE - Temp code to handle error thru API. Will remove this once sdk is ready - if ([200, 201, 202].includes(status)) { - const { taxonomy } = data; - this.taxonomiesSuccess[taxonomy.uid] = pick(taxonomy, ['name', 'description']); - log(this.importConfig, `Taxonomy '${name}' imported successfully!`, 'success'); - } else { - let errorMsg:any; - if ([500, 503, 502].includes(status)) errorMsg = data?.message || data; - else errorMsg = data?.error_message; - if (errorMsg === undefined) { - errorMsg = Object.values(data?.errors) && flatten(Object.values(data.errors)); - } - this.taxonomiesFailed[uid] = `Taxonomy '${name}' failed to be import! ${JSON.stringify(errorMsg)}`; - log(this.importConfig, `Taxonomy '${name}' failed to be import! ${JSON.stringify(errorMsg)}`, 'error'); - } + const onSuccess = ({ response }: any) => { + const { uid, name } = response; + this.taxonomiesSuccess[uid] = pick(response, ['name', 'description']); + log(this.importConfig, `Taxonomy '${name}' imported successfully!`, 'success'); }; const onReject = ({ error, apiData }: any) => { const err = error?.message ? JSON.parse(error.message) : error; const { name } = apiData; - if (err?.errors?.name) { - log(this.importConfig, `Taxonomy '${name}' already exists!`, 'info'); + if (err?.errors?.taxonomy) { + if (err?.errorCode === 0) { + log(this.importConfig, `Taxonomy '${name}' already exists!`, 'info'); + } else { + this.taxonomiesFailed[apiData.uid] = apiData; + log(this.importConfig, `Taxonomy '${name}' failed to be import! ${err.errors.taxonomy}`, 'error'); + } } else { this.taxonomiesFailed[apiData.uid] = apiData; log(this.importConfig, `Taxonomy '${name}' failed to be import! ${formatError(error)}`, 'error'); - log(this.importConfig, error, 'error'); } }; @@ -164,8 +125,7 @@ export default class ImportTaxonomies extends BaseClass { reject: onReject, resolve: onSuccess, entity: 'create-taxonomies', - includeParamOnCompletion: true, - additionalInfo: this.taxonomyPayload, + includeParamOnCompletion: true }, concurrencyLimit: this.importConfig.concurrency || this.importConfig.fetchConcurrency || 1, }, @@ -181,12 +141,6 @@ export default class ImportTaxonomies extends BaseClass { */ serializeTaxonomy(apiOptions: ApiOptions): ApiOptions { const { apiData: taxonomy } = apiOptions; - if (this.taxonomiesSuccess.hasOwnProperty(taxonomy.uid)) { - log(this.importConfig, `Taxonomy '${taxonomy.name}' already exists. Skipping it to avoid duplicates!`, 'info'); - apiOptions.entity = undefined; - } else { - apiOptions.apiData = taxonomy; - } apiOptions.apiData = taxonomy; return apiOptions; } @@ -216,39 +170,28 @@ export default class ImportTaxonomies extends BaseClass { return; } - const onSuccess = ({ - response: { data, status } = { data: null, status: null }, - apiData: { uid, name, taxonomy_uid } = { uid: null, name: '', taxonomy_uid: null }, - }: any) => { - //NOTE - Temp code to handle error thru API. Will remove this once sdk is ready - if ([200, 201, 202].includes(status)) { - if (!this.termsSuccess[taxonomy_uid]) this.termsSuccess[taxonomy_uid] = {}; - const { term } = data; - this.termsSuccess[taxonomy_uid][term.uid] = pick(term, ['name']); - log(this.importConfig, `Term '${name}' imported successfully!`, 'success'); - } else { - if (!this.termsFailed[taxonomy_uid]) this.termsFailed[taxonomy_uid] = {}; - let errorMsg; - if ([500, 503, 502].includes(status)) errorMsg = data?.message || data; - else errorMsg = data?.error_message; - if (errorMsg === undefined) { - errorMsg = Object.values(data?.errors) && flatten(Object.values(data.errors)); - } - this.termsFailed[taxonomy_uid][uid] = `Terms '${name}' failed to be import! ${JSON.stringify(errorMsg)}`; - log(this.importConfig, `Terms '${name}' failed to be import! ${JSON.stringify(errorMsg)}`, 'error'); - } + const onSuccess = ({ response, apiData: { taxonomy_uid } = { taxonomy_uid: null } }: any) => { + const { uid, name } = response; + if (!this.termsSuccess[taxonomy_uid]) this.termsSuccess[taxonomy_uid] = {}; + this.termsSuccess[taxonomy_uid][uid] = pick(response, ['name']); + log(this.importConfig, `Term '${name}' imported successfully!`, 'success'); }; const onReject = ({ error, apiData }: any) => { - const { uid, taxonomy_uid, name } = apiData; + const { taxonomy_uid, name } = apiData; if (!this.termsFailed[taxonomy_uid]) this.termsFailed[taxonomy_uid] = {}; const err = error?.message ? JSON.parse(error.message) : error; - if (err?.errors?.name) { - log(this.importConfig, `Term '${name}' already exists!`, 'info'); + + if (err?.errors?.term) { + if (err?.errorCode === 0) { + log(this.importConfig, `Term '${name}' already exists!`, 'info'); + } else { + this.termsFailed[taxonomy_uid][apiData.uid] = apiData; + log(this.importConfig, `Term '${name}' failed to be import! ${err.errors.term}`, 'error'); + } } else { this.termsFailed[taxonomy_uid][apiData.uid] = apiData; log(this.importConfig, `Term '${name}' failed to be import! ${formatError(error)}`, 'error'); - log(this.importConfig, error, 'error'); } }; @@ -270,8 +213,7 @@ export default class ImportTaxonomies extends BaseClass { reject: onReject, resolve: onSuccess, entity: 'create-terms', - includeParamOnCompletion: true, - additionalInfo: this.taxonomyPayload, + includeParamOnCompletion: true }, concurrencyLimit: this.importConfig.concurrency || this.importConfig.fetchConcurrency || 1, }, diff --git a/packages/contentstack-utilities/package.json b/packages/contentstack-utilities/package.json index c9445b92f1..4fc63b60f8 100644 --- a/packages/contentstack-utilities/package.json +++ b/packages/contentstack-utilities/package.json @@ -32,7 +32,7 @@ "author": "contentstack", "license": "MIT", "dependencies": { - "@contentstack/management": "~1.10.2", + "@contentstack/management": "~1.11.0", "@oclif/core": "^2.9.3", "axios": "1.3.4", "chalk": "^4.0.0", diff --git a/packages/contentstack/package.json b/packages/contentstack/package.json index 5a7e6b70c7..d43ad79951 100755 --- a/packages/contentstack/package.json +++ b/packages/contentstack/package.json @@ -38,7 +38,7 @@ "@contentstack/cli-launch": "~1.0.12", "@contentstack/cli-migration": "~1.3.13", "@contentstack/cli-utilities": "~1.5.3", - "@contentstack/management": "~1.10.0", + "@contentstack/management": "~1.11.0", "@oclif/core": "^2.9.3", "@oclif/plugin-help": "^5", "@oclif/plugin-not-found": "^2.4.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 85de711d9a..12e9519d03 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -26,7 +26,7 @@ importers: '@contentstack/cli-launch': ~1.0.12 '@contentstack/cli-migration': ~1.3.13 '@contentstack/cli-utilities': ~1.5.3 - '@contentstack/management': ~1.10.0 + '@contentstack/management': ~1.11.0 '@oclif/core': ^2.9.3 '@oclif/plugin-help': ^5 '@oclif/plugin-not-found': ^2.4.0 @@ -80,7 +80,7 @@ importers: '@contentstack/cli-launch': link:../contentstack-launch '@contentstack/cli-migration': link:../contentstack-migration '@contentstack/cli-utilities': link:../contentstack-utilities - '@contentstack/management': 1.10.2_debug@4.3.4 + '@contentstack/management': 1.11.0_debug@4.3.4 '@oclif/core': 2.15.0_4qcp7qp4jxxdgb4qbxgwox4hwq '@oclif/plugin-help': 5.2.14_4qcp7qp4jxxdgb4qbxgwox4hwq '@oclif/plugin-not-found': 2.4.0_4qcp7qp4jxxdgb4qbxgwox4hwq @@ -710,7 +710,7 @@ importers: specifiers: '@contentstack/cli-command': ~1.2.13 '@contentstack/cli-utilities': ~1.5.3 - '@contentstack/management': ~1.10.2 + '@contentstack/management': ~1.11.0 '@oclif/core': ^2.9.3 '@oclif/test': ^1.2.6 '@types/big-json': ^3.2.0 @@ -750,7 +750,7 @@ importers: dependencies: '@contentstack/cli-command': link:../contentstack-command '@contentstack/cli-utilities': link:../contentstack-utilities - '@contentstack/management': 1.10.2_debug@4.3.4 + '@contentstack/management': 1.11.0_debug@4.3.4 '@oclif/core': 2.9.4_4qcp7qp4jxxdgb4qbxgwox4hwq big-json: 3.2.0 bluebird: 3.7.2 @@ -1017,7 +1017,7 @@ importers: packages/contentstack-utilities: specifiers: - '@contentstack/management': ~1.10.2 + '@contentstack/management': ~1.11.0 '@oclif/core': ^2.9.3 '@oclif/test': ^2.2.10 '@types/chai': ^4.2.18 @@ -1061,7 +1061,7 @@ importers: winston: ^3.7.2 xdg-basedir: ^4.0.0 dependencies: - '@contentstack/management': 1.10.2_debug@4.3.4 + '@contentstack/management': 1.11.0_debug@4.3.4 '@oclif/core': 2.9.4_4qcp7qp4jxxdgb4qbxgwox4hwq axios: 1.3.4_debug@4.3.4 chalk: 4.1.2 @@ -1548,11 +1548,11 @@ packages: uuid: 8.3.2 dev: false - /@contentstack/management/1.10.2_debug@4.3.4: - resolution: {integrity: sha512-jO24EqcCJhOjqdsqw8y3T0SPPAd0DG4BByjUcV0S28W2yoa8aBbcjcbZioRPzRLYKTmZWsAZissl18cIJm5djQ==} + /@contentstack/management/1.11.0_debug@4.3.4: + resolution: {integrity: sha512-tv4At2Q5iGgkzL1MFGil/o36URKZfO6DY/KtpNJFYjmTHitZNv7uotH8OXkOPBMxB4xz58SG58lWB6fNTkLrpw==} engines: {node: '>=8.0.0'} dependencies: - axios: 1.5.0_debug@4.3.4 + axios: 1.5.1_debug@4.3.4 form-data: 3.0.1 lodash: 4.17.21 qs: 6.11.2 @@ -4512,8 +4512,8 @@ packages: - debug dev: false - /axios/1.5.0_debug@4.3.4: - resolution: {integrity: sha512-D4DdjDo5CY50Qms0qGQTTw6Q44jl7zRwY7bthds06pUGfChBCTcQs+N743eFWGEd6pRTMd6A+I87aWyFV5wiZQ==} + /axios/1.5.1_debug@4.3.4: + resolution: {integrity: sha512-Q28iYCWzNHjAm+yEAot5QaAMxhMghWLFVf7rRdwhUI+c2jix2DUXjAHXVi+s1ibs3mjPO/cCgbA++3BjD0vP/A==} dependencies: follow-redirects: 1.15.2_debug@4.3.4 form-data: 4.0.0 From 5a0bd64bde1041ab989bddb83f7ce733b9db3eb8 Mon Sep 17 00:00:00 2001 From: Aman Kumar Date: Wed, 11 Oct 2023 17:31:15 +0530 Subject: [PATCH 37/45] fix: taxonomies & term error handling --- .../src/import/modules/taxonomies.ts | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/packages/contentstack-import/src/import/modules/taxonomies.ts b/packages/contentstack-import/src/import/modules/taxonomies.ts index 97231f79c5..2ff886609c 100644 --- a/packages/contentstack-import/src/import/modules/taxonomies.ts +++ b/packages/contentstack-import/src/import/modules/taxonomies.ts @@ -104,12 +104,8 @@ export default class ImportTaxonomies extends BaseClass { const err = error?.message ? JSON.parse(error.message) : error; const { name } = apiData; if (err?.errors?.taxonomy) { - if (err?.errorCode === 0) { - log(this.importConfig, `Taxonomy '${name}' already exists!`, 'info'); - } else { - this.taxonomiesFailed[apiData.uid] = apiData; - log(this.importConfig, `Taxonomy '${name}' failed to be import! ${err.errors.taxonomy}`, 'error'); - } + this.taxonomiesFailed[apiData.uid] = apiData; + log(this.importConfig, `Taxonomy '${name}' failed to be import! ${err.errors.taxonomy}`, 'error'); } else { this.taxonomiesFailed[apiData.uid] = apiData; log(this.importConfig, `Taxonomy '${name}' failed to be import! ${formatError(error)}`, 'error'); @@ -183,12 +179,8 @@ export default class ImportTaxonomies extends BaseClass { const err = error?.message ? JSON.parse(error.message) : error; if (err?.errors?.term) { - if (err?.errorCode === 0) { - log(this.importConfig, `Term '${name}' already exists!`, 'info'); - } else { - this.termsFailed[taxonomy_uid][apiData.uid] = apiData; - log(this.importConfig, `Term '${name}' failed to be import! ${err.errors.term}`, 'error'); - } + this.termsFailed[taxonomy_uid][apiData.uid] = apiData; + log(this.importConfig, `Term '${name}' failed to be import! ${err.errors.term}`, 'error'); } else { this.termsFailed[taxonomy_uid][apiData.uid] = apiData; log(this.importConfig, `Term '${name}' failed to be import! ${formatError(error)}`, 'error'); From 434950505c55efbcd80b85238dc62387fe1da6c5 Mon Sep 17 00:00:00 2001 From: Aman Kumar Date: Wed, 11 Oct 2023 17:34:16 +0530 Subject: [PATCH 38/45] fix: taxonomies export error --- .../contentstack-export/src/export/modules/taxonomies.ts | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/packages/contentstack-export/src/export/modules/taxonomies.ts b/packages/contentstack-export/src/export/modules/taxonomies.ts index dc0de7378d..d61da35f21 100644 --- a/packages/contentstack-export/src/export/modules/taxonomies.ts +++ b/packages/contentstack-export/src/export/modules/taxonomies.ts @@ -173,10 +173,12 @@ export default class ExportTaxonomies extends BaseClass { handleErrorMsg(err: any) { if (err?.errorMessage) { - log(this.exportConfig, `Failed to export. ${err.errorMessage}`, 'error'); + log(this.exportConfig, `Failed to export! ${err.errorMessage}`, 'error'); } else if (err?.message) { const errorMsg = err?.errors?.taxonomy || err?.errors?.term || err?.message; - log(this.exportConfig, `Failed to export. ${errorMsg}`, 'error'); + log(this.exportConfig, `Failed to export! ${errorMsg}`, 'error'); + }else{ + log(this.exportConfig, `Failed to export! ${err}`, 'error'); } } } From 6f2716b791830da280cd25161ace04cd584a6867 Mon Sep 17 00:00:00 2001 From: Aman Kumar Date: Thu, 12 Oct 2023 10:09:22 +0530 Subject: [PATCH 39/45] fix: handle parent_uid error in term creation --- .../src/import/modules/taxonomies.ts | 23 +++++++++++++++---- 1 file changed, 19 insertions(+), 4 deletions(-) diff --git a/packages/contentstack-import/src/import/modules/taxonomies.ts b/packages/contentstack-import/src/import/modules/taxonomies.ts index 2ff886609c..8855c16c48 100644 --- a/packages/contentstack-import/src/import/modules/taxonomies.ts +++ b/packages/contentstack-import/src/import/modules/taxonomies.ts @@ -8,7 +8,6 @@ import BaseClass, { ApiOptions } from './base-class'; import { log, formatError, fsUtil, fileHelper } from '../../utils'; import { ModuleClassParams, TaxonomiesConfig, TermsConfig } from '../../types'; - export default class ImportTaxonomies extends BaseClass { private taxonomiesMapperDirPath: string; private taxonomiesFolderPath: string; @@ -121,7 +120,7 @@ export default class ImportTaxonomies extends BaseClass { reject: onReject, resolve: onSuccess, entity: 'create-taxonomies', - includeParamOnCompletion: true + includeParamOnCompletion: true, }, concurrencyLimit: this.importConfig.concurrency || this.importConfig.fetchConcurrency || 1, }, @@ -205,7 +204,7 @@ export default class ImportTaxonomies extends BaseClass { reject: onReject, resolve: onSuccess, entity: 'create-terms', - includeParamOnCompletion: true + includeParamOnCompletion: true, }, concurrencyLimit: this.importConfig.concurrency || this.importConfig.fetchConcurrency || 1, }, @@ -223,7 +222,23 @@ export default class ImportTaxonomies extends BaseClass { */ serializeTerms(apiOptions: ApiOptions): ApiOptions { const { apiData: term } = apiOptions; - apiOptions.apiData = term; + const {parent_uid, taxonomy_uid} = term; + + //check whether parent term exists or not in taxonomy + if (parent_uid !== null) { + if (!this.termsSuccess[taxonomy_uid][parent_uid]) { + log( + this.importConfig, + `Parent term '${term?.parent_uid}' doesn't exist! Skipping '${term.name}' to avoid buggy term.`, + 'info', + ); + apiOptions.apiData = undefined; + } else { + apiOptions.apiData = term; + } + } else { + apiOptions.apiData = term; + } return apiOptions; } From a49a7308ca17b1cf8dd5bdea1fd4258410e7ba77 Mon Sep 17 00:00:00 2001 From: Aman Kumar Date: Thu, 12 Oct 2023 12:16:39 +0530 Subject: [PATCH 40/45] refactor: error message --- packages/contentstack-export/src/config/index.ts | 4 ++-- packages/contentstack-import/src/import/modules/taxonomies.ts | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/contentstack-export/src/config/index.ts b/packages/contentstack-export/src/config/index.ts index 19e9935daa..c00f30b709 100644 --- a/packages/contentstack-export/src/config/index.ts +++ b/packages/contentstack-export/src/config/index.ts @@ -163,12 +163,12 @@ const config: DefaultConfig = { taxonomies: { dirName: 'taxonomies', fileName: 'taxonomies.json', - invalidKeys: ['created_at', 'updated_at', 'created_by', 'updated_by'], + invalidKeys: ['updated_at', 'created_by', 'updated_by'], }, terms: { dirName: 'terms', fileName: 'terms.json', - invalidKeys: ['created_at', 'updated_at', 'created_by', 'updated_by'], + invalidKeys: ['updated_at', 'created_by', 'updated_by'], }, }, languagesCode: [ diff --git a/packages/contentstack-import/src/import/modules/taxonomies.ts b/packages/contentstack-import/src/import/modules/taxonomies.ts index 8855c16c48..23d98e8804 100644 --- a/packages/contentstack-import/src/import/modules/taxonomies.ts +++ b/packages/contentstack-import/src/import/modules/taxonomies.ts @@ -229,7 +229,7 @@ export default class ImportTaxonomies extends BaseClass { if (!this.termsSuccess[taxonomy_uid][parent_uid]) { log( this.importConfig, - `Parent term '${term?.parent_uid}' doesn't exist! Skipping '${term.name}' to avoid buggy term.`, + `Parent term '${term?.parent_uid}' does not exist! Skipping '${term.uid}' creation to avoid further issues.`, 'info', ); apiOptions.apiData = undefined; From fd0361477a7c395e9bec724f06ad983ce0dfba20 Mon Sep 17 00:00:00 2001 From: Aman Kumar Date: Thu, 12 Oct 2023 12:26:18 +0530 Subject: [PATCH 41/45] refactor: taxonomy export & import error msg --- .../src/export/modules/taxonomies.ts | 3 +-- .../src/import/modules/taxonomies.ts | 26 +++++++++---------- 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/packages/contentstack-export/src/export/modules/taxonomies.ts b/packages/contentstack-export/src/export/modules/taxonomies.ts index d61da35f21..f18dfac9c0 100644 --- a/packages/contentstack-export/src/export/modules/taxonomies.ts +++ b/packages/contentstack-export/src/export/modules/taxonomies.ts @@ -97,9 +97,8 @@ export default class ExportTaxonomies extends BaseClass { sanitizeTaxonomiesAttribs(taxonomies: Record[]) { for (let index = 0; index < taxonomies?.length; index++) { const taxonomyUID = taxonomies[index].uid; - const taxonomyName = taxonomies[index]?.name; this.taxonomies[taxonomyUID] = omit(taxonomies[index], this.taxonomiesConfig.invalidKeys); - log(this.exportConfig, `'${taxonomyName}' taxonomy exported successfully!`, 'success'); + log(this.exportConfig, `'${taxonomyUID}' taxonomy exported successfully!`, 'success'); } } diff --git a/packages/contentstack-import/src/import/modules/taxonomies.ts b/packages/contentstack-import/src/import/modules/taxonomies.ts index 23d98e8804..91d2ca0e87 100644 --- a/packages/contentstack-import/src/import/modules/taxonomies.ts +++ b/packages/contentstack-import/src/import/modules/taxonomies.ts @@ -94,20 +94,20 @@ export default class ImportTaxonomies extends BaseClass { this.taxonomyUIDs = keys(this.taxonomies); const onSuccess = ({ response }: any) => { - const { uid, name } = response; + const { uid } = response; this.taxonomiesSuccess[uid] = pick(response, ['name', 'description']); - log(this.importConfig, `Taxonomy '${name}' imported successfully!`, 'success'); + log(this.importConfig, `Taxonomy '${uid}' imported successfully!`, 'success'); }; const onReject = ({ error, apiData }: any) => { const err = error?.message ? JSON.parse(error.message) : error; - const { name } = apiData; + const { uid } = apiData; if (err?.errors?.taxonomy) { - this.taxonomiesFailed[apiData.uid] = apiData; - log(this.importConfig, `Taxonomy '${name}' failed to be import! ${err.errors.taxonomy}`, 'error'); + this.taxonomiesFailed[uid] = apiData; + log(this.importConfig, `Taxonomy '${uid}' failed to be import! ${err.errors.taxonomy}`, 'error'); } else { this.taxonomiesFailed[apiData.uid] = apiData; - log(this.importConfig, `Taxonomy '${name}' failed to be import! ${formatError(error)}`, 'error'); + log(this.importConfig, `Taxonomy '${uid}' failed to be import! ${formatError(error)}`, 'error'); } }; @@ -166,23 +166,23 @@ export default class ImportTaxonomies extends BaseClass { } const onSuccess = ({ response, apiData: { taxonomy_uid } = { taxonomy_uid: null } }: any) => { - const { uid, name } = response; + const { uid } = response; if (!this.termsSuccess[taxonomy_uid]) this.termsSuccess[taxonomy_uid] = {}; this.termsSuccess[taxonomy_uid][uid] = pick(response, ['name']); - log(this.importConfig, `Term '${name}' imported successfully!`, 'success'); + log(this.importConfig, `Term '${uid}' imported successfully!`, 'success'); }; const onReject = ({ error, apiData }: any) => { - const { taxonomy_uid, name } = apiData; + const { taxonomy_uid, uid } = apiData; if (!this.termsFailed[taxonomy_uid]) this.termsFailed[taxonomy_uid] = {}; const err = error?.message ? JSON.parse(error.message) : error; if (err?.errors?.term) { - this.termsFailed[taxonomy_uid][apiData.uid] = apiData; - log(this.importConfig, `Term '${name}' failed to be import! ${err.errors.term}`, 'error'); + this.termsFailed[taxonomy_uid][uid] = apiData; + log(this.importConfig, `Term '${uid}' failed to be import! ${err.errors.term}`, 'error'); } else { - this.termsFailed[taxonomy_uid][apiData.uid] = apiData; - log(this.importConfig, `Term '${name}' failed to be import! ${formatError(error)}`, 'error'); + this.termsFailed[taxonomy_uid][uid] = apiData; + log(this.importConfig, `Term '${uid}' failed to be import! ${formatError(error)}`, 'error'); } }; From 1fd30ededdafde8fb8266a93bd9847e0acb58fd6 Mon Sep 17 00:00:00 2001 From: Aman Kumar Date: Wed, 18 Oct 2023 14:49:31 +0530 Subject: [PATCH 42/45] refactor: log warning msg, update sdk version & handle taxonomies query --- package-lock.json | 24 ++++++------ .../src/util/config.js | 2 +- .../src/util/index.js | 13 +++---- .../contentstack-export/src/config/index.ts | 2 +- .../src/export/modules/taxonomies.ts | 11 +++--- packages/contentstack-import/package.json | 2 +- .../src/import/modules/content-types.ts | 2 +- .../src/import/modules/entries.ts | 2 +- .../contentstack-import/src/utils/logger.ts | 12 ++++-- .../src/utils/taxonomies-helper.ts | 38 ++++++++++++++----- packages/contentstack-utilities/package.json | 2 +- packages/contentstack/package.json | 2 +- pnpm-lock.yaml | 16 ++++---- 13 files changed, 75 insertions(+), 53 deletions(-) diff --git a/package-lock.json b/package-lock.json index e47ba377ed..f8e46afb30 100644 --- a/package-lock.json +++ b/package-lock.json @@ -811,9 +811,9 @@ } }, "node_modules/@contentstack/management": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/@contentstack/management/-/management-1.11.0.tgz", - "integrity": "sha512-tv4At2Q5iGgkzL1MFGil/o36URKZfO6DY/KtpNJFYjmTHitZNv7uotH8OXkOPBMxB4xz58SG58lWB6fNTkLrpw==", + "version": "1.12.0", + "resolved": "https://registry.npmjs.org/@contentstack/management/-/management-1.12.0.tgz", + "integrity": "sha512-+L+WVhSYEtfdG9v794TjLT8Fd6fCB8meqoho666mg1kNufzXcsqr7hjubX5cSL7GcZFdKntkDpZ2RaOnTHReJg==", "dependencies": { "axios": "^1.5.1", "form-data": "^3.0.1", @@ -22783,7 +22783,7 @@ "@contentstack/cli-launch": "~1.0.12", "@contentstack/cli-migration": "~1.3.13", "@contentstack/cli-utilities": "~1.5.3", - "@contentstack/management": "~1.11.0", + "@contentstack/management": "~1.12.0", "@oclif/core": "^2.9.3", "@oclif/plugin-help": "^5", "@oclif/plugin-not-found": "^2.4.0", @@ -24614,7 +24614,7 @@ "dependencies": { "@contentstack/cli-command": "~1.2.13", "@contentstack/cli-utilities": "~1.5.3", - "@contentstack/management": "~1.11.0", + "@contentstack/management": "~1.12.0", "@oclif/core": "^2.9.3", "big-json": "^3.2.0", "bluebird": "^3.7.2", @@ -25400,7 +25400,7 @@ "version": "1.5.3", "license": "MIT", "dependencies": { - "@contentstack/management": "~1.11.0", + "@contentstack/management": "~1.12.0", "@oclif/core": "^2.9.3", "axios": "1.3.4", "chalk": "^4.0.0", @@ -27013,7 +27013,7 @@ "@contentstack/cli-launch": "~1.0.12", "@contentstack/cli-migration": "~1.3.13", "@contentstack/cli-utilities": "~1.5.3", - "@contentstack/management": "~1.11.0", + "@contentstack/management": "~1.12.0", "@oclif/core": "^2.9.3", "@oclif/plugin-help": "^5", "@oclif/plugin-not-found": "^2.4.0", @@ -28140,7 +28140,7 @@ "requires": { "@contentstack/cli-command": "~1.2.13", "@contentstack/cli-utilities": "~1.5.3", - "@contentstack/management": "~1.11.0", + "@contentstack/management": "~1.12.0", "@oclif/core": "^2.9.3", "@oclif/test": "^1.2.6", "@types/big-json": "^3.2.0", @@ -29017,7 +29017,7 @@ "@contentstack/cli-utilities": { "version": "file:packages/contentstack-utilities", "requires": { - "@contentstack/management": "~1.11.0", + "@contentstack/management": "~1.12.0", "@oclif/core": "^2.9.3", "@oclif/test": "^2.2.10", "@types/chai": "^4.2.18", @@ -29871,9 +29871,9 @@ } }, "@contentstack/management": { - "version": "1.11.0", - "resolved": "https://registry.npmjs.org/@contentstack/management/-/management-1.11.0.tgz", - "integrity": "sha512-tv4At2Q5iGgkzL1MFGil/o36URKZfO6DY/KtpNJFYjmTHitZNv7uotH8OXkOPBMxB4xz58SG58lWB6fNTkLrpw==", + "version": "1.12.0", + "resolved": "https://registry.npmjs.org/@contentstack/management/-/management-1.12.0.tgz", + "integrity": "sha512-+L+WVhSYEtfdG9v794TjLT8Fd6fCB8meqoho666mg1kNufzXcsqr7hjubX5cSL7GcZFdKntkDpZ2RaOnTHReJg==", "requires": { "axios": "^1.5.1", "form-data": "^3.0.1", diff --git a/packages/contentstack-export-to-csv/src/util/config.js b/packages/contentstack-export-to-csv/src/util/config.js index 64f986352c..8690274366 100644 --- a/packages/contentstack-export-to-csv/src/util/config.js +++ b/packages/contentstack-export-to-csv/src/util/config.js @@ -2,7 +2,7 @@ module.exports = { limit:100, cancelString: 'Cancel and Exit', exportEntries: 'Export entries to a .CSV file', - exportUsers: "Export organization users' data to a .CSV file", + exportUsers: "Export organization users data to a .CSV file", exportTaxonomies: 'Export taxonomies to a .CSV file', adminError: "Unable to export data. Make sure you're an admin or owner of this organization", organizationNameRegex: /\'/, diff --git a/packages/contentstack-export-to-csv/src/util/index.js b/packages/contentstack-export-to-csv/src/util/index.js index fb1cc21ceb..6396e16d51 100644 --- a/packages/contentstack-export-to-csv/src/util/index.js +++ b/packages/contentstack-export-to-csv/src/util/index.js @@ -699,12 +699,11 @@ function wait(time) { */ async function getAllTaxonomies(payload, skip = 0, taxonomies = []) { payload['type'] = 'taxonomies'; - const response = await taxonomySDKHandler(payload, skip); - if (response) { + const { items, count } = await taxonomySDKHandler(payload, skip); + if (items) { skip += payload.limit; - //TODO - replace response.taxonomies with items - taxonomies = [...taxonomies, ...response.taxonomies]; - if (skip >= response?.count) { + taxonomies = [...taxonomies, ...items]; + if (skip >= count) { return taxonomies; } else { return getAllTaxonomies(payload, skip, taxonomies); @@ -765,11 +764,10 @@ async function taxonomySDKHandler(payload, skip) { switch (type) { case 'taxonomies': - //TODO - replace count with find return await stackAPIClient .taxonomy() .query(queryParams) - .count() + .find() .then((data) => data) .catch((err) => handleErrorMsg(err)); case 'taxonomy': @@ -824,6 +822,7 @@ function formatTermsOfTaxonomyData(terms, taxonomyUID) { UID: term.uid, Name: term.name, 'Parent UID': term.parent_uid, + Depth: term.depth }); }); return formattedTerms; diff --git a/packages/contentstack-export/src/config/index.ts b/packages/contentstack-export/src/config/index.ts index c00f30b709..75fd504a73 100644 --- a/packages/contentstack-export/src/config/index.ts +++ b/packages/contentstack-export/src/config/index.ts @@ -168,7 +168,7 @@ const config: DefaultConfig = { terms: { dirName: 'terms', fileName: 'terms.json', - invalidKeys: ['updated_at', 'created_by', 'updated_by'], + invalidKeys: ['updated_at', 'created_by', 'updated_by', 'stackHeaders', 'urlPath'], }, }, languagesCode: [ diff --git a/packages/contentstack-export/src/export/modules/taxonomies.ts b/packages/contentstack-export/src/export/modules/taxonomies.ts index f18dfac9c0..19f1448b01 100644 --- a/packages/contentstack-export/src/export/modules/taxonomies.ts +++ b/packages/contentstack-export/src/export/modules/taxonomies.ts @@ -66,17 +66,16 @@ export default class ExportTaxonomies extends BaseClass { if (skip) { this.qs.skip = skip; } - //TODO - replace count with find await this.stack .taxonomy() .query(this.qs) - .count() + .find() .then(async (data: any) => { - const { taxonomies, count } = data; - const taxonomiesCount = count !== undefined ? count : taxonomies?.length; + const { items, count } = data; + const taxonomiesCount = count !== undefined ? count : items?.length; - if (taxonomies?.length) { - this.sanitizeTaxonomiesAttribs(taxonomies); + if (items?.length) { + this.sanitizeTaxonomiesAttribs(items); skip += this.taxonomiesConfig.limit || 100; if (skip >= taxonomiesCount) { return; diff --git a/packages/contentstack-import/package.json b/packages/contentstack-import/package.json index b0582e60a2..7c7d2150eb 100644 --- a/packages/contentstack-import/package.json +++ b/packages/contentstack-import/package.json @@ -7,7 +7,7 @@ "dependencies": { "@contentstack/cli-command": "~1.2.13", "@contentstack/cli-utilities": "~1.5.3", - "@contentstack/management": "~1.11.0", + "@contentstack/management": "~1.12.0", "@oclif/core": "^2.9.3", "big-json": "^3.2.0", "bluebird": "^3.7.2", diff --git a/packages/contentstack-import/src/import/modules/content-types.ts b/packages/contentstack-import/src/import/modules/content-types.ts index 049fd0eedb..0ca255da31 100644 --- a/packages/contentstack-import/src/import/modules/content-types.ts +++ b/packages/contentstack-import/src/import/modules/content-types.ts @@ -190,7 +190,7 @@ export default class ContentTypesImport extends BaseClass { this.fieldRules.push(contentType.uid); } //will remove taxonomy if taxonomy doesn't exists in stack - lookUpTaxonomy(contentType.schema, this.taxonomies); + lookUpTaxonomy(this.importConfig, contentType.schema, this.taxonomies); lookupExtension( this.importConfig, contentType.schema, diff --git a/packages/contentstack-import/src/import/modules/entries.ts b/packages/contentstack-import/src/import/modules/entries.ts index a886687b7d..bcfd111657 100644 --- a/packages/contentstack-import/src/import/modules/entries.ts +++ b/packages/contentstack-import/src/import/modules/entries.ts @@ -372,7 +372,7 @@ export default class EntriesImport extends BaseClass { entry = removeEntryRefsFromJSONRTE(entry, contentType.schema); } //will remove term if term doesn't exists in taxonomy - lookUpTerms(contentType?.schema, entry, this.taxonomies); + lookUpTerms(contentType?.schema, entry, this.taxonomies, this.importConfig); // will replace all old asset uid/urls with new ones entry = lookupAssets( { diff --git a/packages/contentstack-import/src/utils/logger.ts b/packages/contentstack-import/src/utils/logger.ts index d6ca174a75..de906a7e6f 100644 --- a/packages/contentstack-import/src/utils/logger.ts +++ b/packages/contentstack-import/src/utils/logger.ts @@ -81,7 +81,12 @@ function init(_logPath: string) { logger = winston.createLogger({ transports: [ new winston.transports.File(successTransport), - new winston.transports.Console({ format: winston.format.simple() }), + new winston.transports.Console({ + format: winston.format.combine( + winston.format.simple(), + winston.format.colorize({ all: true, colors: { warn: 'yellow', info: 'white' } }), + ), + }), ], levels: myCustomLevels.levels, }); @@ -109,7 +114,7 @@ function init(_logPath: string) { logger.log('info', logString); } }, - warn: function () { + warn: function (message: any) { let args = slice.call(arguments); let logString = returnString(args); if (logString) { @@ -138,7 +143,8 @@ export const log = async (config: ImportConfig, message: any, type: string) => { // ignoring the type argument, as we are not using it to create a logfile anymore if (type !== 'error') { // removed type argument from init method - init(config.data).log(message); + if (type === 'warn') init(config.data).warn(message); //logged warning message in log file + else init(config.data).log(message); } else { init(config.data).error(message); } diff --git a/packages/contentstack-import/src/utils/taxonomies-helper.ts b/packages/contentstack-import/src/utils/taxonomies-helper.ts index 7d62242927..09755fc2e2 100644 --- a/packages/contentstack-import/src/utils/taxonomies-helper.ts +++ b/packages/contentstack-import/src/utils/taxonomies-helper.ts @@ -1,18 +1,24 @@ /** * taxonomy lookup */ -import { cliux } from '@contentstack/cli-utilities'; +import { log } from './'; +import { ImportConfig } from '../types'; /** * check and remove if referenced taxonomy doesn't exists in stack * @param {any} schema content type schema * @param {Record} taxonomies created taxonomies + * @param {ImportConfig} importConfig */ -export const lookUpTaxonomy = function (schema: any, taxonomies: Record) { +export const lookUpTaxonomy = function (importConfig: ImportConfig, schema: any, taxonomies: Record) { for (let i in schema) { if (schema[i].data_type === 'taxonomy') { const taxonomyFieldData = schema[i].taxonomies as Record[]; - const { updatedTaxonomyData, isTaxonomyFieldRemoved } = verifyAndRemoveTaxonomy(taxonomyFieldData, taxonomies); + const { updatedTaxonomyData, isTaxonomyFieldRemoved } = verifyAndRemoveTaxonomy( + taxonomyFieldData, + taxonomies, + importConfig, + ); //Handle API error -> The 'taxonomies' property must have atleast one taxonomy object. Remove taxonomy field from schema. if (isTaxonomyFieldRemoved) { @@ -28,11 +34,13 @@ export const lookUpTaxonomy = function (schema: any, taxonomies: Record[]} taxonomyFieldData * @param {Record} taxonomies created taxonomies + * @param {ImportConfig} importConfig * @returns */ const verifyAndRemoveTaxonomy = function ( taxonomyFieldData: Record[], taxonomies: Record, + importConfig: ImportConfig, ): { updatedTaxonomyData: Record[]; isTaxonomyFieldRemoved: boolean; @@ -44,16 +52,22 @@ const verifyAndRemoveTaxonomy = function ( if (taxonomies === undefined || !taxonomies.hasOwnProperty(taxonomyData?.taxonomy_uid)) { // remove taxonomy from taxonomies field data with warning if respective taxonomy doesn't exists - cliux.print(`Taxonomy '${taxonomyData?.taxonomy_uid}' does not exist. Removing the data from the taxonomies field`, { - color: 'yellow', - }); + log( + importConfig, + `Taxonomy '${taxonomyData?.taxonomy_uid}' does not exist. Removing the data from the taxonomies field`, + 'warn', + ); taxonomyFieldData.splice(index, 1); --index; } } if (!taxonomyFieldData?.length) { - cliux.print('Taxonomy does not exist. Removing the field from content type', { color: 'yellow' }); + log( + importConfig, + 'Taxonomy does not exist. Removing the field from content type', + 'warn', + ); isTaxonomyFieldRemoved = true; } @@ -68,16 +82,18 @@ const verifyAndRemoveTaxonomy = function ( * @param {Record[]} ctSchema content type schema * @param {any} entry * @param {Record} taxonomiesAndTermData created taxonomies and terms + * @param {ImportConfig} importConfig */ export const lookUpTerms = function ( ctSchema: Record[], entry: any, taxonomiesAndTermData: Record, + importConfig: ImportConfig, ) { for (let index = 0; index < ctSchema?.length; index++) { if (ctSchema[index].data_type === 'taxonomy') { const taxonomyFieldData = entry[ctSchema[index].uid]; - const updatedTaxonomyData = verifyAndRemoveTerms(taxonomyFieldData, taxonomiesAndTermData); + const updatedTaxonomyData = verifyAndRemoveTerms(taxonomyFieldData, taxonomiesAndTermData, importConfig); entry[ctSchema[index].uid] = updatedTaxonomyData; } } @@ -87,11 +103,13 @@ export const lookUpTerms = function ( * verify and remove referenced term with warning from respective entry * @param {Record[]} taxonomyFieldData entry taxonomies data * @param {Record} taxonomiesAndTermData created taxonomies and terms + * @param {ImportConfig} importConfig * @returns { Record[]} */ const verifyAndRemoveTerms = function ( taxonomyFieldData: Record[], taxonomiesAndTermData: Record, + importConfig: ImportConfig, ): Record[] { for (let index = 0; index < taxonomyFieldData?.length; index++) { const taxonomyData = taxonomyFieldData[index]; @@ -104,11 +122,11 @@ const verifyAndRemoveTerms = function ( (taxonomiesAndTermData.hasOwnProperty(taxUID) && !taxonomiesAndTermData[taxUID].hasOwnProperty(termUID)) ) { // remove term from taxonomies field data with warning if respective term doesn't exists - cliux.print(`Term '${termUID}' does not exist. Removing it from taxonomy - '${taxUID}'`, { color: 'yellow' }); + log(importConfig, `Term '${termUID}' does not exist. Removing it from taxonomy - '${taxUID}'`, 'warn'); taxonomyFieldData.splice(index, 1); --index; } } return taxonomyFieldData; -}; +}; \ No newline at end of file diff --git a/packages/contentstack-utilities/package.json b/packages/contentstack-utilities/package.json index 4fc63b60f8..05811e26ef 100644 --- a/packages/contentstack-utilities/package.json +++ b/packages/contentstack-utilities/package.json @@ -32,7 +32,7 @@ "author": "contentstack", "license": "MIT", "dependencies": { - "@contentstack/management": "~1.11.0", + "@contentstack/management": "~1.12.0", "@oclif/core": "^2.9.3", "axios": "1.3.4", "chalk": "^4.0.0", diff --git a/packages/contentstack/package.json b/packages/contentstack/package.json index 60a71fcfa8..0c8eaf86fa 100755 --- a/packages/contentstack/package.json +++ b/packages/contentstack/package.json @@ -38,7 +38,7 @@ "@contentstack/cli-launch": "~1.0.12", "@contentstack/cli-migration": "~1.3.13", "@contentstack/cli-utilities": "~1.5.3", - "@contentstack/management": "~1.11.0", + "@contentstack/management": "~1.12.0", "@oclif/core": "^2.9.3", "@oclif/plugin-help": "^5", "@oclif/plugin-not-found": "^2.4.0", diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 5c5f64c15d..a8bf456c02 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -26,7 +26,7 @@ importers: '@contentstack/cli-launch': ~1.0.12 '@contentstack/cli-migration': ~1.3.13 '@contentstack/cli-utilities': ~1.5.3 - '@contentstack/management': ~1.11.0 + '@contentstack/management': ~1.12.0 '@oclif/core': ^2.9.3 '@oclif/plugin-help': ^5 '@oclif/plugin-not-found': ^2.4.0 @@ -80,7 +80,7 @@ importers: '@contentstack/cli-launch': link:../contentstack-launch '@contentstack/cli-migration': link:../contentstack-migration '@contentstack/cli-utilities': link:../contentstack-utilities - '@contentstack/management': 1.11.0_debug@4.3.4 + '@contentstack/management': 1.12.0_debug@4.3.4 '@oclif/core': 2.15.0_4qcp7qp4jxxdgb4qbxgwox4hwq '@oclif/plugin-help': 5.2.14_4qcp7qp4jxxdgb4qbxgwox4hwq '@oclif/plugin-not-found': 2.4.0_4qcp7qp4jxxdgb4qbxgwox4hwq @@ -718,7 +718,7 @@ importers: specifiers: '@contentstack/cli-command': ~1.2.13 '@contentstack/cli-utilities': ~1.5.3 - '@contentstack/management': ~1.11.0 + '@contentstack/management': ~1.12.0 '@oclif/core': ^2.9.3 '@oclif/test': ^1.2.6 '@types/big-json': ^3.2.0 @@ -758,7 +758,7 @@ importers: dependencies: '@contentstack/cli-command': link:../contentstack-command '@contentstack/cli-utilities': link:../contentstack-utilities - '@contentstack/management': 1.11.0_debug@4.3.4 + '@contentstack/management': 1.12.0_debug@4.3.4 '@oclif/core': 2.9.4_4qcp7qp4jxxdgb4qbxgwox4hwq big-json: 3.2.0 bluebird: 3.7.2 @@ -1025,7 +1025,7 @@ importers: packages/contentstack-utilities: specifiers: - '@contentstack/management': ~1.11.0 + '@contentstack/management': ~1.12.0 '@oclif/core': ^2.9.3 '@oclif/test': ^2.2.10 '@types/chai': ^4.2.18 @@ -1069,7 +1069,7 @@ importers: winston: ^3.7.2 xdg-basedir: ^4.0.0 dependencies: - '@contentstack/management': 1.11.0_debug@4.3.4 + '@contentstack/management': 1.12.0_debug@4.3.4 '@oclif/core': 2.9.4_4qcp7qp4jxxdgb4qbxgwox4hwq axios: 1.3.4_debug@4.3.4 chalk: 4.1.2 @@ -1556,8 +1556,8 @@ packages: uuid: 8.3.2 dev: false - /@contentstack/management/1.11.0_debug@4.3.4: - resolution: {integrity: sha512-tv4At2Q5iGgkzL1MFGil/o36URKZfO6DY/KtpNJFYjmTHitZNv7uotH8OXkOPBMxB4xz58SG58lWB6fNTkLrpw==} + /@contentstack/management/1.12.0_debug@4.3.4: + resolution: {integrity: sha512-+L+WVhSYEtfdG9v794TjLT8Fd6fCB8meqoho666mg1kNufzXcsqr7hjubX5cSL7GcZFdKntkDpZ2RaOnTHReJg==} engines: {node: '>=8.0.0'} dependencies: axios: 1.5.1_debug@4.3.4 From 0228e73ad28ce07611f45d5e9f77625552888d71 Mon Sep 17 00:00:00 2001 From: Aman Kumar Date: Wed, 18 Oct 2023 15:43:20 +0530 Subject: [PATCH 43/45] refactor: replace spread operator with push --- packages/contentstack-export-to-csv/src/util/index.js | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/contentstack-export-to-csv/src/util/index.js b/packages/contentstack-export-to-csv/src/util/index.js index 6396e16d51..b38c1c1d54 100644 --- a/packages/contentstack-export-to-csv/src/util/index.js +++ b/packages/contentstack-export-to-csv/src/util/index.js @@ -702,7 +702,7 @@ async function getAllTaxonomies(payload, skip = 0, taxonomies = []) { const { items, count } = await taxonomySDKHandler(payload, skip); if (items) { skip += payload.limit; - taxonomies = [...taxonomies, ...items]; + taxonomies.push(...items); if (skip >= count) { return taxonomies; } else { @@ -725,7 +725,7 @@ async function getAllTermsOfTaxonomy(payload, skip = 0, terms = []) { const { items, count } = await taxonomySDKHandler(payload, skip); if (items) { skip += payload.limit; - terms = [...terms, ...items]; + terms.push(...items); if (skip >= count) { return terms; } else { From 0d19525bd71811bb4f25bba2ea41b1e30b9c2ae5 Mon Sep 17 00:00:00 2001 From: Aman Kumar Date: Thu, 26 Oct 2023 17:19:04 +0530 Subject: [PATCH 44/45] version bumped --- package-lock.json | 60 +++++++++---------- packages/contentstack-bootstrap/README.md | 2 +- packages/contentstack-bootstrap/package.json | 4 +- packages/contentstack-clone/README.md | 2 +- packages/contentstack-clone/package.json | 6 +- .../contentstack-export-to-csv/package.json | 2 +- packages/contentstack-export/README.md | 2 +- packages/contentstack-export/package.json | 2 +- packages/contentstack-import/README.md | 2 +- packages/contentstack-import/package.json | 2 +- packages/contentstack-migration/README.md | 2 +- packages/contentstack-migration/package.json | 2 +- packages/contentstack-seed/package.json | 4 +- packages/contentstack/README.md | 33 ++++++---- packages/contentstack/package.json | 16 ++--- pnpm-lock.yaml | 22 +++---- 16 files changed, 86 insertions(+), 77 deletions(-) diff --git a/package-lock.json b/package-lock.json index 89995648fc..560081d042 100644 --- a/package-lock.json +++ b/package-lock.json @@ -22764,24 +22764,24 @@ }, "packages/contentstack": { "name": "@contentstack/cli", - "version": "1.10.0", + "version": "1.11.0", "license": "MIT", "dependencies": { "@contentstack/cli-audit": "~1.2.0", "@contentstack/cli-auth": "~1.3.15", - "@contentstack/cli-cm-bootstrap": "~1.6.0", + "@contentstack/cli-cm-bootstrap": "~1.6.1", "@contentstack/cli-cm-branches": "~1.0.15", "@contentstack/cli-cm-bulk-publish": "~1.3.13", - "@contentstack/cli-cm-clone": "~1.6.0", - "@contentstack/cli-cm-export": "~1.9.2", - "@contentstack/cli-cm-export-to-csv": "~1.4.4", - "@contentstack/cli-cm-import": "~1.10.0", + "@contentstack/cli-cm-clone": "~1.7.0", + "@contentstack/cli-cm-export": "~1.10.0", + "@contentstack/cli-cm-export-to-csv": "~1.5.0", + "@contentstack/cli-cm-import": "~1.11.0", "@contentstack/cli-cm-migrate-rte": "~1.4.13", - "@contentstack/cli-cm-seed": "~1.6.0", + "@contentstack/cli-cm-seed": "~1.6.1", "@contentstack/cli-command": "~1.2.14", "@contentstack/cli-config": "~1.4.13", "@contentstack/cli-launch": "~1.0.13", - "@contentstack/cli-migration": "~1.3.14", + "@contentstack/cli-migration": "~1.4.0", "@contentstack/cli-utilities": "~1.5.4", "@contentstack/management": "~1.12.0", "@oclif/core": "^2.9.3", @@ -23522,10 +23522,10 @@ }, "packages/contentstack-bootstrap": { "name": "@contentstack/cli-cm-bootstrap", - "version": "1.6.0", + "version": "1.6.1", "license": "MIT", "dependencies": { - "@contentstack/cli-cm-seed": "~1.6.0", + "@contentstack/cli-cm-seed": "~1.6.1", "@contentstack/cli-command": "~1.2.14", "@contentstack/cli-utilities": "~1.5.4", "inquirer": "8.2.4", @@ -23711,12 +23711,12 @@ }, "packages/contentstack-clone": { "name": "@contentstack/cli-cm-clone", - "version": "1.6.0", + "version": "1.7.0", "license": "MIT", "dependencies": { "@colors/colors": "^1.5.0", - "@contentstack/cli-cm-export": "~1.9.0", - "@contentstack/cli-cm-import": "~1.10.0", + "@contentstack/cli-cm-export": "~1.10.0", + "@contentstack/cli-cm-import": "~1.11.0", "@contentstack/cli-command": "~1.2.14", "@contentstack/cli-utilities": "~1.5.4", "async": "^3.2.4", @@ -24135,7 +24135,7 @@ }, "packages/contentstack-export": { "name": "@contentstack/cli-cm-export", - "version": "1.9.2", + "version": "1.10.0", "license": "MIT", "dependencies": { "@contentstack/cli-command": "~1.2.14", @@ -24185,7 +24185,7 @@ }, "packages/contentstack-export-to-csv": { "name": "@contentstack/cli-cm-export-to-csv", - "version": "1.4.4", + "version": "1.5.0", "license": "MIT", "dependencies": { "@contentstack/cli-command": "~1.2.14", @@ -24609,7 +24609,7 @@ }, "packages/contentstack-import": { "name": "@contentstack/cli-cm-import", - "version": "1.10.0", + "version": "1.11.0", "license": "MIT", "dependencies": { "@contentstack/cli-command": "~1.2.14", @@ -25271,7 +25271,7 @@ }, "packages/contentstack-migration": { "name": "@contentstack/cli-migration", - "version": "1.3.14", + "version": "1.4.0", "license": "MIT", "dependencies": { "@contentstack/cli-command": "~1.2.14", @@ -25304,10 +25304,10 @@ }, "packages/contentstack-seed": { "name": "@contentstack/cli-cm-seed", - "version": "1.6.0", + "version": "1.6.1", "license": "MIT", "dependencies": { - "@contentstack/cli-cm-import": "~1.10.0", + "@contentstack/cli-cm-import": "~1.11.0", "@contentstack/cli-command": "~1.2.14", "@contentstack/cli-utilities": "~1.5.4", "axios": "1.3.4", @@ -26999,19 +26999,19 @@ "requires": { "@contentstack/cli-audit": "~1.2.0", "@contentstack/cli-auth": "~1.3.15", - "@contentstack/cli-cm-bootstrap": "~1.6.0", + "@contentstack/cli-cm-bootstrap": "~1.6.1", "@contentstack/cli-cm-branches": "~1.0.15", "@contentstack/cli-cm-bulk-publish": "~1.3.13", - "@contentstack/cli-cm-clone": "~1.6.0", - "@contentstack/cli-cm-export": "~1.9.2", - "@contentstack/cli-cm-export-to-csv": "~1.4.4", - "@contentstack/cli-cm-import": "~1.10.0", + "@contentstack/cli-cm-clone": "~1.7.0", + "@contentstack/cli-cm-export": "~1.10.0", + "@contentstack/cli-cm-export-to-csv": "~1.5.0", + "@contentstack/cli-cm-import": "~1.11.0", "@contentstack/cli-cm-migrate-rte": "~1.4.13", - "@contentstack/cli-cm-seed": "~1.6.0", + "@contentstack/cli-cm-seed": "~1.6.1", "@contentstack/cli-command": "~1.2.14", "@contentstack/cli-config": "~1.4.13", "@contentstack/cli-launch": "~1.0.13", - "@contentstack/cli-migration": "~1.3.14", + "@contentstack/cli-migration": "~1.4.0", "@contentstack/cli-utilities": "~1.5.4", "@contentstack/management": "~1.12.0", "@oclif/core": "^2.9.3", @@ -27560,7 +27560,7 @@ "@contentstack/cli-cm-bootstrap": { "version": "file:packages/contentstack-bootstrap", "requires": { - "@contentstack/cli-cm-seed": "~1.6.0", + "@contentstack/cli-cm-seed": "~1.6.1", "@contentstack/cli-command": "~1.2.14", "@contentstack/cli-utilities": "~1.5.4", "@oclif/test": "^2.2.10", @@ -27714,8 +27714,8 @@ "version": "file:packages/contentstack-clone", "requires": { "@colors/colors": "^1.5.0", - "@contentstack/cli-cm-export": "~1.9.0", - "@contentstack/cli-cm-import": "~1.10.0", + "@contentstack/cli-cm-export": "~1.10.0", + "@contentstack/cli-cm-import": "~1.11.0", "@contentstack/cli-command": "~1.2.14", "@contentstack/cli-utilities": "~1.5.4", "@oclif/test": "^1.2.7", @@ -28289,7 +28289,7 @@ "@contentstack/cli-cm-seed": { "version": "file:packages/contentstack-seed", "requires": { - "@contentstack/cli-cm-import": "~1.10.0", + "@contentstack/cli-cm-import": "~1.11.0", "@contentstack/cli-command": "~1.2.14", "@contentstack/cli-utilities": "~1.5.4", "@oclif/plugin-help": "^5.1.19", diff --git a/packages/contentstack-bootstrap/README.md b/packages/contentstack-bootstrap/README.md index dff552cae8..50a4f8f2e7 100644 --- a/packages/contentstack-bootstrap/README.md +++ b/packages/contentstack-bootstrap/README.md @@ -15,7 +15,7 @@ $ npm install -g @contentstack/cli-cm-bootstrap $ csdx COMMAND running command... $ csdx (--version) -@contentstack/cli-cm-bootstrap/1.6.0 darwin-arm64 node-v20.8.0 +@contentstack/cli-cm-bootstrap/1.6.1 darwin-arm64 node-v20.8.0 $ csdx --help [COMMAND] USAGE $ csdx COMMAND diff --git a/packages/contentstack-bootstrap/package.json b/packages/contentstack-bootstrap/package.json index e8914a7b3e..144b233512 100644 --- a/packages/contentstack-bootstrap/package.json +++ b/packages/contentstack-bootstrap/package.json @@ -1,7 +1,7 @@ { "name": "@contentstack/cli-cm-bootstrap", "description": "Bootstrap contentstack apps", - "version": "1.6.0", + "version": "1.6.1", "author": "Contentstack", "bugs": "https://github.com/contentstack/cli/issues", "scripts": { @@ -17,7 +17,7 @@ "test:report": "nyc --reporter=lcov mocha \"test/**/*.test.js\"" }, "dependencies": { - "@contentstack/cli-cm-seed": "~1.6.0", + "@contentstack/cli-cm-seed": "~1.6.1", "@contentstack/cli-command": "~1.2.14", "@contentstack/cli-utilities": "~1.5.4", "inquirer": "8.2.4", diff --git a/packages/contentstack-clone/README.md b/packages/contentstack-clone/README.md index 648300f4db..b93bca2f5e 100644 --- a/packages/contentstack-clone/README.md +++ b/packages/contentstack-clone/README.md @@ -16,7 +16,7 @@ $ npm install -g @contentstack/cli-cm-clone $ csdx COMMAND running command... $ csdx (--version) -@contentstack/cli-cm-clone/1.6.0 darwin-arm64 node-v20.8.0 +@contentstack/cli-cm-clone/1.7.0 darwin-arm64 node-v20.8.0 $ csdx --help [COMMAND] USAGE $ csdx COMMAND diff --git a/packages/contentstack-clone/package.json b/packages/contentstack-clone/package.json index d4b345e29f..14c7e4bd1a 100644 --- a/packages/contentstack-clone/package.json +++ b/packages/contentstack-clone/package.json @@ -1,12 +1,12 @@ { "name": "@contentstack/cli-cm-clone", "description": "Contentstack stack clone plugin", - "version": "1.6.0", + "version": "1.7.0", "author": "Contentstack", "bugs": "https://github.com/rohitmishra209/cli-cm-clone/issues", "dependencies": { - "@contentstack/cli-cm-export": "~1.9.0", - "@contentstack/cli-cm-import": "~1.10.0", + "@contentstack/cli-cm-export": "~1.10.0", + "@contentstack/cli-cm-import": "~1.11.0", "@contentstack/cli-command": "~1.2.14", "@contentstack/cli-utilities": "~1.5.4", "@colors/colors": "^1.5.0", diff --git a/packages/contentstack-export-to-csv/package.json b/packages/contentstack-export-to-csv/package.json index 06d99f739a..fcbb2c40fe 100644 --- a/packages/contentstack-export-to-csv/package.json +++ b/packages/contentstack-export-to-csv/package.json @@ -1,7 +1,7 @@ { "name": "@contentstack/cli-cm-export-to-csv", "description": "Export entities to csv", - "version": "1.4.4", + "version": "1.5.0", "author": "Abhinav Gupta @abhinav-from-contentstack", "bugs": "https://github.com/contentstack/cli/issues", "dependencies": { diff --git a/packages/contentstack-export/README.md b/packages/contentstack-export/README.md index c36ef67f18..ac60d614d2 100755 --- a/packages/contentstack-export/README.md +++ b/packages/contentstack-export/README.md @@ -48,7 +48,7 @@ $ npm install -g @contentstack/cli-cm-export $ csdx COMMAND running command... $ csdx (--version) -@contentstack/cli-cm-export/1.9.2 darwin-arm64 node-v20.8.0 +@contentstack/cli-cm-export/1.10.0 darwin-arm64 node-v20.8.0 $ csdx --help [COMMAND] USAGE $ csdx COMMAND diff --git a/packages/contentstack-export/package.json b/packages/contentstack-export/package.json index b0c89fbd78..33b67d5bec 100644 --- a/packages/contentstack-export/package.json +++ b/packages/contentstack-export/package.json @@ -1,7 +1,7 @@ { "name": "@contentstack/cli-cm-export", "description": "Contentstack CLI plugin to export content from stack", - "version": "1.9.2", + "version": "1.10.0", "author": "Contentstack", "bugs": "https://github.com/contentstack/cli/issues", "dependencies": { diff --git a/packages/contentstack-import/README.md b/packages/contentstack-import/README.md index d8d1ef8968..960b49f1de 100644 --- a/packages/contentstack-import/README.md +++ b/packages/contentstack-import/README.md @@ -47,7 +47,7 @@ $ npm install -g @contentstack/cli-cm-import $ csdx COMMAND running command... $ csdx (--version) -@contentstack/cli-cm-import/1.10.0 darwin-arm64 node-v20.8.0 +@contentstack/cli-cm-import/1.11.0 darwin-arm64 node-v20.8.0 $ csdx --help [COMMAND] USAGE $ csdx COMMAND diff --git a/packages/contentstack-import/package.json b/packages/contentstack-import/package.json index cf40a453d2..678ae5eaae 100644 --- a/packages/contentstack-import/package.json +++ b/packages/contentstack-import/package.json @@ -1,7 +1,7 @@ { "name": "@contentstack/cli-cm-import", "description": "Contentstack CLI plugin to import content into stack", - "version": "1.10.0", + "version": "1.11.0", "author": "Contentstack", "bugs": "https://github.com/contentstack/cli/issues", "dependencies": { diff --git a/packages/contentstack-migration/README.md b/packages/contentstack-migration/README.md index fbed0cb23b..ddeacecf2a 100644 --- a/packages/contentstack-migration/README.md +++ b/packages/contentstack-migration/README.md @@ -21,7 +21,7 @@ $ npm install -g @contentstack/cli-migration $ csdx COMMAND running command... $ csdx (--version) -@contentstack/cli-migration/1.3.14 darwin-arm64 node-v20.8.0 +@contentstack/cli-migration/1.4.0 darwin-arm64 node-v20.8.0 $ csdx --help [COMMAND] USAGE $ csdx COMMAND diff --git a/packages/contentstack-migration/package.json b/packages/contentstack-migration/package.json index 61be6b7dc9..2257fcab4a 100644 --- a/packages/contentstack-migration/package.json +++ b/packages/contentstack-migration/package.json @@ -1,6 +1,6 @@ { "name": "@contentstack/cli-migration", - "version": "1.3.14", + "version": "1.4.0", "author": "@contentstack", "bugs": "https://github.com/contentstack/cli/issues", "dependencies": { diff --git a/packages/contentstack-seed/package.json b/packages/contentstack-seed/package.json index 9db67abe81..3a39f4158d 100644 --- a/packages/contentstack-seed/package.json +++ b/packages/contentstack-seed/package.json @@ -1,11 +1,11 @@ { "name": "@contentstack/cli-cm-seed", "description": "create a Stack from existing content types, entries, assets, etc.", - "version": "1.6.0", + "version": "1.6.1", "author": "Contentstack", "bugs": "https://github.com/contentstack/cli/issues", "dependencies": { - "@contentstack/cli-cm-import": "~1.10.0", + "@contentstack/cli-cm-import": "~1.11.0", "@contentstack/cli-command": "~1.2.14", "@contentstack/cli-utilities": "~1.5.4", "axios": "1.3.4", diff --git a/packages/contentstack/README.md b/packages/contentstack/README.md index 1cabac530d..9650e540fe 100644 --- a/packages/contentstack/README.md +++ b/packages/contentstack/README.md @@ -18,7 +18,7 @@ $ npm install -g @contentstack/cli $ csdx COMMAND running command... $ csdx (--version|-v) -@contentstack/cli/1.10.0 darwin-arm64 node-v20.8.0 +@contentstack/cli/1.11.0 darwin-arm64 node-v20.8.0 $ csdx --help [COMMAND] USAGE $ csdx COMMAND @@ -2014,27 +2014,28 @@ EXAMPLES ## `csdx cm:export-to-csv` -Export entries or organization users to csv using this command +Export entries, taxonomies, terms or organization users to csv using this command ``` USAGE - $ csdx cm:export-to-csv [--action entries|users] [-a ] [--org ] [-n ] [-k ] [--org-name - ] [--locale ] [--content-type ] [--branch ] + $ csdx cm:export-to-csv [--action entries|users|taxonomies] [-a ] [--org ] [-n ] [-k ] + [--org-name ] [--locale ] [--content-type ] [--branch ] [--taxonomy-uid ] FLAGS -a, --alias= Alias of the management token -k, --stack-api-key= API key of the source stack -n, --stack-name= Name of the stack that needs to be created as csv filename. --action=