Skip to content

Commit

Permalink
Merge pull request #1644 from contentstack/feat/import-setup-asset-ma…
Browse files Browse the repository at this point in the history
…pper

asset mapper file creation
  • Loading branch information
shafeeqd959 authored Oct 25, 2024
2 parents 9294202 + 83f9970 commit de03d55
Show file tree
Hide file tree
Showing 6 changed files with 349 additions and 4 deletions.
4 changes: 3 additions & 1 deletion packages/contentstack-import-setup/src/config/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -31,11 +31,12 @@ const config: DefaultConfig = {
assets: {
dirName: 'assets',
fileName: 'assets.json',
fetchConcurrency: 5,
},
'content-types': {
dirName: 'content_types',
fileName: 'content_types.json',
dependencies: ['extensions', 'taxonomies'],
dependencies: ['assets', 'extensions', 'taxonomies'],
},
entries: {
dirName: 'entries',
Expand Down Expand Up @@ -71,6 +72,7 @@ const config: DefaultConfig = {
],
},
},
fetchConcurrency: 5,
};

export default config;
121 changes: 121 additions & 0 deletions packages/contentstack-import-setup/src/import/modules/assets.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,121 @@
import * as chalk from 'chalk';
import { log, fsUtil } from '../../utils';
import { join } from 'path';
import { AssetRecord, ImportConfig, ModuleClassParams } from '../../types';
import { isEmpty, orderBy, values } from 'lodash';
import { formatError, FsUtility } from '@contentstack/cli-utilities';
import BaseImportSetup from './base-setup';

export default class AssetImportSetup extends BaseImportSetup {
private assetsFilePath: string;
private assetUidMapper: Record<string, string>;
private assetUrlMapper: Record<string, string>;
private duplicateAssets: Record<string, string>;
private assetsConfig: ImportConfig['modules']['assets'];
private mapperDirPath: string;
private assetsFolderPath: string;
private assetUidMapperPath: string;
private assetUrlMapperPath: string;
private duplicateAssetPath: string;

constructor({ config, stackAPIClient, dependencies }: ModuleClassParams) {
super({ config, stackAPIClient, dependencies });
this.assetsFolderPath = join(this.config.contentDir, 'assets');
this.assetsFilePath = join(this.config.contentDir, 'assets', 'assets.json');
this.assetsConfig = config.modules.assets;
this.mapperDirPath = join(this.config.backupDir, 'mapper', 'assets');
this.assetUidMapperPath = join(this.config.backupDir, 'mapper', 'assets', 'uid-mapping.json');
this.assetUrlMapperPath = join(this.config.backupDir, 'mapper', 'assets', 'url-mapping.json');
this.duplicateAssetPath = join(this.config.backupDir, 'mapper', 'assets', 'duplicate-assets.json');
this.assetUidMapper = {};
this.assetUrlMapper = {};
this.duplicateAssets = {};
}

/**
* Start the asset import setup
* This method reads the assets from the content folder and generates a mapper file
* @returns {Promise<void>}
*/
async start() {
try {
fsUtil.makeDirectory(this.mapperDirPath);
await this.fetchAndMapAssets();
log(this.config, `Generated required setup files for asset`, 'success');
} catch (error) {
log(this.config, `Error generating asset mapper: ${formatError(error)}`, 'error');
}
}

/**
* @method importAssets
* @param {boolean} isVersion boolean
* @returns {Promise<void>} Promise<void>
*/
async fetchAndMapAssets(): Promise<void> {
const processName = 'mapping assets';
const indexFileName = 'assets.json';
const basePath = this.assetsFolderPath;
const fs = new FsUtility({ basePath, indexFileName });
const indexer = fs.indexFileContent;
const indexerCount = values(indexer).length;

const onSuccess = ({
response: { items = [] as AssetRecord[] } = {},
apiData: { uid, url, title } = undefined,
}: any) => {
if (items.length === 1) {
this.assetUidMapper[uid] = items[0].uid;
this.assetUrlMapper[url] = items[0].url;
log(this.config, `Mapped asset: '${title}'`, 'info');
} else if (items.length > 1) {
this.duplicateAssets[uid] = items.map((asset: any) => {
return { uid: asset.uid, title: asset.title, url: asset.url };
});
log(this.config, `Multiple assets found with title '${title}'`, 'error');
} else {
log(this.config, `Asset with title '${title}' not found in the stack!`, 'error');
}
};
const onReject = ({ error, apiData: { title } = undefined }: any) => {
log(this.config, `${title} asset mapping failed.!`, 'error');
log(this.config, formatError(error), 'error');
};

/* eslint-disable @typescript-eslint/no-unused-vars, guard-for-in */
for (const index in indexer) {
const chunk = await fs.readChunkFiles.next().catch((error) => {
log(this.config, error, 'error');
});

if (chunk) {
let apiContent = orderBy(values(chunk as Record<string, any>[]), '_version');

await this.makeConcurrentCall(
{
apiContent,
processName,
indexerCount,
currentIndexer: +index,
apiParams: {
reject: onReject,
resolve: onSuccess,
entity: 'fetch-assets',
includeParamOnCompletion: true,
},
concurrencyLimit: this.assetsConfig.fetchConcurrency,
},
undefined,
);
}
}

if (!isEmpty(this.assetUidMapper) || !isEmpty(this.assetUrlMapper)) {
fsUtil.writeFile(this.assetUidMapperPath, this.assetUidMapper);
fsUtil.writeFile(this.assetUrlMapperPath, this.assetUrlMapper);
}
if (!isEmpty(this.duplicateAssets)) {
fsUtil.writeFile(this.duplicateAssetPath, this.duplicateAssets);
}
}
}
177 changes: 176 additions & 1 deletion packages/contentstack-import-setup/src/import/modules/base-setup.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,6 @@
import { log, fsUtil } from '../../utils';
import { ImportConfig, ModuleClassParams } from '../../types';
import { ApiOptions, CustomPromiseHandler, EnvType, ImportConfig, ModuleClassParams } from '../../types';
import { chunk, entries, isEmpty, isEqual, last } from 'lodash';

export default class BaseImportSetup {
public config: ImportConfig;
Expand Down Expand Up @@ -30,4 +31,178 @@ export default class BaseImportSetup {
}
}
}

/**
* @method delay
* @param {number} ms number
* @returns {Promise} Promise<void>
*/
delay(ms: number): Promise<void> {
/* eslint-disable no-promise-executor-return */
return new Promise((resolve) => setTimeout(resolve, ms <= 0 ? 0 : ms));
}

/**
* @method makeConcurrentCall
* @param {Record<string, any>} env EnvType
* @param {CustomPromiseHandler} promisifyHandler CustomPromiseHandler
* @param {boolean} logBatchCompletionMsg boolean
* @returns {Promise} Promise<void>
*/
makeConcurrentCall(
env: EnvType,
promisifyHandler?: CustomPromiseHandler,
logBatchCompletionMsg = true,
): Promise<void> {
const {
apiParams,
apiContent,
processName,
indexerCount,
currentIndexer,
concurrencyLimit = this.config.fetchConcurrency,
} = env;

/* eslint-disable no-async-promise-executor */
return new Promise(async (resolve) => {
let batchNo = 0;
let isLastRequest = false;
const batches: Array<Record<string, any>> = chunk(apiContent, concurrencyLimit);

/* eslint-disable no-promise-executor-return */
if (isEmpty(batches)) return resolve();

for (const [batchIndex, batch] of entries(batches)) {
batchNo += 1;
const allPromise = [];
const start = Date.now();

for (const [index, element] of entries(batch)) {
let promise = Promise.resolve();
isLastRequest = isEqual(last(batch as ArrayLike<any>), element) && isEqual(last(batches), batch);

if (promisifyHandler instanceof Function) {
promise = promisifyHandler({
apiParams,
isLastRequest,
element,
index: Number(index),
batchIndex: Number(batchIndex),
});
} else if (apiParams) {
apiParams.apiData = element;
promise = this.makeAPICall(apiParams, isLastRequest);
}

allPromise.push(promise);
}

/* eslint-disable no-await-in-loop */
await Promise.allSettled(allPromise);

/* eslint-disable no-await-in-loop */
await this.logMsgAndWaitIfRequired(
processName,
start,
batches.length,
batchNo,
logBatchCompletionMsg,
indexerCount,
currentIndexer,
);

if (isLastRequest) resolve();
}
});
}

/**
* @method logMsgAndWaitIfRequired
* @param {string} processName string
* @param {number} start number
* @param {number} batchNo - number
* @returns {Promise} Promise<void>
*/
async logMsgAndWaitIfRequired(
processName: string,
start: number,
totelBatches: number,
batchNo: number,
logBatchCompletionMsg = true,
indexerCount?: number,
currentIndexer?: number,
): Promise<void> {
const end = Date.now();
const exeTime = end - start;

if (logBatchCompletionMsg) {
let batchMsg = '';
// info: Batch No. 20 of import assets is complete
if (currentIndexer) batchMsg += `Current chunk processing is (${currentIndexer}/${indexerCount})`;

log(this.config, `Batch No. (${batchNo}/${totelBatches}) of ${processName} is complete`, 'success');
}

// if (this.config.modules.assets.displayExecutionTime) {
// console.log(
// `Time taken to execute: ${exeTime} milliseconds; wait time: ${
// exeTime < 1000 ? 1000 - exeTime : 0
// } milliseconds`,
// );
// }

if (exeTime < 1000) await this.delay(1000 - exeTime);
}

/**
* @method makeAPICall
* @param {Record<string, any>} apiOptions - Api related params
* @param {Record<string, any>} isLastRequest - Boolean
* @return {Promise} Promise<void>
*/
makeAPICall(apiOptions: ApiOptions, isLastRequest = false): Promise<void> {
if (apiOptions.serializeData instanceof Function) {
apiOptions = apiOptions.serializeData(apiOptions);
}

const { uid, entity, reject, resolve, apiData, additionalInfo = {}, includeParamOnCompletion } = apiOptions;

const onSuccess = (response: any) =>
resolve({
response,
isLastRequest,
additionalInfo,
apiData: includeParamOnCompletion ? apiData : undefined,
});
const onReject = (error: Error) =>
reject({
error,
isLastRequest,
additionalInfo,
apiData: includeParamOnCompletion ? apiData : undefined,
});

if (!apiData) {
return Promise.resolve();
}
switch (entity) {
case 'fetch-assets':
return this.stackAPIClient
.asset()
.query({
query: {
$and: [
{ file_size: Number(apiData.file_size) },
{ filename: apiData.filename },
{ title: apiData.title },
],
},
})
.find()
.then(onSuccess)
.catch(onReject);
default:
return Promise.resolve();
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import { log, fsUtil } from '../../utils';
import { join } from 'path';
import { ImportConfig, ModuleClassParams } from '../../types';
import { isEmpty } from 'lodash';
import { formatError } from '@contentstack/cli-utilities';

export default class ExtensionImportSetup {
private config: ImportConfig;
Expand Down Expand Up @@ -48,12 +49,12 @@ export default class ExtensionImportSetup {

await fsUtil.writeFile(this.extUidMapperPath, this.extensionMapper);

log(this.config, `Generate required setup files for extension`, 'success');
log(this.config, `Generated required setup files for extension`, 'success');
} else {
log(this.config, 'No extensions found in the content folder!', 'error');
}
} catch (error) {
log(this.config, `Error generating extension mapper: ${error.message}`, 'error');
log(this.config, `Error generating extension mapper: ${formatError(error)}`, 'error');
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ export default interface DefaultConfig {
dirName: string;
fileName: string;
dependencies?: Modules[];
fetchConcurrency: number;
};
'content-types': {
dirName: string;
Expand Down Expand Up @@ -50,4 +51,5 @@ export default interface DefaultConfig {
invalidKeys: string[];
};
};
fetchConcurrency: number;
}
Loading

0 comments on commit de03d55

Please sign in to comment.