From 4b7b3f12239f37e195f12b8413a5ae0c3b6d84ee Mon Sep 17 00:00:00 2001 From: Richard Sustek Date: Mon, 9 Dec 2024 15:18:48 +0100 Subject: [PATCH 01/12] Creates 'ItemProcessingResult' to distinguish between success / failed processing requests --- lib/core/models/core.models.ts | 6 + lib/core/utils/processing-utils.ts | 49 +++++--- lib/export/context/export-context-fetcher.ts | 108 ++++++++++-------- lib/export/export-manager.ts | 104 +++++++++-------- lib/import/context/import-context-fetcher.ts | 18 ++- lib/import/import-manager.ts | 12 +- lib/import/import.models.ts | 23 +++- lib/import/importers/assets-importer.ts | 12 +- .../importers/content-items-importer.ts | 20 +--- .../importers/language-variant-importer.ts | 66 ++++++----- 10 files changed, 227 insertions(+), 191 deletions(-) diff --git a/lib/core/models/core.models.ts b/lib/core/models/core.models.ts index 4293a1a..1b7129b 100644 --- a/lib/core/models/core.models.ts +++ b/lib/core/models/core.models.ts @@ -145,3 +145,9 @@ export interface OriginalManagementError { }; }; } + +export type ItemProcessingResult = { + readonly inputItem: InputItem; + readonly outputItem: OutputItem | undefined; + readonly error?: unknown; +}; diff --git a/lib/core/utils/processing-utils.ts b/lib/core/utils/processing-utils.ts index 85f1dd3..5ec9a7e 100644 --- a/lib/core/utils/processing-utils.ts +++ b/lib/core/utils/processing-utils.ts @@ -1,6 +1,6 @@ import chalk from 'chalk'; import pLimit from 'p-limit'; -import { ItemInfo } from '../models/core.models.js'; +import { ItemInfo, ItemProcessingResult } from '../models/core.models.js'; import { LogSpinnerData, Logger } from '../models/log.models.js'; type ProcessSetAction = @@ -21,7 +21,7 @@ export async function processItemsAsync(data: { readonly parallelLimit: number; readonly processAsync: (item: Readonly, logSpinner: LogSpinnerData) => Promise>; readonly itemInfo: (item: Readonly) => ItemInfo; -}): Promise { +}): Promise[]> { if (!data.items.length) { return []; } @@ -32,21 +32,36 @@ export async function processItemsAsync(data: { const limit = pLimit(data.parallelLimit); let processedItemsCount: number = 1; - const requests: Promise[] = data.items.map((item) => + const requests: Promise>[] = data.items.map((item) => limit(() => { - return data.processAsync(item, logSpinner).then((output) => { - const itemInfo = data.itemInfo(item); - const prefix = getPercentagePrefix(processedItemsCount, data.items.length); + return data + .processAsync(item, logSpinner) + .then((output) => { + const itemInfo = data.itemInfo(item); + const prefix = getPercentagePrefix(processedItemsCount, data.items.length); - logSpinner({ - prefix: prefix, - message: itemInfo.title, - type: itemInfo.itemType - }); + logSpinner({ + prefix: prefix, + message: itemInfo.title, + type: itemInfo.itemType + }); - processedItemsCount++; - return output; - }); + processedItemsCount++; + return output; + }) + .then>((outputItem) => { + return { + inputItem: item, + outputItem: outputItem + }; + }) + .catch>((error) => { + return { + inputItem: item, + outputItem: undefined, + error: error + }; + }); }) ); @@ -58,11 +73,11 @@ export async function processItemsAsync(data: { }); // Only '' promises at a time - const outputItems = await Promise.all(requests); + const resultItems = await Promise.all(requests); - logSpinner({ type: 'info', message: `Completed '${chalk.yellow(data.action)}' (${outputItems.length})` }); + logSpinner({ type: 'info', message: `Completed '${chalk.yellow(data.action)}' (${resultItems.length})` }); - return outputItems; + return resultItems; }); } diff --git a/lib/export/context/export-context-fetcher.ts b/lib/export/context/export-context-fetcher.ts index f702123..abe9715 100644 --- a/lib/export/context/export-context-fetcher.ts +++ b/lib/export/context/export-context-fetcher.ts @@ -1,23 +1,23 @@ import { - WorkflowModels, - LanguageVariantModels, - ContentItemModels, AssetModels, CollectionModels, - LanguageModels + ContentItemModels, + LanguageModels, + LanguageVariantModels, + WorkflowModels } from '@kontent-ai/management-sdk'; import chalk from 'chalk'; import { - processItemsAsync, - runMapiRequestAsync, - is404Error, - ItemStateInSourceEnvironmentById, AssetStateInSourceEnvironmentById, + findRequired, FlattenedContentType, + is404Error, isNotUndefined, - managementClientUtils, + ItemStateInSourceEnvironmentById, LogSpinnerData, - findRequired, + managementClientUtils, + processItemsAsync, + runMapiRequestAsync, workflowHelper } from '../../core/index.js'; import { itemsExtractionProcessor } from '../../translation/index.js'; @@ -25,10 +25,10 @@ import { DefaultExportContextConfig, ExportContext, ExportContextEnvironmentData, - SourceExportItem, ExportItem, + ExportItemVersion, GetFlattenedElementByIds, - ExportItemVersion + SourceExportItem } from '../export.models.js'; import { throwErrorForItemRequest } from '../utils/export.utils.js'; @@ -234,44 +234,48 @@ export async function exportContextFetcherAsync(config: DefaultExportContextConf message: `Preparing '${chalk.yellow(config.exportItems.length.toString())}' items for export` }); - return await processItemsAsync({ - logger: config.logger, - action: 'Preparing content items & language variants', - parallelLimit: 1, - itemInfo: (input) => { - return { - title: `${input.itemCodename} (${input.languageCodename})`, - itemType: 'exportItem' - }; - }, - items: exportItems, - processAsync: async (requestItem, logSpinner) => { - const contentItem = await getContentItemAsync(requestItem, logSpinner); - const versions = await getExportItemVersionsAsync(requestItem, contentItem, logSpinner); - - // get shared attributes from any version - const anyVersion = versions[0]; - if (!anyVersion) { - throwErrorForItemRequest(requestItem, `Expected at least 1 version of the content item`); - } + return ( + await processItemsAsync({ + logger: config.logger, + action: 'Preparing content items & language variants', + parallelLimit: 1, + itemInfo: (input) => { + return { + title: `${input.itemCodename} (${input.languageCodename})`, + itemType: 'exportItem' + }; + }, + items: exportItems, + processAsync: async (requestItem, logSpinner) => { + const contentItem = await getContentItemAsync(requestItem, logSpinner); + const versions = await getExportItemVersionsAsync(requestItem, contentItem, logSpinner); + + // get shared attributes from any version + const anyVersion = versions[0]; + if (!anyVersion) { + throwErrorForItemRequest(requestItem, `Expected at least 1 version of the content item`); + } - const { collection, contentType, language, workflow } = validateExportItem({ - sourceItem: requestItem, - contentItem: contentItem, - languageVariant: anyVersion.languageVariant - }); + const { collection, contentType, language, workflow } = validateExportItem({ + sourceItem: requestItem, + contentItem: contentItem, + languageVariant: anyVersion.languageVariant + }); - return { - contentItem, - versions, - contentType, - requestItem, - workflow, - collection, - language - }; - } - }); + return { + contentItem, + versions, + contentType, + requestItem, + workflow, + collection, + language + }; + } + }) + ) + .map((m) => m.outputItem) + .filter(isNotUndefined); }; const getContentItemsByIdsAsync = async (itemIds: ReadonlySet): Promise[]> => { @@ -306,7 +310,9 @@ export async function exportContextFetcherAsync(config: DefaultExportContextConf } } }) - ).filter(isNotUndefined); + ) + .map((m) => m.outputItem) + .filter(isNotUndefined); }; const getAssetsByIdsAsync = async (itemIds: ReadonlySet): Promise[]> => { @@ -341,7 +347,9 @@ export async function exportContextFetcherAsync(config: DefaultExportContextConf } } }) - ).filter(isNotUndefined); + ) + .map((m) => m.outputItem) + .filter(isNotUndefined); }; const getItemStatesAsync = async (itemIds: ReadonlySet): Promise => { diff --git a/lib/export/export-manager.ts b/lib/export/export-manager.ts index 04d8c12..2d37b7c 100644 --- a/lib/export/export-manager.ts +++ b/lib/export/export-manager.ts @@ -175,58 +175,62 @@ export function exportManager(config: ExportConfig) { message: `Preparing to download '${chalk.yellow(assets.length.toString())}' assets` }); - return await processItemsAsync, MigrationAsset>({ - action: 'Downloading assets', - logger: logger, - parallelLimit: 5, - itemInfo: (input) => { - return { - title: input.codename, - itemType: 'asset' - }; - }, - items: assets, - processAsync: async (asset, logSpinner) => { - const assetCollection: Readonly | undefined = context.environmentData.collections.find( - (m) => m.id === asset.collection?.reference?.id - ); - const assetFolder: Readonly | undefined = context.environmentData.assetFolders.find( - (m) => m.id === asset.folder?.id - ); + return ( + await processItemsAsync, MigrationAsset>({ + action: 'Downloading assets', + logger: logger, + parallelLimit: 5, + itemInfo: (input) => { + return { + title: input.codename, + itemType: 'asset' + }; + }, + items: assets, + processAsync: async (asset, logSpinner) => { + const assetCollection: Readonly | undefined = context.environmentData.collections.find( + (m) => m.id === asset.collection?.reference?.id + ); + const assetFolder: Readonly | undefined = context.environmentData.assetFolders.find( + (m) => m.id === asset.folder?.id + ); - logSpinner({ - type: 'download', - message: `${asset.url}` - }); - - const migrationAsset: MigrationAsset = { - filename: asset.fileName, - title: asset.title ?? '', - codename: asset.codename, - binary_data: (await getBinaryDataFromUrlAsync(asset.url)).data, - collection: assetCollection ? { codename: assetCollection.codename } : undefined, - folder: assetFolder ? { codename: assetFolder.codename } : undefined, - descriptions: asset.descriptions.map((description) => { - const language = findRequired( - context.environmentData.languages, - (language) => language.id === description.language.id, - `Could not find language with id '${chalk.red(description.language.id)}' requested by asset '${chalk.red( - asset.codename - )}'` - ); - - return { - description: description.description ?? undefined, - language: { - codename: language.codename - } - }; - }) - }; + logSpinner({ + type: 'download', + message: `${asset.url}` + }); - return migrationAsset; - } - }); + const migrationAsset: MigrationAsset = { + filename: asset.fileName, + title: asset.title ?? '', + codename: asset.codename, + binary_data: (await getBinaryDataFromUrlAsync(asset.url)).data, + collection: assetCollection ? { codename: assetCollection.codename } : undefined, + folder: assetFolder ? { codename: assetFolder.codename } : undefined, + descriptions: asset.descriptions.map((description) => { + const language = findRequired( + context.environmentData.languages, + (language) => language.id === description.language.id, + `Could not find language with id '${chalk.red(description.language.id)}' requested by asset '${chalk.red( + asset.codename + )}'` + ); + + return { + description: description.description ?? undefined, + language: { + codename: language.codename + } + }; + }) + }; + + return migrationAsset; + } + }) + ) + .map((m) => m.outputItem) + .filter(isNotUndefined); }; return { diff --git a/lib/import/context/import-context-fetcher.ts b/lib/import/context/import-context-fetcher.ts index b91bf4a..b9f416d 100644 --- a/lib/import/context/import-context-fetcher.ts +++ b/lib/import/context/import-context-fetcher.ts @@ -1,4 +1,6 @@ import { AssetModels, ContentItemModels, LanguageVariantModels } from '@kontent-ai/management-sdk'; +import chalk from 'chalk'; +import { match } from 'ts-pattern'; import { AssetStateInTargetEnvironmentByCodename, ItemStateInTargetEnvironmentByCodename, @@ -18,10 +20,8 @@ import { runMapiRequestAsync, workflowHelper as workflowHelperInit } from '../../core/index.js'; -import { GetFlattenedElementByCodenames, ImportContext, ImportContextConfig, ImportContextEnvironmentData } from '../import.models.js'; import { ExtractItemByCodename, itemsExtractionProcessor } from '../../translation/index.js'; -import chalk from 'chalk'; -import { match } from 'ts-pattern'; +import { GetFlattenedElementByCodenames, ImportContext, ImportContextConfig, ImportContextEnvironmentData } from '../import.models.js'; interface LanguageVariantWrapper { readonly draftLanguageVariant: Readonly | undefined; @@ -186,7 +186,9 @@ export async function importContextFetcherAsync(config: ImportContextConfig) { }; } }) - ).filter(isNotUndefined); + ) + .map((m) => m.outputItem) + .filter(isNotUndefined); }; const getContentItemsByCodenamesAsync = async ( @@ -223,7 +225,9 @@ export async function importContextFetcherAsync(config: ImportContextConfig) { } } }) - ).filter(isNotUndefined); + ) + .map((m) => m.outputItem) + .filter(isNotUndefined); }; const getAssetsByCodenamesAsync = async (assetCodenames: ReadonlySet): Promise => { @@ -258,7 +262,9 @@ export async function importContextFetcherAsync(config: ImportContextConfig) { } } }) - ).filter(isNotUndefined); + ) + .map((m) => m.outputItem) + .filter(isNotUndefined); }; const getVariantState = (languageVariant: Readonly): LanguageVariantStateData => { diff --git a/lib/import/import-manager.ts b/lib/import/import-manager.ts index fa89151..674edca 100644 --- a/lib/import/import-manager.ts +++ b/lib/import/import-manager.ts @@ -1,8 +1,7 @@ -import { ContentItemModels, LanguageVariantModels, ManagementClient } from '@kontent-ai/management-sdk'; - +import { ManagementClient } from '@kontent-ai/management-sdk'; import { defaultExternalIdGenerator, getDefaultLogger, getMigrationManagementClient, Logger } from '../core/index.js'; import { importContextFetcherAsync } from './context/import-context-fetcher.js'; -import { ImportConfig, ImportContext, ImportResult } from './import.models.js'; +import { ImportConfig, ImportContext, ImportedItem, ImportedLanguageVariant, ImportResult } from './import.models.js'; import { assetsImporter } from './importers/assets-importer.js'; import { contentItemsImporter } from './importers/content-items-importer.js'; import { languageVariantImporter } from './importers/language-variant-importer.js'; @@ -29,7 +28,8 @@ export function importManager(config: ImportConfig) { logger: logger }).importAsync(); }; - const importContentItemsAsync = async (importContext: ImportContext): Promise[]> => { + + const importContentItemsAsync = async (importContext: ImportContext): Promise => { if (!importContext.categorizedImportData.contentItems.length) { logger.log({ type: 'info', @@ -46,8 +46,8 @@ export function importManager(config: ImportConfig) { const importLanguageVariantsAsync = async ( importContext: ImportContext, - contentItems: readonly Readonly[] - ): Promise[]> => { + contentItems: readonly ImportedItem[] + ): Promise => { if (!importContext.categorizedImportData.contentItems.length) { logger.log({ type: 'info', diff --git a/lib/import/import.models.ts b/lib/import/import.models.ts index f4d7912..c934519 100644 --- a/lib/import/import.models.ts +++ b/lib/import/import.models.ts @@ -14,6 +14,7 @@ import { ExternalIdGenerator, FlattenedContentType, FlattenedContentTypeElement, + ItemProcessingResult, ItemStateInTargetEnvironmentByCodename, LanguageVariantStateInTargetEnvironmentByCodename, Logger, @@ -78,9 +79,23 @@ export interface ImportConfig extends ManagementClientConfig { readonly logger?: Logger; } +export interface AssetToEdit { + migrationAsset: MigrationAsset; + targetAsset: Readonly; + replaceBinaryFile: boolean; +} + +export type ImportedItem = ItemProcessingResult>; +export type ImportedLanguageVariant = ItemProcessingResult< + MigrationItem, + readonly Readonly[] +>; +export type EditedAsset = ItemProcessingResult>; +export type ImportedAsset = ItemProcessingResult>; + export interface ImportResult { - readonly uploadedAssets: readonly Readonly[]; - readonly editedAssets: readonly Readonly[]; - readonly contentItems: readonly Readonly[]; - readonly languageVariants: readonly Readonly[]; + readonly uploadedAssets: readonly ImportedAsset[]; + readonly editedAssets: readonly EditedAsset[]; + readonly contentItems: readonly ImportedItem[]; + readonly languageVariants: readonly ImportedLanguageVariant[]; } diff --git a/lib/import/importers/assets-importer.ts b/lib/import/importers/assets-importer.ts index b6c5ff7..51ed484 100644 --- a/lib/import/importers/assets-importer.ts +++ b/lib/import/importers/assets-importer.ts @@ -13,13 +13,7 @@ import { runMapiRequestAsync } from '../../core/index.js'; import { shouldReplaceBinaryFile, shouldUpdateAsset } from '../comparers/asset-comparer.js'; -import { ImportContext, ImportResult } from '../import.models.js'; - -interface AssetToEdit { - migrationAsset: MigrationAsset; - targetAsset: Readonly; - replaceBinaryFile: boolean; -} +import { AssetToEdit, EditedAsset, ImportContext, ImportedAsset, ImportResult } from '../import.models.js'; export function assetsImporter(data: { readonly logger: Logger; @@ -70,7 +64,7 @@ export function assetsImporter(data: { .filter(isNotUndefined); }; - const editAssets = async (assetsToEdit: readonly AssetToEdit[]): Promise[]> => { + const editAssets = async (assetsToEdit: readonly AssetToEdit[]): Promise => { data.logger.log({ type: 'upsert', message: `Upserting '${chalk.yellow(assetsToEdit.length.toString())}' assets` @@ -189,7 +183,7 @@ export function assetsImporter(data: { }); }; - const uploadAssetsAsync = async (assetsToUpload: readonly MigrationAsset[]): Promise[]> => { + const uploadAssetsAsync = async (assetsToUpload: readonly MigrationAsset[]): Promise => { data.logger.log({ type: 'upload', message: `Uploading '${chalk.yellow(assetsToUpload.length.toString())}' assets` diff --git a/lib/import/importers/content-items-importer.ts b/lib/import/importers/content-items-importer.ts index 345c57d..9bd034d 100644 --- a/lib/import/importers/content-items-importer.ts +++ b/lib/import/importers/content-items-importer.ts @@ -1,14 +1,7 @@ import { ContentItemModels, ManagementClient } from '@kontent-ai/management-sdk'; -import { - Logger, - processItemsAsync, - runMapiRequestAsync, - MigrationItem, - LogSpinnerData, - findRequired -} from '../../core/index.js'; import chalk from 'chalk'; -import { ImportContext } from '../import.models.js'; +import { LogSpinnerData, Logger, MigrationItem, findRequired, processItemsAsync, runMapiRequestAsync } from '../../core/index.js'; +import { ImportContext, ImportedItem } from '../import.models.js'; export function contentItemsImporter(data: { readonly logger: Logger; @@ -26,8 +19,7 @@ export function contentItemsImporter(data: { ); return ( - migrationContentItem.system.name !== contentItem.name || - migrationContentItem.system.collection.codename !== collection.codename + migrationContentItem.system.name !== contentItem.name || migrationContentItem.system.collection.codename !== collection.codename ); }; @@ -35,9 +27,7 @@ export function contentItemsImporter(data: { logSpinner: LogSpinnerData, migrationContentItem: MigrationItem ): Promise<{ contentItem: Readonly; status: 'created' | 'itemAlreadyExists' }> => { - const itemStateInTargetEnv = data.importContext.getItemStateInTargetEnvironment( - migrationContentItem.system.codename - ); + const itemStateInTargetEnv = data.importContext.getItemStateInTargetEnvironment(migrationContentItem.system.codename); if (itemStateInTargetEnv.state === 'exists' && itemStateInTargetEnv.item) { return { @@ -111,7 +101,7 @@ export function contentItemsImporter(data: { return preparedContentItemResult.contentItem; }; - const importAsync = async (): Promise[]> => { + const importAsync = async (): Promise => { const contentItemsToImport = data.importContext.categorizedImportData.contentItems; data.logger.log({ diff --git a/lib/import/importers/language-variant-importer.ts b/lib/import/importers/language-variant-importer.ts index b74293a..656e978 100644 --- a/lib/import/importers/language-variant-importer.ts +++ b/lib/import/importers/language-variant-importer.ts @@ -1,4 +1,4 @@ -import { ContentItemModels, ElementContracts, LanguageVariantModels, ManagementClient, WorkflowModels } from '@kontent-ai/management-sdk'; +import { ElementContracts, LanguageVariantModels, ManagementClient, WorkflowModels } from '@kontent-ai/management-sdk'; import chalk from 'chalk'; import { match } from 'ts-pattern'; import { @@ -16,13 +16,13 @@ import { workflowHelper } from '../../core/index.js'; import { importTransforms } from '../../translation/index.js'; -import { ImportContext } from '../import.models.js'; +import { ImportContext, ImportedItem, ImportedLanguageVariant } from '../import.models.js'; import { throwErrorForMigrationItem } from '../utils/import.utils.js'; import { workflowImporter as workflowImporterInit } from './workflow-importer.js'; export function languageVariantImporter(config: { readonly logger: Logger; - readonly preparedContentItems: readonly ContentItemModels.ContentItem[]; + readonly preparedContentItems: readonly ImportedItem[]; readonly importContext: ImportContext; readonly client: Readonly; }) { @@ -37,7 +37,7 @@ export function languageVariantImporter(config: { readonly logSpinner: LogSpinnerData; readonly migrationItem: MigrationItem; readonly migrationItemVersion: MigrationItemVersion; - readonly preparedContentItem: Readonly; + readonly preparedContentItem: ImportedItem; }): Promise> => { return await runMapiRequestAsync({ logger: config.logger, @@ -45,7 +45,7 @@ export function languageVariantImporter(config: { return ( await config.client .upsertLanguageVariant() - .byItemCodename(data.preparedContentItem.codename) + .byItemCodename(data.preparedContentItem.inputItem.system.codename) .byLanguageCodename(data.migrationItem.system.language.codename) .withData(() => { return { @@ -110,7 +110,7 @@ export function languageVariantImporter(config: { readonly logSpinner: LogSpinnerData; readonly migrationItem: MigrationItem; readonly migrationItemVersion: MigrationItemVersion; - readonly preparedContentItem: Readonly; + readonly preparedContentItem: ImportedItem; readonly createNewVersion?: boolean; }): Promise> => { // validate workflow @@ -158,7 +158,7 @@ export function languageVariantImporter(config: { const importLanguageVariantAsync = async ( logSpinner: LogSpinnerData, migrationItem: MigrationItem, - preparedContentItem: Readonly + preparedContentItem: ImportedItem ): Promise => { const { draftVersion, publishedVersion } = categorizeVersions(migrationItem); @@ -297,7 +297,7 @@ export function languageVariantImporter(config: { return importTransformResult; }; - const importAsync = async (): Promise[]> => { + const importAsync = async (): Promise => { config.logger.log({ type: 'info', message: `Importing '${chalk.yellow( @@ -305,32 +305,30 @@ export function languageVariantImporter(config: { )}' language variants` }); - return ( - await processItemsAsync[]>({ - action: 'Importing language variants', - logger: config.logger, - parallelLimit: 1, - items: config.importContext.categorizedImportData.contentItems, - itemInfo: (input) => { - return { - itemType: 'languageVariant', - title: input.system.name, - partA: input.system.language.codename - }; - }, - processAsync: async (migrationItem, logSpinner) => { - const contentItem = findRequired( - config.preparedContentItems, - (item) => item.codename === migrationItem.system.codename, - `Missing content item with codename '${chalk.red( - migrationItem.system.codename - )}'. Content item should have been prepepared.` - ); - - return await importLanguageVariantAsync(logSpinner, migrationItem, contentItem); - } - }) - ).flatMap((m) => m.map((s) => s)); + return await processItemsAsync[]>({ + action: 'Importing language variants', + logger: config.logger, + parallelLimit: 1, + items: config.importContext.categorizedImportData.contentItems, + itemInfo: (input) => { + return { + itemType: 'languageVariant', + title: input.system.name, + partA: input.system.language.codename + }; + }, + processAsync: async (migrationItem, logSpinner) => { + const contentItem = findRequired( + config.preparedContentItems, + (item) => item.inputItem.system.codename === migrationItem.system.codename, + `Missing content item with codename '${chalk.red( + migrationItem.system.codename + )}'. Content item should have been prepepared.` + ); + + return await importLanguageVariantAsync(logSpinner, migrationItem, contentItem); + } + }); }; return { From ca0208ad5762255af693a4f526eb6592c3b333b1 Mon Sep 17 00:00:00 2001 From: Richard Sustek Date: Mon, 9 Dec 2024 15:29:33 +0100 Subject: [PATCH 02/12] Adds readonly modifiers --- lib/export/context/export-context-fetcher.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/export/context/export-context-fetcher.ts b/lib/export/context/export-context-fetcher.ts index abe9715..229b97f 100644 --- a/lib/export/context/export-context-fetcher.ts +++ b/lib/export/context/export-context-fetcher.ts @@ -173,11 +173,11 @@ export async function exportContextFetcherAsync(config: DefaultExportContextConf readonly contentItem: Readonly; readonly languageVariant: Readonly; }): { - collection: Readonly; - language: Readonly; - workflow: Readonly; - contentType: Readonly; - workflowStepCodename: string; + readonly collection: Readonly; + readonly language: Readonly; + readonly workflow: Readonly; + readonly contentType: Readonly; + readonly workflowStepCodename: string; } => { const collection = findRequired( environmentData.collections, From e9e5b9e2231dde8bd07ee32cf241e95d0057a97d Mon Sep 17 00:00:00 2001 From: Richard Sustek Date: Mon, 9 Dec 2024 15:34:23 +0100 Subject: [PATCH 03/12] Adds readonly modifiers --- lib/export/export-manager.ts | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/export/export-manager.ts b/lib/export/export-manager.ts index 2d37b7c..da938e4 100644 --- a/lib/export/export-manager.ts +++ b/lib/export/export-manager.ts @@ -124,10 +124,10 @@ export function exportManager(config: ExportConfig) { }; const getMigrationElementToStore = (data: { - context: ExportContext; - contentType: FlattenedContentType; - typeElement: FlattenedContentTypeElement; - exportElement: ElementModels.ContentItemElement; + readonly context: ExportContext; + readonly contentType: FlattenedContentType; + readonly typeElement: FlattenedContentTypeElement; + readonly exportElement: ElementModels.ContentItemElement; }): MigrationElementTransformData => { try { return exportTransforms[data.typeElement.type]({ From 14d084d0a7b24f4dd22f621b63721e6a8cb4e496 Mon Sep 17 00:00:00 2001 From: Richard Sustek Date: Mon, 9 Dec 2024 15:55:28 +0100 Subject: [PATCH 04/12] Adds success / failed items count + handle undefined case for processed items --- lib/core/utils/processing-utils.ts | 10 +++++++++- lib/export/context/export-context-fetcher.ts | 10 ++++++---- lib/import/context/import-context-fetcher.ts | 19 +++++++++++-------- 3 files changed, 26 insertions(+), 13 deletions(-) diff --git a/lib/core/utils/processing-utils.ts b/lib/core/utils/processing-utils.ts index 5ec9a7e..e098e3f 100644 --- a/lib/core/utils/processing-utils.ts +++ b/lib/core/utils/processing-utils.ts @@ -75,7 +75,15 @@ export async function processItemsAsync(data: { // Only '' promises at a time const resultItems = await Promise.all(requests); - logSpinner({ type: 'info', message: `Completed '${chalk.yellow(data.action)}' (${resultItems.length})` }); + const failedItemsCount = resultItems.filter((m) => !m.outputItem).length; + const failedText = failedItemsCount ? ` Failed (${chalk.red()}) items` : ``; + + logSpinner({ + type: 'info', + message: `Completed '${chalk.yellow(data.action)}'. Processed (${chalk.green( + resultItems.filter((m) => m.outputItem).length + )}) items.${failedText}` + }); return resultItems; }); diff --git a/lib/export/context/export-context-fetcher.ts b/lib/export/context/export-context-fetcher.ts index 229b97f..aeeaaa4 100644 --- a/lib/export/context/export-context-fetcher.ts +++ b/lib/export/context/export-context-fetcher.ts @@ -280,7 +280,7 @@ export async function exportContextFetcherAsync(config: DefaultExportContextConf const getContentItemsByIdsAsync = async (itemIds: ReadonlySet): Promise[]> => { return ( - await processItemsAsync | undefined>({ + await processItemsAsync | '404'>({ logger: config.logger, action: 'Fetching content items', parallelLimit: 1, @@ -306,18 +306,19 @@ export async function exportContextFetcherAsync(config: DefaultExportContextConf throw error; } - return undefined; + return '404'; } } }) ) .map((m) => m.outputItem) + .filter((m) => m !== '404') .filter(isNotUndefined); }; const getAssetsByIdsAsync = async (itemIds: ReadonlySet): Promise[]> => { return ( - await processItemsAsync | undefined>({ + await processItemsAsync | '404'>({ logger: config.logger, action: 'Fetching assets', parallelLimit: 1, @@ -343,12 +344,13 @@ export async function exportContextFetcherAsync(config: DefaultExportContextConf throw error; } - return undefined; + return '404'; } } }) ) .map((m) => m.outputItem) + .filter((m) => m !== '404') .filter(isNotUndefined); }; diff --git a/lib/import/context/import-context-fetcher.ts b/lib/import/context/import-context-fetcher.ts index b9f416d..6873042 100644 --- a/lib/import/context/import-context-fetcher.ts +++ b/lib/import/context/import-context-fetcher.ts @@ -150,7 +150,7 @@ export async function importContextFetcherAsync(config: ImportContextConfig) { migrationItems: readonly MigrationItem[] ): Promise => { return ( - await processItemsAsync({ + await processItemsAsync({ action: 'Fetching language variants', logger: config.logger, parallelLimit: 1, @@ -166,7 +166,7 @@ export async function importContextFetcherAsync(config: ImportContextConfig) { if (!latestLanguageVariant) { // there is neither published or draft version as latest version does not exist at all - return undefined; + return '404'; } if (workflowHelper.isPublishedStepById(latestLanguageVariant.workflow.stepIdentifier.id ?? '')) { @@ -188,6 +188,7 @@ export async function importContextFetcherAsync(config: ImportContextConfig) { }) ) .map((m) => m.outputItem) + .filter((m) => m !== '404') .filter(isNotUndefined); }; @@ -195,7 +196,7 @@ export async function importContextFetcherAsync(config: ImportContextConfig) { itemCodenames: ReadonlySet ): Promise => { return ( - await processItemsAsync | undefined>({ + await processItemsAsync | '404'>({ action: 'Fetching content items', logger: config.logger, parallelLimit: 1, @@ -221,18 +222,19 @@ export async function importContextFetcherAsync(config: ImportContextConfig) { throw error; } - return undefined; + return '404'; } } }) ) .map((m) => m.outputItem) - .filter(isNotUndefined); + .filter(isNotUndefined) + .filter((m) => m !== '404'); }; const getAssetsByCodenamesAsync = async (assetCodenames: ReadonlySet): Promise => { return ( - await processItemsAsync | undefined>({ + await processItemsAsync | '404'>({ action: 'Fetching assets', logger: config.logger, parallelLimit: 1, @@ -258,13 +260,14 @@ export async function importContextFetcherAsync(config: ImportContextConfig) { throw error; } - return undefined; + return '404'; } } }) ) .map((m) => m.outputItem) - .filter(isNotUndefined); + .filter(isNotUndefined) + .filter((m) => m !== '404'); }; const getVariantState = (languageVariant: Readonly): LanguageVariantStateData => { From 6f71ec683f51f07ad0e1748a7cde827f810fd187 Mon Sep 17 00:00:00 2001 From: Richard Sustek Date: Tue, 10 Dec 2024 10:24:39 +0100 Subject: [PATCH 05/12] Logs errors regarding failed items / assets / variants to console --- lib/core/logs/loggers.ts | 2 +- lib/core/models/log.models.ts | 4 +- lib/core/utils/processing-utils.ts | 4 +- lib/import/import-manager.ts | 64 +++++++++++++++++-- .../extraction/items-extraction.processor.ts | 6 +- 5 files changed, 66 insertions(+), 14 deletions(-) diff --git a/lib/core/logs/loggers.ts b/lib/core/logs/loggers.ts index 0f39f5e..9cb2035 100644 --- a/lib/core/logs/loggers.ts +++ b/lib/core/logs/loggers.ts @@ -1,7 +1,7 @@ import chalk, { ChalkInstance } from 'chalk'; +import { match, P } from 'ts-pattern'; import { Logger, LogSpinnerMessage } from '../models/log.models.js'; import { getCurrentEnvironment } from '../utils/global.utils.js'; -import { match, P } from 'ts-pattern'; const originalWarn = console.warn; diff --git a/lib/core/models/log.models.ts b/lib/core/models/log.models.ts index fdc7c31..d66b047 100644 --- a/lib/core/models/log.models.ts +++ b/lib/core/models/log.models.ts @@ -1,4 +1,4 @@ -import { MapiAction, MapiType, MigrationItemType } from '../index.js'; +import { LiteralUnion, MapiAction, MapiType, MigrationItemType } from '../index.js'; export type DebugType = | 'error' @@ -17,7 +17,7 @@ export type DebugType = | MapiAction; export interface LogMessage { - readonly type: DebugType; + readonly type: LiteralUnion; readonly message: string; } diff --git a/lib/core/utils/processing-utils.ts b/lib/core/utils/processing-utils.ts index e098e3f..0e4ec71 100644 --- a/lib/core/utils/processing-utils.ts +++ b/lib/core/utils/processing-utils.ts @@ -76,11 +76,11 @@ export async function processItemsAsync(data: { const resultItems = await Promise.all(requests); const failedItemsCount = resultItems.filter((m) => !m.outputItem).length; - const failedText = failedItemsCount ? ` Failed (${chalk.red()}) items` : ``; + const failedText = failedItemsCount ? ` Failed (${chalk.red(failedItemsCount)}) items` : ``; logSpinner({ type: 'info', - message: `Completed '${chalk.yellow(data.action)}'. Processed (${chalk.green( + message: `Completed '${chalk.yellow(data.action)}'. Successfully processed (${chalk.green( resultItems.filter((m) => m.outputItem).length )}) items.${failedText}` }); diff --git a/lib/import/import-manager.ts b/lib/import/import-manager.ts index 674edca..010fc09 100644 --- a/lib/import/import-manager.ts +++ b/lib/import/import-manager.ts @@ -1,7 +1,16 @@ import { ManagementClient } from '@kontent-ai/management-sdk'; -import { defaultExternalIdGenerator, getDefaultLogger, getMigrationManagementClient, Logger } from '../core/index.js'; +import chalk from 'chalk'; +import { defaultExternalIdGenerator, extractErrorData, getDefaultLogger, getMigrationManagementClient, Logger } from '../core/index.js'; import { importContextFetcherAsync } from './context/import-context-fetcher.js'; -import { ImportConfig, ImportContext, ImportedItem, ImportedLanguageVariant, ImportResult } from './import.models.js'; +import { + EditedAsset, + ImportConfig, + ImportContext, + ImportedAsset, + ImportedItem, + ImportedLanguageVariant, + ImportResult +} from './import.models.js'; import { assetsImporter } from './importers/assets-importer.js'; import { contentItemsImporter } from './importers/content-items-importer.js'; import { languageVariantImporter } from './importers/language-variant-importer.js'; @@ -63,6 +72,39 @@ export function importManager(config: ImportConfig) { }).importAsync(); }; + const getImportErrors = ( + contentItems: readonly ImportedItem[], + languageVariants: readonly ImportedLanguageVariant[], + importedAssets: readonly ImportedAsset[], + editedAssets: readonly EditedAsset[] + ): readonly string[] => { + return [ + ...editedAssets + .filter((m) => m.error) + .map( + (m) => + `Failed to edit asset '${chalk.yellow(m.inputItem.migrationAsset.codename)}': ${extractErrorData(m.error).message}` + ), + ...importedAssets + .filter((m) => m.error) + .map((m) => `Failed to upload asset '${chalk.yellow(m.inputItem.codename)}': ${extractErrorData(m.error).message}`), + ...contentItems + .filter((m) => m.error) + .map( + (m) => + `Failed to import content item '${chalk.yellow(m.inputItem.system.codename)}': ${extractErrorData(m.error).message}` + ), + ...languageVariants + .filter((m) => m.error) + .map( + (m) => + `Failed to import language variant '${chalk.yellow(m.inputItem.system.codename)}' in language '${chalk.yellow( + m.inputItem.system.language.codename + )}': ${extractErrorData(m.error).message}` + ) + ]; + }; + return { async importAsync(): Promise { const importContext = await ( @@ -83,10 +125,20 @@ export function importManager(config: ImportConfig) { // #3 Language variants const languageVariants = await importLanguageVariantsAsync(importContext, contentItems); - logger.log({ - type: 'completed', - message: `Finished import` - }); + const importErrors = getImportErrors(contentItems, languageVariants, uploadedAssets, editedAssets); + + if (importErrors.length) { + importErrors.forEach((error, index) => logger.log({ type: chalk.red(`#${index + 1}`), message: error })); + logger.log({ + type: 'completed', + message: `Finished import with '${chalk.red(importErrors.length)}' errors` + }); + } else { + logger.log({ + type: 'completed', + message: `Finished import` + }); + } return { editedAssets, diff --git a/lib/translation/extraction/items-extraction.processor.ts b/lib/translation/extraction/items-extraction.processor.ts index b4b2ebb..43ef75e 100644 --- a/lib/translation/extraction/items-extraction.processor.ts +++ b/lib/translation/extraction/items-extraction.processor.ts @@ -12,7 +12,7 @@ import { GetFlattenedElementByCodenames } from '../../import/index.js'; import { richTextProcessor } from '../index.js'; export interface ExtractItemById { - readonly elements: Readonly[]; + readonly elements: readonly Readonly[]; readonly contentTypeId: string; } @@ -95,7 +95,7 @@ export function itemsExtractionProcessor() { { itemIds: new Set(), assetIds: new Set() } ); - return { + return { itemIds: extractedIds.itemIds, assetIds: extractedIds.assetIds }; @@ -175,7 +175,7 @@ export function itemsExtractionProcessor() { } ); - return { + return { itemCodenames: extractedCodenames.itemCodenames, assetCodenames: extractedCodenames.assetCodenames }; From 779d62f65807208d0b0d41241f6da7b43a904603 Mon Sep 17 00:00:00 2001 From: Richard Sustek Date: Tue, 10 Dec 2024 15:29:44 +0100 Subject: [PATCH 06/12] Adds new 'createReportFile' config option for creating import report file within current folder --- lib/core/logs/loggers.ts | 4 + lib/core/models/log.models.ts | 2 +- lib/import/import-manager.ts | 198 +++++++++++++++++++++++++------- lib/import/import.models.ts | 7 +- lib/toolkit/import.ts | 18 ++- scripts/test/import-report.json | 1 + scripts/test/test-import.ts | 5 +- 7 files changed, 184 insertions(+), 51 deletions(-) create mode 100644 scripts/test/import-report.json diff --git a/lib/core/logs/loggers.ts b/lib/core/logs/loggers.ts index 9cb2035..771a45f 100644 --- a/lib/core/logs/loggers.ts +++ b/lib/core/logs/loggers.ts @@ -55,6 +55,10 @@ const defaultBrowserLogger: Logger = { }; function getLogDataMessage(data: LogSpinnerMessage): string { + if (!data.type) { + return data.message; + } + const color = match(data.type) .returnType() .with('info', () => chalk.cyan) diff --git a/lib/core/models/log.models.ts b/lib/core/models/log.models.ts index d66b047..3d79840 100644 --- a/lib/core/models/log.models.ts +++ b/lib/core/models/log.models.ts @@ -17,7 +17,7 @@ export type DebugType = | MapiAction; export interface LogMessage { - readonly type: LiteralUnion; + readonly type?: LiteralUnion; readonly message: string; } diff --git a/lib/import/import-manager.ts b/lib/import/import-manager.ts index 010fc09..1871f9e 100644 --- a/lib/import/import-manager.ts +++ b/lib/import/import-manager.ts @@ -2,19 +2,41 @@ import { ManagementClient } from '@kontent-ai/management-sdk'; import chalk from 'chalk'; import { defaultExternalIdGenerator, extractErrorData, getDefaultLogger, getMigrationManagementClient, Logger } from '../core/index.js'; import { importContextFetcherAsync } from './context/import-context-fetcher.js'; -import { - EditedAsset, - ImportConfig, - ImportContext, - ImportedAsset, - ImportedItem, - ImportedLanguageVariant, - ImportResult -} from './import.models.js'; +import { ImportConfig, ImportContext, ImportedItem, ImportedLanguageVariant, ImportResult } from './import.models.js'; import { assetsImporter } from './importers/assets-importer.js'; import { contentItemsImporter } from './importers/content-items-importer.js'; import { languageVariantImporter } from './importers/language-variant-importer.js'; +const reportFilename: string = `import-report.json`; + +type ReportResult = { + readonly errorsCount: number; + readonly assets: { + readonly count: number; + readonly successful: Array<{ readonly codename: string }>; + readonly failed: Array<{ readonly codename: string; readonly error: string }>; + }; + readonly languageVariants: { + readonly count: number; + readonly successful: Array<{ + readonly codename: string; + readonly language: { readonly codename: string }; + readonly type: { readonly codename: string }; + }>; + readonly failed: Array<{ + readonly codename: string; + readonly language: { readonly codename: string }; + readonly type: { readonly codename: string }; + readonly error: string; + }>; + }; + readonly contentItems: { + readonly count: number; + readonly successful: Array<{ readonly codename: string }>; + readonly failed: Array<{ readonly codename: string; readonly type: { readonly codename: string }; readonly error: string }>; + }; +}; + export function importManager(config: ImportConfig) { const logger: Logger = config.logger ?? getDefaultLogger(); const targetEnvironmentClient: Readonly = getMigrationManagementClient(config); @@ -72,37 +94,114 @@ export function importManager(config: ImportConfig) { }).importAsync(); }; - const getImportErrors = ( - contentItems: readonly ImportedItem[], - languageVariants: readonly ImportedLanguageVariant[], - importedAssets: readonly ImportedAsset[], - editedAssets: readonly EditedAsset[] - ): readonly string[] => { - return [ - ...editedAssets - .filter((m) => m.error) - .map( - (m) => - `Failed to edit asset '${chalk.yellow(m.inputItem.migrationAsset.codename)}': ${extractErrorData(m.error).message}` - ), - ...importedAssets - .filter((m) => m.error) - .map((m) => `Failed to upload asset '${chalk.yellow(m.inputItem.codename)}': ${extractErrorData(m.error).message}`), - ...contentItems - .filter((m) => m.error) - .map( - (m) => - `Failed to import content item '${chalk.yellow(m.inputItem.system.codename)}': ${extractErrorData(m.error).message}` - ), - ...languageVariants - .filter((m) => m.error) - .map( - (m) => - `Failed to import language variant '${chalk.yellow(m.inputItem.system.codename)}' in language '${chalk.yellow( - m.inputItem.system.language.codename - )}': ${extractErrorData(m.error).message}` - ) + const getReportResult = (importResult: ImportResult): ReportResult => { + return { + errorsCount: + importResult.editedAssets.filter((m) => !m.outputItem).length + + importResult.uploadedAssets.filter((m) => !m.outputItem).length + + importResult.contentItems.filter((m) => !m.outputItem).length + + importResult.languageVariants.filter((m) => !m.outputItem).length, + assets: { + count: importResult.uploadedAssets.length + importResult.editedAssets.length, + successful: [ + ...importResult.uploadedAssets.filter((m) => m.outputItem).map((m) => m.inputItem.codename), + ...importResult.editedAssets.filter((m) => m.outputItem).map((m) => m.inputItem.migrationAsset.codename) + ].map((m) => { + return { + codename: m + }; + }), + failed: [ + ...importResult.uploadedAssets + .filter((m) => !m.outputItem) + .map((m) => { + return { + codename: m.inputItem.codename, + error: extractErrorData(m.error).message + }; + }), + ...importResult.editedAssets + .filter((m) => !m.outputItem) + .map((m) => { + return { + codename: m.inputItem.migrationAsset.codename, + error: extractErrorData(m.error).message + }; + }) + ] + }, + contentItems: { + count: importResult.contentItems.length, + successful: importResult.contentItems + .filter((m) => m.outputItem) + .map((m) => { + return { + codename: m.inputItem.system.codename + }; + }), + failed: importResult.contentItems + .filter((m) => !m.outputItem) + .map((m) => { + return { + codename: m.inputItem.system.codename, + type: m.inputItem.system.type, + error: extractErrorData(m.error).message + }; + }) + }, + languageVariants: { + count: importResult.languageVariants.length, + successful: importResult.languageVariants + .filter((m) => m.outputItem) + .map((m) => { + return { + codename: m.inputItem.system.codename, + language: m.inputItem.system.language, + type: m.inputItem.system.type + }; + }), + failed: importResult.languageVariants + .filter((m) => !m.outputItem) + .map((m) => { + return { + codename: m.inputItem.system.codename, + language: m.inputItem.system.language, + type: m.inputItem.system.type, + error: extractErrorData(m.error).message + }; + }) + } + }; + }; + + const printReportToConsole = (reportResult: ReportResult, logger: Logger): void => { + const errors = [ + ...reportResult.assets.failed.map((m) => [ + `Object type: ${chalk.yellow('Asset')}`, + `Codename: ${chalk.yellow(m.codename)}`, + `${chalk.red(m.error)}` + ]), + ...reportResult.contentItems.failed.map((m) => [ + `Object type: ${chalk.yellow('Content item')}`, + `Codename:${chalk.yellow(m.codename)}`, + `Content Type: ${chalk.yellow(m.type.codename)}`, + `${chalk.red(m.error)}` + ]), + ...reportResult.languageVariants.failed.map((m) => [ + `Object type: ${chalk.yellow('Language variant')}`, + `Codename: ${chalk.yellow(m.codename)}`, + `Language: ${chalk.yellow(m.language.codename)}`, + `Content Type: ${chalk.yellow(m.type.codename)}`, + `${chalk.red(m.error)}` + ]) ]; + + errors.forEach((error, index) => { + logger.log({ message: `${chalk.red(`\nError #${index + 1}`)}` }); + error.forEach((m) => { + logger.log({ message: m }); + }); + }); }; return { @@ -125,13 +224,20 @@ export function importManager(config: ImportConfig) { // #3 Language variants const languageVariants = await importLanguageVariantsAsync(importContext, contentItems); - const importErrors = getImportErrors(contentItems, languageVariants, uploadedAssets, editedAssets); + const reportResult = getReportResult({ + contentItems, + editedAssets, + languageVariants, + uploadedAssets + }); - if (importErrors.length) { - importErrors.forEach((error, index) => logger.log({ type: chalk.red(`#${index + 1}`), message: error })); + if (reportResult.errorsCount) { + printReportToConsole(reportResult, logger); logger.log({ type: 'completed', - message: `Finished import with '${chalk.red(importErrors.length)}' errors` + message: `Finished import with '${chalk.red(reportResult.errorsCount)}' ${ + reportResult.errorsCount === 1 ? 'error' : 'errors' + }` }); } else { logger.log({ @@ -146,6 +252,12 @@ export function importManager(config: ImportConfig) { contentItems, languageVariants }; + }, + getReportFile(importResult: ImportResult): { filename: string; content: string } { + return { + filename: reportFilename, + content: JSON.stringify(getReportResult(importResult)) + }; } }; } diff --git a/lib/import/import.models.ts b/lib/import/import.models.ts index c934519..7119031 100644 --- a/lib/import/import.models.ts +++ b/lib/import/import.models.ts @@ -76,13 +76,14 @@ export type ImportTransformFunc = (data: { export interface ImportConfig extends ManagementClientConfig { readonly data: MigrationData; readonly externalIdGenerator?: ExternalIdGenerator; + readonly createReportFile?: boolean; readonly logger?: Logger; } export interface AssetToEdit { - migrationAsset: MigrationAsset; - targetAsset: Readonly; - replaceBinaryFile: boolean; + readonly migrationAsset: MigrationAsset; + readonly targetAsset: Readonly; + readonly replaceBinaryFile: boolean; } export type ImportedItem = ItemProcessingResult>; diff --git a/lib/toolkit/import.ts b/lib/toolkit/import.ts index 35d590d..e428f9b 100644 --- a/lib/toolkit/import.ts +++ b/lib/toolkit/import.ts @@ -1,5 +1,7 @@ +import chalk from 'chalk'; +import { writeFile } from 'fs/promises'; import { executeWithTrackingAsync } from '../core/index.js'; -import { ImportConfig, ImportResult, importManager } from '../import/index.js'; +import { ImportConfig, ImportResult, importManager as _importManager } from '../import/index.js'; import { libMetadata } from '../metadata.js'; export async function importAsync(config: ImportConfig): Promise { @@ -15,7 +17,19 @@ export async function importAsync(config: ImportConfig): Promise { details: {} }, func: async () => { - return await importManager(config).importAsync(); + const importManager = _importManager(config); + const importResult = await importManager.importAsync(); + + if (config.createReportFile) { + const reportFile = importManager.getReportFile(importResult); + await writeFile(reportFile.filename, reportFile.content); + config.logger?.log({ + type: 'writeFs', + message: `Report '${chalk.yellow(reportFile.filename)}' was created` + }); + } + + return importResult; }, logger: config.logger }); diff --git a/scripts/test/import-report.json b/scripts/test/import-report.json new file mode 100644 index 0000000..36809f1 --- /dev/null +++ b/scripts/test/import-report.json @@ -0,0 +1 @@ +{"errorsCount":1,"assets":{"count":0,"successful":[],"failed":[]},"contentItems":{"count":2,"successful":[{"codename":"warrior"},{"codename":"warrior2"}],"failed":[]},"languageVariants":{"count":2,"successful":[{"codename":"warrior2","language":{"codename":"en"},"type":{"codename":"movie"}}],"failed":[{"codename":"warrior","language":{"codename":"en"},"type":{"codename":"movie"},"error":"The provided request body is invalid. See 'validation_errors' for more information and specify a valid JSON object. The requested taxonomy term 'comedy_invalid' for the element 'taxonomy_snippet_test__test_taxonomy' was not found."}]}} \ No newline at end of file diff --git a/scripts/test/test-import.ts b/scripts/test/test-import.ts index e2d632b..fa1365e 100644 --- a/scripts/test/test-import.ts +++ b/scripts/test/test-import.ts @@ -1,5 +1,5 @@ import * as dotenv from 'dotenv'; -import { confirmImportAsync, extractAsync, importAsync, getDefaultLogger, handleError } from '../../lib/index.js'; +import { confirmImportAsync, extractAsync, getDefaultLogger, handleError, importAsync } from '../../lib/index.js'; import { getEnvironmentRequiredValue } from './utils/test.utils.js'; const run = async () => { @@ -26,7 +26,8 @@ const run = async () => { logger: log, data: data, environmentId: environmentId, - apiKey: apiKey + apiKey: apiKey, + createReportFile: true }); }; From fedc23ef66f0b409555e14cf5f0adde5d3a15b09 Mon Sep 17 00:00:00 2001 From: Richard Sustek Date: Tue, 10 Dec 2024 15:30:21 +0100 Subject: [PATCH 07/12] Adds readonly modifiers --- lib/import/import-manager.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/import/import-manager.ts b/lib/import/import-manager.ts index 1871f9e..2d24f32 100644 --- a/lib/import/import-manager.ts +++ b/lib/import/import-manager.ts @@ -253,7 +253,7 @@ export function importManager(config: ImportConfig) { languageVariants }; }, - getReportFile(importResult: ImportResult): { filename: string; content: string } { + getReportFile(importResult: ImportResult): { readonly filename: string; readonly content: string } { return { filename: reportFilename, content: JSON.stringify(getReportResult(importResult)) From 0383b525cc9bdf0c1d12291862fd12c23d828fb9 Mon Sep 17 00:00:00 2001 From: Richard Sustek Date: Wed, 11 Dec 2024 08:12:55 +0100 Subject: [PATCH 08/12] fix: Remove unnecessary handling for not found error code (fixes https://github.com/kontent-ai/kontent-ai-migration-toolkit/issues/20) --- lib/core/utils/http.utils.ts | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/lib/core/utils/http.utils.ts b/lib/core/utils/http.utils.ts index b7ce5b7..37f5369 100644 --- a/lib/core/utils/http.utils.ts +++ b/lib/core/utils/http.utils.ts @@ -1,9 +1,8 @@ import { HttpService, IRetryStrategyOptions } from '@kontent-ai/core-sdk'; -import { OriginalManagementError } from '../models/core.models.js'; import { match } from 'ts-pattern'; +import { OriginalManagementError } from '../models/core.models.js'; const rateExceededErrorCode: number = 10000; -const notFoundErrorCode: number = 10000; export const defaultHttpService: Readonly = new HttpService({ logErrorsToConsole: false @@ -19,8 +18,6 @@ export const defaultRetryStrategy: Readonly = { match(errorCode) // retry rate exceeded error .with(rateExceededErrorCode, () => true) - // do not retry errors indicating resource does not exist - .with(notFoundErrorCode, () => false) // if error code is set, do not retry the request .when( (errorCode) => errorCode >= 0, From 3bd9ac66b2abe0d5497e0b02e7ca8dabecef8ec2 Mon Sep 17 00:00:00 2001 From: Richard Sustek Date: Wed, 11 Dec 2024 09:03:40 +0100 Subject: [PATCH 09/12] Simplifies handling of 404 items within processing func --- lib/core/utils/processing-utils.ts | 13 ++++++++++--- lib/export/context/export-context-fetcher.ts | 6 ++---- lib/import/context/import-context-fetcher.ts | 13 +++++-------- 3 files changed, 17 insertions(+), 15 deletions(-) diff --git a/lib/core/utils/processing-utils.ts b/lib/core/utils/processing-utils.ts index 0e4ec71..17ef3e1 100644 --- a/lib/core/utils/processing-utils.ts +++ b/lib/core/utils/processing-utils.ts @@ -19,7 +19,7 @@ export async function processItemsAsync(data: { readonly logger: Logger; readonly items: Readonly; readonly parallelLimit: number; - readonly processAsync: (item: Readonly, logSpinner: LogSpinnerData) => Promise>; + readonly processAsync: (item: Readonly, logSpinner: LogSpinnerData) => Promise | '404'>; readonly itemInfo: (item: Readonly) => ItemInfo; }): Promise[]> { if (!data.items.length) { @@ -36,7 +36,7 @@ export async function processItemsAsync(data: { limit(() => { return data .processAsync(item, logSpinner) - .then((output) => { + .then((output) => { const itemInfo = data.itemInfo(item); const prefix = getPercentagePrefix(processedItemsCount, data.items.length); @@ -50,6 +50,13 @@ export async function processItemsAsync(data: { return output; }) .then>((outputItem) => { + if (outputItem === '404') { + return { + inputItem: item, + outputItem: undefined, + error: undefined + }; + } return { inputItem: item, outputItem: outputItem @@ -75,7 +82,7 @@ export async function processItemsAsync(data: { // Only '' promises at a time const resultItems = await Promise.all(requests); - const failedItemsCount = resultItems.filter((m) => !m.outputItem).length; + const failedItemsCount = resultItems.filter((m) => m.error).length; const failedText = failedItemsCount ? ` Failed (${chalk.red(failedItemsCount)}) items` : ``; logSpinner({ diff --git a/lib/export/context/export-context-fetcher.ts b/lib/export/context/export-context-fetcher.ts index aeeaaa4..c3682a9 100644 --- a/lib/export/context/export-context-fetcher.ts +++ b/lib/export/context/export-context-fetcher.ts @@ -280,7 +280,7 @@ export async function exportContextFetcherAsync(config: DefaultExportContextConf const getContentItemsByIdsAsync = async (itemIds: ReadonlySet): Promise[]> => { return ( - await processItemsAsync | '404'>({ + await processItemsAsync>({ logger: config.logger, action: 'Fetching content items', parallelLimit: 1, @@ -312,13 +312,12 @@ export async function exportContextFetcherAsync(config: DefaultExportContextConf }) ) .map((m) => m.outputItem) - .filter((m) => m !== '404') .filter(isNotUndefined); }; const getAssetsByIdsAsync = async (itemIds: ReadonlySet): Promise[]> => { return ( - await processItemsAsync | '404'>({ + await processItemsAsync>({ logger: config.logger, action: 'Fetching assets', parallelLimit: 1, @@ -350,7 +349,6 @@ export async function exportContextFetcherAsync(config: DefaultExportContextConf }) ) .map((m) => m.outputItem) - .filter((m) => m !== '404') .filter(isNotUndefined); }; diff --git a/lib/import/context/import-context-fetcher.ts b/lib/import/context/import-context-fetcher.ts index 6873042..209bcd0 100644 --- a/lib/import/context/import-context-fetcher.ts +++ b/lib/import/context/import-context-fetcher.ts @@ -150,7 +150,7 @@ export async function importContextFetcherAsync(config: ImportContextConfig) { migrationItems: readonly MigrationItem[] ): Promise => { return ( - await processItemsAsync({ + await processItemsAsync({ action: 'Fetching language variants', logger: config.logger, parallelLimit: 1, @@ -188,7 +188,6 @@ export async function importContextFetcherAsync(config: ImportContextConfig) { }) ) .map((m) => m.outputItem) - .filter((m) => m !== '404') .filter(isNotUndefined); }; @@ -196,7 +195,7 @@ export async function importContextFetcherAsync(config: ImportContextConfig) { itemCodenames: ReadonlySet ): Promise => { return ( - await processItemsAsync | '404'>({ + await processItemsAsync>({ action: 'Fetching content items', logger: config.logger, parallelLimit: 1, @@ -228,13 +227,12 @@ export async function importContextFetcherAsync(config: ImportContextConfig) { }) ) .map((m) => m.outputItem) - .filter(isNotUndefined) - .filter((m) => m !== '404'); + .filter(isNotUndefined); }; const getAssetsByCodenamesAsync = async (assetCodenames: ReadonlySet): Promise => { return ( - await processItemsAsync | '404'>({ + await processItemsAsync>({ action: 'Fetching assets', logger: config.logger, parallelLimit: 1, @@ -266,8 +264,7 @@ export async function importContextFetcherAsync(config: ImportContextConfig) { }) ) .map((m) => m.outputItem) - .filter(isNotUndefined) - .filter((m) => m !== '404'); + .filter(isNotUndefined); }; const getVariantState = (languageVariant: Readonly): LanguageVariantStateData => { From 3595f8cf64e25409928f91566299c538d12aac3b Mon Sep 17 00:00:00 2001 From: Richard Sustek Date: Wed, 11 Dec 2024 09:42:52 +0100 Subject: [PATCH 10/12] Updates filter condition for getting errored items --- lib/import/import-manager.ts | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/lib/import/import-manager.ts b/lib/import/import-manager.ts index 2d24f32..f046197 100644 --- a/lib/import/import-manager.ts +++ b/lib/import/import-manager.ts @@ -97,10 +97,10 @@ export function importManager(config: ImportConfig) { const getReportResult = (importResult: ImportResult): ReportResult => { return { errorsCount: - importResult.editedAssets.filter((m) => !m.outputItem).length + - importResult.uploadedAssets.filter((m) => !m.outputItem).length + - importResult.contentItems.filter((m) => !m.outputItem).length + - importResult.languageVariants.filter((m) => !m.outputItem).length, + importResult.editedAssets.filter((m) => m.error).length + + importResult.uploadedAssets.filter((m) => m.error).length + + importResult.contentItems.filter((m) => m.error).length + + importResult.languageVariants.filter((m) => m.error).length, assets: { count: importResult.uploadedAssets.length + importResult.editedAssets.length, successful: [ @@ -113,7 +113,7 @@ export function importManager(config: ImportConfig) { }), failed: [ ...importResult.uploadedAssets - .filter((m) => !m.outputItem) + .filter((m) => m.error) .map((m) => { return { codename: m.inputItem.codename, @@ -121,7 +121,7 @@ export function importManager(config: ImportConfig) { }; }), ...importResult.editedAssets - .filter((m) => !m.outputItem) + .filter((m) => m.error) .map((m) => { return { codename: m.inputItem.migrationAsset.codename, @@ -140,7 +140,7 @@ export function importManager(config: ImportConfig) { }; }), failed: importResult.contentItems - .filter((m) => !m.outputItem) + .filter((m) => m.error) .map((m) => { return { codename: m.inputItem.system.codename, @@ -161,7 +161,7 @@ export function importManager(config: ImportConfig) { }; }), failed: importResult.languageVariants - .filter((m) => !m.outputItem) + .filter((m) => m.error) .map((m) => { return { codename: m.inputItem.system.codename, From 0aeff7a7c2f1088cd1a2ba9fe518cf398a07224a Mon Sep 17 00:00:00 2001 From: Richard Sustek Date: Wed, 11 Dec 2024 14:16:39 +0100 Subject: [PATCH 11/12] fix: Fixes import when multiple language variants reference same content item (fixes https://github.com/kontent-ai/kontent-ai-migration-toolkit/issues/19) --- lib/import/importers/content-items-importer.ts | 15 +++++++++++++-- scripts/test/import-report.json | 2 +- scripts/test/test-export.ts | 4 ++++ 3 files changed, 18 insertions(+), 3 deletions(-) diff --git a/lib/import/importers/content-items-importer.ts b/lib/import/importers/content-items-importer.ts index 9bd034d..eca6d52 100644 --- a/lib/import/importers/content-items-importer.ts +++ b/lib/import/importers/content-items-importer.ts @@ -102,7 +102,18 @@ export function contentItemsImporter(data: { }; const importAsync = async (): Promise => { - const contentItemsToImport = data.importContext.categorizedImportData.contentItems; + // Only import unique content items based on their codename. The input may contain items in various language variants which share + // the same underlying content item. + const contentItemsToImport = data.importContext.categorizedImportData.contentItems.reduce>( + (filteredItems, item) => { + if (filteredItems.some((m) => m.system.codename === item.system.codename)) { + return filteredItems; + } + + return [...filteredItems, item]; + }, + [] + ); data.logger.log({ type: 'info', @@ -117,7 +128,7 @@ export function contentItemsImporter(data: { itemInfo: (item) => { return { itemType: 'contentItem', - title: `${item.system.codename} -> ${item.system.language.codename}` + title: `${item.system.codename} -> ${item.system.type.codename}` }; }, processAsync: async (item, logSpinner) => { diff --git a/scripts/test/import-report.json b/scripts/test/import-report.json index 36809f1..ee8ecf5 100644 --- a/scripts/test/import-report.json +++ b/scripts/test/import-report.json @@ -1 +1 @@ -{"errorsCount":1,"assets":{"count":0,"successful":[],"failed":[]},"contentItems":{"count":2,"successful":[{"codename":"warrior"},{"codename":"warrior2"}],"failed":[]},"languageVariants":{"count":2,"successful":[{"codename":"warrior2","language":{"codename":"en"},"type":{"codename":"movie"}}],"failed":[{"codename":"warrior","language":{"codename":"en"},"type":{"codename":"movie"},"error":"The provided request body is invalid. See 'validation_errors' for more information and specify a valid JSON object. The requested taxonomy term 'comedy_invalid' for the element 'taxonomy_snippet_test__test_taxonomy' was not found."}]}} \ No newline at end of file +{"errorsCount":0,"assets":{"count":0,"successful":[],"failed":[]},"contentItems":{"count":1,"successful":[{"codename":"warrior"}],"failed":[]},"languageVariants":{"count":2,"successful":[{"codename":"warrior","language":{"codename":"en"},"type":{"codename":"movie"}},{"codename":"warrior","language":{"codename":"es"},"type":{"codename":"movie"}}],"failed":[]}} \ No newline at end of file diff --git a/scripts/test/test-export.ts b/scripts/test/test-export.ts index a130162..c35bf39 100644 --- a/scripts/test/test-export.ts +++ b/scripts/test/test-export.ts @@ -14,6 +14,10 @@ const run = async () => { { itemCodename: getEnvironmentRequiredValue('item'), languageCodename: getEnvironmentRequiredValue('language') + }, + { + itemCodename: getEnvironmentRequiredValue('item'), + languageCodename: getEnvironmentRequiredValue('languageSecondary') } ]; From a5dcc606651f75207fd169f55a668a60e9f2fe94 Mon Sep 17 00:00:00 2001 From: Richard Sustek Date: Wed, 11 Dec 2024 15:52:28 +0100 Subject: [PATCH 12/12] Use discriminated union for ItemProcessingResult --- lib/core/models/core.models.ts | 20 +++++++++--- lib/core/utils/processing-utils.ts | 14 ++++----- lib/export/context/export-context-fetcher.ts | 13 ++++---- lib/export/export-manager.ts | 4 +-- lib/import/context/import-context-fetcher.ts | 13 ++++---- lib/import/import-manager.ts | 32 ++++++++++---------- 6 files changed, 52 insertions(+), 44 deletions(-) diff --git a/lib/core/models/core.models.ts b/lib/core/models/core.models.ts index 1b7129b..ffc9f59 100644 --- a/lib/core/models/core.models.ts +++ b/lib/core/models/core.models.ts @@ -146,8 +146,18 @@ export interface OriginalManagementError { }; } -export type ItemProcessingResult = { - readonly inputItem: InputItem; - readonly outputItem: OutputItem | undefined; - readonly error?: unknown; -}; +export type ItemProcessingResult = + | { + readonly state: 'valid'; + readonly inputItem: InputItem; + readonly outputItem: OutputItem; + } + | { + readonly state: 'error'; + readonly inputItem: InputItem; + readonly error: unknown; + } + | { + readonly state: '404'; + readonly inputItem: InputItem; + }; diff --git a/lib/core/utils/processing-utils.ts b/lib/core/utils/processing-utils.ts index 17ef3e1..a6fda3a 100644 --- a/lib/core/utils/processing-utils.ts +++ b/lib/core/utils/processing-utils.ts @@ -52,20 +52,20 @@ export async function processItemsAsync(data: { .then>((outputItem) => { if (outputItem === '404') { return { - inputItem: item, - outputItem: undefined, - error: undefined + state: '404', + inputItem: item }; } return { inputItem: item, - outputItem: outputItem + outputItem: outputItem, + state: 'valid' }; }) .catch>((error) => { return { + state: 'error', inputItem: item, - outputItem: undefined, error: error }; }); @@ -82,13 +82,13 @@ export async function processItemsAsync(data: { // Only '' promises at a time const resultItems = await Promise.all(requests); - const failedItemsCount = resultItems.filter((m) => m.error).length; + const failedItemsCount = resultItems.filter((m) => m.state === 'error').length; const failedText = failedItemsCount ? ` Failed (${chalk.red(failedItemsCount)}) items` : ``; logSpinner({ type: 'info', message: `Completed '${chalk.yellow(data.action)}'. Successfully processed (${chalk.green( - resultItems.filter((m) => m.outputItem).length + resultItems.filter((m) => m.state === 'valid').length )}) items.${failedText}` }); diff --git a/lib/export/context/export-context-fetcher.ts b/lib/export/context/export-context-fetcher.ts index c3682a9..f1ccc56 100644 --- a/lib/export/context/export-context-fetcher.ts +++ b/lib/export/context/export-context-fetcher.ts @@ -12,7 +12,6 @@ import { findRequired, FlattenedContentType, is404Error, - isNotUndefined, ItemStateInSourceEnvironmentById, LogSpinnerData, managementClientUtils, @@ -274,8 +273,8 @@ export async function exportContextFetcherAsync(config: DefaultExportContextConf } }) ) - .map((m) => m.outputItem) - .filter(isNotUndefined); + .filter((m) => m.state === 'valid') + .map((m) => m.outputItem); }; const getContentItemsByIdsAsync = async (itemIds: ReadonlySet): Promise[]> => { @@ -311,8 +310,8 @@ export async function exportContextFetcherAsync(config: DefaultExportContextConf } }) ) - .map((m) => m.outputItem) - .filter(isNotUndefined); + .filter((m) => m.state === 'valid') + .map((m) => m.outputItem); }; const getAssetsByIdsAsync = async (itemIds: ReadonlySet): Promise[]> => { @@ -348,8 +347,8 @@ export async function exportContextFetcherAsync(config: DefaultExportContextConf } }) ) - .map((m) => m.outputItem) - .filter(isNotUndefined); + .filter((m) => m.state === 'valid') + .map((m) => m.outputItem); }; const getItemStatesAsync = async (itemIds: ReadonlySet): Promise => { diff --git a/lib/export/export-manager.ts b/lib/export/export-manager.ts index da938e4..9a117c2 100644 --- a/lib/export/export-manager.ts +++ b/lib/export/export-manager.ts @@ -229,8 +229,8 @@ export function exportManager(config: ExportConfig) { } }) ) - .map((m) => m.outputItem) - .filter(isNotUndefined); + .filter((m) => m.state === 'valid') + .map((m) => m.outputItem); }; return { diff --git a/lib/import/context/import-context-fetcher.ts b/lib/import/context/import-context-fetcher.ts index 209bcd0..dc98c01 100644 --- a/lib/import/context/import-context-fetcher.ts +++ b/lib/import/context/import-context-fetcher.ts @@ -14,7 +14,6 @@ import { WorkflowStep, findRequired, is404Error, - isNotUndefined, managementClientUtils, processItemsAsync, runMapiRequestAsync, @@ -187,8 +186,8 @@ export async function importContextFetcherAsync(config: ImportContextConfig) { } }) ) - .map((m) => m.outputItem) - .filter(isNotUndefined); + .filter((m) => m.state === 'valid') + .map((m) => m.outputItem); }; const getContentItemsByCodenamesAsync = async ( @@ -226,8 +225,8 @@ export async function importContextFetcherAsync(config: ImportContextConfig) { } }) ) - .map((m) => m.outputItem) - .filter(isNotUndefined); + .filter((m) => m.state === 'valid') + .map((m) => m.outputItem); }; const getAssetsByCodenamesAsync = async (assetCodenames: ReadonlySet): Promise => { @@ -263,8 +262,8 @@ export async function importContextFetcherAsync(config: ImportContextConfig) { } }) ) - .map((m) => m.outputItem) - .filter(isNotUndefined); + .filter((m) => m.state === 'valid') + .map((m) => m.outputItem); }; const getVariantState = (languageVariant: Readonly): LanguageVariantStateData => { diff --git a/lib/import/import-manager.ts b/lib/import/import-manager.ts index f046197..a449b86 100644 --- a/lib/import/import-manager.ts +++ b/lib/import/import-manager.ts @@ -97,15 +97,15 @@ export function importManager(config: ImportConfig) { const getReportResult = (importResult: ImportResult): ReportResult => { return { errorsCount: - importResult.editedAssets.filter((m) => m.error).length + - importResult.uploadedAssets.filter((m) => m.error).length + - importResult.contentItems.filter((m) => m.error).length + - importResult.languageVariants.filter((m) => m.error).length, + importResult.editedAssets.filter((m) => m.state === 'error').length + + importResult.uploadedAssets.filter((m) => m.state === 'error').length + + importResult.contentItems.filter((m) => m.state === 'error').length + + importResult.languageVariants.filter((m) => m.state === 'error').length, assets: { count: importResult.uploadedAssets.length + importResult.editedAssets.length, successful: [ - ...importResult.uploadedAssets.filter((m) => m.outputItem).map((m) => m.inputItem.codename), - ...importResult.editedAssets.filter((m) => m.outputItem).map((m) => m.inputItem.migrationAsset.codename) + ...importResult.uploadedAssets.filter((m) => m.state === 'valid').map((m) => m.inputItem.codename), + ...importResult.editedAssets.filter((m) => m.state === 'valid').map((m) => m.inputItem.migrationAsset.codename) ].map((m) => { return { codename: m @@ -113,19 +113,19 @@ export function importManager(config: ImportConfig) { }), failed: [ ...importResult.uploadedAssets - .filter((m) => m.error) + .filter((m) => m.state === 'error') .map((m) => { return { codename: m.inputItem.codename, - error: extractErrorData(m.error).message + error: extractErrorData(m.state === 'error').message }; }), ...importResult.editedAssets - .filter((m) => m.error) + .filter((m) => m.state === 'error') .map((m) => { return { codename: m.inputItem.migrationAsset.codename, - error: extractErrorData(m.error).message + error: extractErrorData(m.state === 'error').message }; }) ] @@ -133,26 +133,26 @@ export function importManager(config: ImportConfig) { contentItems: { count: importResult.contentItems.length, successful: importResult.contentItems - .filter((m) => m.outputItem) + .filter((m) => m.state === 'valid') .map((m) => { return { codename: m.inputItem.system.codename }; }), failed: importResult.contentItems - .filter((m) => m.error) + .filter((m) => m.state === 'error') .map((m) => { return { codename: m.inputItem.system.codename, type: m.inputItem.system.type, - error: extractErrorData(m.error).message + error: extractErrorData(m.state === 'error').message }; }) }, languageVariants: { count: importResult.languageVariants.length, successful: importResult.languageVariants - .filter((m) => m.outputItem) + .filter((m) => m.state === 'valid') .map((m) => { return { codename: m.inputItem.system.codename, @@ -161,13 +161,13 @@ export function importManager(config: ImportConfig) { }; }), failed: importResult.languageVariants - .filter((m) => m.error) + .filter((m) => m.state === 'error') .map((m) => { return { codename: m.inputItem.system.codename, language: m.inputItem.system.language, type: m.inputItem.system.type, - error: extractErrorData(m.error).message + error: extractErrorData(m.state === 'error').message }; }) }