feat: Skips failed imported items, supports detailed log in console / FS indicating what failed and why, extends result models for import functionality

lib/core/logs/loggers.ts

@@ -1,7 +1,7 @@
 import chalk, { ChalkInstance } from 'chalk';
+import { match, P } from 'ts-pattern';
 import { Logger, LogSpinnerMessage } from '../models/log.models.js';
 import { getCurrentEnvironment } from '../utils/global.utils.js';
-import { match, P } from 'ts-pattern';
 
 const originalWarn = console.warn;
 
@@ -55,6 +55,10 @@ const defaultBrowserLogger: Logger = {
 };
 
 function getLogDataMessage(data: LogSpinnerMessage): string {
+    if (!data.type) {
+        return data.message;
+    }
+
     const color = match(data.type)
         .returnType<ChalkInstance>()
         .with('info', () => chalk.cyan)

lib/core/models/core.models.ts

@@ -145,3 +145,19 @@ export interface OriginalManagementError {
         };
     };
 }
+
+export type ItemProcessingResult<InputItem, OutputItem> =
+    | {
+          readonly state: 'valid';
+          readonly inputItem: InputItem;
+          readonly outputItem: OutputItem;
+      }
+    | {
+          readonly state: 'error';
+          readonly inputItem: InputItem;
+          readonly error: unknown;
+      }
+    | {
+          readonly state: '404';
+          readonly inputItem: InputItem;
+      };

lib/core/models/log.models.ts

@@ -1,4 +1,4 @@
-import { MapiAction, MapiType, MigrationItemType } from '../index.js';
+import { LiteralUnion, MapiAction, MapiType, MigrationItemType } from '../index.js';
 
 export type DebugType =
     | 'error'
@@ -17,7 +17,7 @@ export type DebugType =
     | MapiAction;
 
 export interface LogMessage {
-    readonly type: DebugType;
+    readonly type?: LiteralUnion<DebugType>;
     readonly message: string;
 }
 

lib/core/utils/http.utils.ts

@@ -1,9 +1,8 @@
 import { HttpService, IRetryStrategyOptions } from '@kontent-ai/core-sdk';
-import { OriginalManagementError } from '../models/core.models.js';
 import { match } from 'ts-pattern';
+import { OriginalManagementError } from '../models/core.models.js';
 
 const rateExceededErrorCode: number = 10000;
-const notFoundErrorCode: number = 10000;
 
 export const defaultHttpService: Readonly<HttpService> = new HttpService({
     logErrorsToConsole: false
@@ -19,8 +18,6 @@ export const defaultRetryStrategy: Readonly<IRetryStrategyOptions> = {
             match(errorCode)
                 // retry rate exceeded error
                 .with(rateExceededErrorCode, () => true)
-                // do not retry errors indicating resource does not exist
-                .with(notFoundErrorCode, () => false)
                 // if error code is set, do not retry the request
                 .when(
                     (errorCode) => errorCode >= 0,

lib/core/utils/processing-utils.ts

@@ -1,6 +1,6 @@
 import chalk from 'chalk';
 import pLimit from 'p-limit';
-import { ItemInfo } from '../models/core.models.js';
+import { ItemInfo, ItemProcessingResult } from '../models/core.models.js';
 import { LogSpinnerData, Logger } from '../models/log.models.js';
 
 type ProcessSetAction =
@@ -19,9 +19,9 @@ export async function processItemsAsync<InputItem, OutputItem>(data: {
     readonly logger: Logger;
     readonly items: Readonly<InputItem[]>;
     readonly parallelLimit: number;
-    readonly processAsync: (item: Readonly<InputItem>, logSpinner: LogSpinnerData) => Promise<Readonly<OutputItem>>;
+    readonly processAsync: (item: Readonly<InputItem>, logSpinner: LogSpinnerData) => Promise<Readonly<OutputItem> | '404'>;
     readonly itemInfo: (item: Readonly<InputItem>) => ItemInfo;
-}): Promise<readonly OutputItem[]> {
+}): Promise<readonly ItemProcessingResult<InputItem, OutputItem>[]> {
     if (!data.items.length) {
         return [];
     }
@@ -32,21 +32,43 @@ export async function processItemsAsync<InputItem, OutputItem>(data: {
         const limit = pLimit(data.parallelLimit);
         let processedItemsCount: number = 1;
 
-        const requests: Promise<OutputItem>[] = data.items.map((item) =>
+        const requests: Promise<ItemProcessingResult<InputItem, OutputItem>>[] = data.items.map((item) =>
             limit(() => {
-                return data.processAsync(item, logSpinner).then((output) => {
-                    const itemInfo = data.itemInfo(item);
-                    const prefix = getPercentagePrefix(processedItemsCount, data.items.length);
+                return data
+                    .processAsync(item, logSpinner)
+                    .then<OutputItem | '404'>((output) => {
+                        const itemInfo = data.itemInfo(item);
+                        const prefix = getPercentagePrefix(processedItemsCount, data.items.length);
 
-                    logSpinner({
-                        prefix: prefix,
-                        message: itemInfo.title,
-                        type: itemInfo.itemType
-                    });
+                        logSpinner({
+                            prefix: prefix,
+                            message: itemInfo.title,
+                            type: itemInfo.itemType
+                        });
 
-                    processedItemsCount++;
-                    return output;
-                });
+                        processedItemsCount++;
+                        return output;
+                    })
+                    .then<ItemProcessingResult<InputItem, OutputItem>>((outputItem) => {
+                        if (outputItem === '404') {
+                            return {
+                                state: '404',
+                                inputItem: item
+                            };
+                        }
+                        return {
+                            inputItem: item,
+                            outputItem: outputItem,
+                            state: 'valid'
+                        };
+                    })
+                    .catch<ItemProcessingResult<InputItem, OutputItem>>((error) => {
+                        return {
+                            state: 'error',
+                            inputItem: item,
+                            error: error
+                        };
+                    });
             })
         );
 
@@ -58,11 +80,19 @@ export async function processItemsAsync<InputItem, OutputItem>(data: {
         });
 
         // Only '<parallelLimit>' promises at a time
-        const outputItems = await Promise.all(requests);
+        const resultItems = await Promise.all(requests);
+
+        const failedItemsCount = resultItems.filter((m) => m.state === 'error').length;
+        const failedText = failedItemsCount ? ` Failed (${chalk.red(failedItemsCount)}) items` : ``;
 
-        logSpinner({ type: 'info', message: `Completed '${chalk.yellow(data.action)}' (${outputItems.length})` });
+        logSpinner({
+            type: 'info',
+            message: `Completed '${chalk.yellow(data.action)}'. Successfully processed (${chalk.green(
+                resultItems.filter((m) => m.state === 'valid').length
+            )}) items.${failedText}`
+        });
 
-        return outputItems;
+        return resultItems;
     });
 }
 

lib/export/context/export-context-fetcher.ts

@@ -1,34 +1,33 @@
 import {
-    WorkflowModels,
-    LanguageVariantModels,
-    ContentItemModels,
     AssetModels,
     CollectionModels,
-    LanguageModels
+    ContentItemModels,
+    LanguageModels,
+    LanguageVariantModels,
+    WorkflowModels
 } from '@kontent-ai/management-sdk';
 import chalk from 'chalk';
 import {
-    processItemsAsync,
-    runMapiRequestAsync,
-    is404Error,
-    ItemStateInSourceEnvironmentById,
     AssetStateInSourceEnvironmentById,
+    findRequired,
     FlattenedContentType,
-    isNotUndefined,
-    managementClientUtils,
+    is404Error,
+    ItemStateInSourceEnvironmentById,
     LogSpinnerData,
-    findRequired,
+    managementClientUtils,
+    processItemsAsync,
+    runMapiRequestAsync,
     workflowHelper
 } from '../../core/index.js';
 import { itemsExtractionProcessor } from '../../translation/index.js';
 import {
     DefaultExportContextConfig,
     ExportContext,
     ExportContextEnvironmentData,
-    SourceExportItem,
     ExportItem,
+    ExportItemVersion,
     GetFlattenedElementByIds,
-    ExportItemVersion
+    SourceExportItem
 } from '../export.models.js';
 import { throwErrorForItemRequest } from '../utils/export.utils.js';
 
@@ -173,11 +172,11 @@ export async function exportContextFetcherAsync(config: DefaultExportContextConf
         readonly contentItem: Readonly<ContentItemModels.ContentItem>;
         readonly languageVariant: Readonly<LanguageVariantModels.ContentItemLanguageVariant>;
     }): {
-        collection: Readonly<CollectionModels.Collection>;
-        language: Readonly<LanguageModels.LanguageModel>;
-        workflow: Readonly<WorkflowModels.Workflow>;
-        contentType: Readonly<FlattenedContentType>;
-        workflowStepCodename: string;
+        readonly collection: Readonly<CollectionModels.Collection>;
+        readonly language: Readonly<LanguageModels.LanguageModel>;
+        readonly workflow: Readonly<WorkflowModels.Workflow>;
+        readonly contentType: Readonly<FlattenedContentType>;
+        readonly workflowStepCodename: string;
     } => {
         const collection = findRequired(
             environmentData.collections,
@@ -234,49 +233,53 @@ export async function exportContextFetcherAsync(config: DefaultExportContextConf
             message: `Preparing '${chalk.yellow(config.exportItems.length.toString())}' items for export`
         });
 
-        return await processItemsAsync<SourceExportItem, ExportItem>({
-            logger: config.logger,
-            action: 'Preparing content items & language variants',
-            parallelLimit: 1,
-            itemInfo: (input) => {
-                return {
-                    title: `${input.itemCodename} (${input.languageCodename})`,
-                    itemType: 'exportItem'
-                };
-            },
-            items: exportItems,
-            processAsync: async (requestItem, logSpinner) => {
-                const contentItem = await getContentItemAsync(requestItem, logSpinner);
-                const versions = await getExportItemVersionsAsync(requestItem, contentItem, logSpinner);
-
-                // get shared attributes from any version
-                const anyVersion = versions[0];
-                if (!anyVersion) {
-                    throwErrorForItemRequest(requestItem, `Expected at least 1 version of the content item`);
-                }
+        return (
+            await processItemsAsync<SourceExportItem, ExportItem>({
+                logger: config.logger,
+                action: 'Preparing content items & language variants',
+                parallelLimit: 1,
+                itemInfo: (input) => {
+                    return {
+                        title: `${input.itemCodename} (${input.languageCodename})`,
+                        itemType: 'exportItem'
+                    };
+                },
+                items: exportItems,
+                processAsync: async (requestItem, logSpinner) => {
+                    const contentItem = await getContentItemAsync(requestItem, logSpinner);
+                    const versions = await getExportItemVersionsAsync(requestItem, contentItem, logSpinner);
+
+                    // get shared attributes from any version
+                    const anyVersion = versions[0];
+                    if (!anyVersion) {
+                        throwErrorForItemRequest(requestItem, `Expected at least 1 version of the content item`);
+                    }
 
-                const { collection, contentType, language, workflow } = validateExportItem({
-                    sourceItem: requestItem,
-                    contentItem: contentItem,
-                    languageVariant: anyVersion.languageVariant
-                });
+                    const { collection, contentType, language, workflow } = validateExportItem({
+                        sourceItem: requestItem,
+                        contentItem: contentItem,
+                        languageVariant: anyVersion.languageVariant
+                    });
 
-                return {
-                    contentItem,
-                    versions,
-                    contentType,
-                    requestItem,
-                    workflow,
-                    collection,
-                    language
-                };
-            }
-        });
+                    return {
+                        contentItem,
+                        versions,
+                        contentType,
+                        requestItem,
+                        workflow,
+                        collection,
+                        language
+                    };
+                }
+            })
+        )
+            .filter((m) => m.state === 'valid')
+            .map((m) => m.outputItem);
     };
 
     const getContentItemsByIdsAsync = async (itemIds: ReadonlySet<string>): Promise<readonly Readonly<ContentItemModels.ContentItem>[]> => {
         return (
-            await processItemsAsync<string, Readonly<ContentItemModels.ContentItem> | undefined>({
+            await processItemsAsync<string, Readonly<ContentItemModels.ContentItem>>({
                 logger: config.logger,
                 action: 'Fetching content items',
                 parallelLimit: 1,
@@ -302,16 +305,18 @@ export async function exportContextFetcherAsync(config: DefaultExportContextConf
                             throw error;
                         }
 
-                        return undefined;
+                        return '404';
                     }
                 }
             })
-        ).filter(isNotUndefined);
+        )
+            .filter((m) => m.state === 'valid')
+            .map((m) => m.outputItem);
     };
 
     const getAssetsByIdsAsync = async (itemIds: ReadonlySet<string>): Promise<readonly Readonly<AssetModels.Asset>[]> => {
         return (
-            await processItemsAsync<string, Readonly<AssetModels.Asset> | undefined>({
+            await processItemsAsync<string, Readonly<AssetModels.Asset>>({
                 logger: config.logger,
                 action: 'Fetching assets',
                 parallelLimit: 1,
@@ -337,11 +342,13 @@ export async function exportContextFetcherAsync(config: DefaultExportContextConf
                             throw error;
                         }
 
-                        return undefined;
+                        return '404';
                     }
                 }
             })
-        ).filter(isNotUndefined);
+        )
+            .filter((m) => m.state === 'valid')
+            .map((m) => m.outputItem);
     };
 
     const getItemStatesAsync = async (itemIds: ReadonlySet<string>): Promise<readonly ItemStateInSourceEnvironmentById[]> => {